GAM 5.01

See example failure at: https://travis-ci.org/github/jay0lee/GAM/jobs/667171534#L879

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - pinned
+  - enhancement
+  - help wanted
+  - security
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs.
+  
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/pythonpackage.yml

@@ -1,24 +0,0 @@
-name: GAM
-
-on: [push]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-18.04, ubuntu-16.04, windows-2019, windows-2016, macOS-10.14]
-        python-version: [3.5, 3.6, 3.7]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up $(( matrix.runs-on )) Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r src/requirements.txt
-    - name: Run GAM
-      run: |
-        python src/gam.py version extended

.travis.yml

@@ -1,9 +1,13 @@
 if: tag IS blank
+os: linux
+language: python
 
 env:
   global:
-    - BUILD_PYTHON_VERSION=3.7.4
-    - BUILD_OPENSSL_VERSION=1.1.1d
+    - BUILD_PYTHON_VERSION=3.8.2
+    - MIN_PYTHON_VERSION=3.8.1
+    - BUILD_OPENSSL_VERSION=1.1.1e
+    - MIN_OPENSSL_VERSION=1.1.1d
     - PATCHELF_VERSION=0.9
     - PYINSTALLER_VERSION=3.5
     - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
@@ -22,7 +26,7 @@ env:
     - secure: "Is2Lv/rxKKrXnxFns9KQYseD02tjY9qbgSteVtJavG1cLJDvkTwb6X+Thvgo4cxk5fQPiXScrQaYzHjVVuNleD+dyD1HC/8CU2Xq+tjBhPjdcccHFSbk06DpcETmLGRyMORXG9JkYlgXHLLXtu/9icppWEHgra+zvchVL2YDhofYne8FMNBb/lq0AAC2wgzAS33tW1+57HaYnzl0hf6+Q6lwqoH2/aTfGMRFDgyJ0HK+5IVfLnQJ+OuGFSrj8/0FWSggR5+EXDIddDgovFgaCghMjHYp21bzn0eIAJtuNFFultwk/UC1lT9joXTKEzgLTAh+w13yz1T3x9rNuv6FDKCotBIS/ZDtPmgvyZ8xvB4SzyULnTRSVn7YvspKR6PAO/qxGNudUD5H8tRKer4qKnKjHzSUcVBlRHb4yE6FqqY1Z9RLEcomWO43nJwb6saNHR9BYedyi0gA+EbA+P259QFClW1dWEQ2LQhDa+0VRssOqZ0BQblPFyz+e5Vc9kfAMbOuoss8fjkiYv+twXv7nT27xrVT5okfKDSiy5opZD6d36N5FibZPYiMrVx00YZdkFB+5EqQuJ7lqKUMkZJTeApLzj+h+/4aAOWd3paj5ghv7m+9ReohsNKFHyjaSy97RhMAZjzqgMMdD8rjUSKDhvNKvvQECWHlaXwL129GB0s="
     - secure: "l7vWxfcu1RgXbStq76Mzz2I5Iu4e31729OyLYqulXZzft3wO+idvgQKy/JSwajiKgOxlpBuI0wrncgIUYshcRvE4yB0y9+QIMDTegJzTADtRSUyVNCIZfTgvtOvzrlW0iCdVsxLBtYcJWJVPdjF2q59ED1jahd1AuJJqX5e61gfr0eZ9+cNiHbX1u3VpmGchFNWQF4KvebE4WKs5xWEds+AtbTODdQq3H6kKQK3fTVJnbz6WMO+bEWgWI7orfSE20lku/3Q3eMLhNOcPwH8WnUoTTvDWol5Cq5NfPhkKF5aV7kIbNXkxswM7yBPAPumBiXdM1BHpfd4+0YQ/fqRtnqxw85HEYpT8dRewHimCl4IccgGCR+G0tK8RNleKL28GWrz86gRVpXMEhOU3ILwb8as3SdQWLwhXXy5uKGJmsdrCNAH4/8eu3XczO5VN2wOo7narYuBgGcl2eLW9TOLyNVKFKxbQnDLBiOybLNoOV42DCRYt7v3Eknkv1Z0dmX6n23q1z9if9kkfAFgRQWsTbNZyWeIwWuX8b2a0Zq0znS3JflSKxzqS7RHADfBOJEVM86AiGkw0XkR4o31yDrY+zOrkJ3GxfV/HpqG7LzCd9tFeexanneDCE1FJhCkDjpnc0Mdyx+1gVf+u2MkgrU0BbCf2EESBAlC/FTRvxTmk/6s="
     - secure: "bvtQ348bxKwTtB8X0zMxeTsM0jEJozbS6/rzH/88Fk90a+KO8SdXen0Kj9/LahV4duMn2xTTRmxMCVj424FbcVTgBkJpIO7btqTcNASORkmK+9wGTK6Bgb9R5sHLbVrbrMYJzsMQR0MWxE8ibPLlyom+ssUIqr7HAjnRyYSDiKChGhgBfdE/0G9OE/DuQaU/ZkTuEYfc4527QNLJ8Bt1aLDdCHJLxzCojNaTHB215Tz7dmpnJjWa45BXxkMwLTpxIJuY7wA7K+2UBJfLvZv5QiPYyeUlosV7FwhCN9e90+dc2x8HPJbwe/Ysv5dm8/FfNTgTe6IkKw1z9kev9/W1tYCtqMzn9GnlIH2000ZUomhHKbc4UUl8sskF2jGr14rSz7pfaTyvXNvK7nKLKM7mZAO/cAaQvZoGEx0s6B7a7YX38NmJWcM0UUD603n4G9uVZNoxjyr8HWC6pho8MY8/u5aFKMPd+C8DaDWeSzNJDZRyi62v+t2iyyZuQUtVXtnDpv8GQW4tYFIJCkkrTm5VNNgD2x3WRsEr3LsBa3BonOmi3a1VrdIpyidDzBEWFUR59a/C0lU5J2UX5W1vu/Pnr9/8p898sDvvDbluz47OR9xoVOjtmGdt/ENJa4EOY1q/eRIN6zrt/cwwjIKgTcYOVk7DFDMXQuYlDeEFuHSJ5Xs="
-    - secure: "I05ZVphCT2Alu4najchCjDLIgflsdSCR/Z20OkMbcESprwJB5h7nb64kuVkqPQT7GeMcXMnK1xahx0KvE1Gn8OqOlDPt5WgnvcYERnVOKEJabmRXws8nM20AehwAzRoc6uDhaTnmwHRuliPSeuxiwY2Rkyj5hdyvEMzaOSY+qoIOYtDYQu0MM1XIC3cAjSbEfEGuqnKcN0p9VG0l7WscEUtcKPSFBShOlJZWDiHy7lisgO6OEzvAeDpZU/1ZdljXBl0RoDX8l4LqH4SzQqE68kK7qqIhZNmx4hkrh7qtx8tVqVKy/R3IUYhdxrJ84eJxrex6RQb9SgJQRGwi/3+rkKxVS41KiQ4quJHJhnqd2A1+5rDMN/fsjbjtM1I3VIl6w4oaT1C1QtwSjQOvuZCKDLIszVutW0PuGHveNyjwqQSm2PboOkLTg0dNrFIflT2hGMynv1s19WzGY8qQDP2UgODNR4EQVBy0PRk43ne5W/AFtzY1wUUUCBfKi7WYi4uURG/1Xu456GEElbIOwx9F1nC0s3z6ZFVJ3bVbmFxUlHtCyL5wJ3fiVHkAoPSygVARvv2UhHmXvJXXjxBMOvsnZZbpjvcbTeZoguffxMZEMNhuI4W22WjLdVTLxi9uh1VTdm8vkfAeNjMPw53gwPaSz7Jv6A82LFqpmbwAHxNAEy8="
+    - secure: "VWY3aJGMFB+E5ObJpUUXNQrnkEpA1pbfey8Z0pe9X/n5ubmDBIAfCPP7hqT/lAF/1hj5+HA6oWEa2FzeJZOL9q8cuymHzkehiX9n3f6GktirMtEoG3Hk/CiW/ZxKf9KyBNRVsqsOiXvfzMcMSU6ZrSV+4ZHIVl0zO5cehSaaiTTqVU5K474N5TCc8SAvpSRpxjZVRXy8rWF7WMaG4Xm6aa/IELccjJL8s6sahk3wT++5pzewxv4XBWf4bNA7+MNLpnzKmm2S5T/uohsw00F/j82g9xFT3qYm6vgaR4ql0aNvxd9gZ/6vdVpqBIUZmcFi9yTSFSsSSH2cNGFk3IU9Ecdh1Psi3B5pTpYzy7BbLWyKo70wBkOoY8qCwJ/wtj+2TcfR72qR8ryXgkbIoXxe04VkPqRNEezk3U+UxUvLLDEWthxjuCRCydT8FaCfYm9N+l5Un8TNmY4W9mOSu5IXKJ7oSQDrJG65Z2uZKQrWm0ToOo1nNLFclFuM+K8JWNOKysfh451LHdbmA0rkoaLUat2ttDBtQpbw3h+Hwn3SW6BfELg6WDJXHyAPNRMPSc2Grk2o7mFfTMJhHQG4U+k2bL+AHJU0nPWzuEJyMraquqKOCcHz56A+6j1PN3wpLbIC18nNPN9Q3TDPiBmwzwEg+zLu0DKTu3Oe4IRKRNTuocI="
     - secure: "BSZK3v2RYsNKv/8lBZ/dKagYF+znGXIeDY271A2nCS8DXx+y0octwI5RT08+UqdD1AvhPOPjdnSXG+M2NOSLlfBoXF/0t9S39+qy3EhGgxYAcWcVDXnrOrHEOALg1cQkCA2D/CG+eeG/36SmAn6FINTLaAOilxWGGCeL0KOB5mqPPnPogaYhowIwDkj0Hfsfrw8011XzgbufNr15GIYJ+nP7f41NcgIbnNkqXY6Q1jiWs1DuKXNDr5WGPqztHtVf7RtcpVmShzptAcAMdRqCGKF7atCZhXPbd69j3qf7F533/tCSCIrszR/Wp4BKlJfVCRCk1oiEcfRvprJqeFJ+0WJCVzyD7hXfnkBg2Tb5PmzvxOFM+hsqRu4mY3UdjqXukzAe5O0O6Uo8sqzqQUIjUooRnNQ4GdPd+7wMbyBZn4PJaW6YTi/7zg7mAegqot6uyEGGpWb6iFYrf6DX75GUfTciNktv+ez0AeqYFwNBLgcOsmaq+3V+aR+dIxsxIzC/i5GCTHvMYOIp6Q2tPCuIug0tX6uxm++vMOoMLK+vG1fVQ0Cd8m6yQIWEXUu0VBs1MTlJD+vX6LsK8CGo0Dt3ZS3JmC4TGGo9CBSEkeDvnKvQeX6x3NBIFWvgt+Hjxh4S9U+vW+AaUXpV9k+NEhesC/8Ys0UGbGYsTDzHdx9TMBY="
     - secure: "sskZYJ/+xmHh5GNvw3QRf10+sBGOjlub29jOTxjXIRYUyZfkhjsF8CR9NP59CBFuqgN716Mj1uEVEcx0aaepr/zWweQSnqa9lZgFD9nDYALVNh1b4oyqfhYG1sw57ZHdBsJOBF4mKBnahly/QQj11Ya25GyIS2IewwCWVNtCOJYietSssG9l0+qc+RPG5Ub/lEKA3VRrtbUuApcIYszt33DumgZ4wzkzICl1SuOy8V15vHVlkBqASJheEDa4Hia+eAMo6e7OCE5KWjl7K2MxTvtszwFi7lgMCyPCOy6DkaULACfBnQeloh42B2Qhd/ta04vuRKg6y4fKSrnegpeOAa9aGxF1ufvG9IvPsRTsu4+w63b9xsQUN9MyDgxcJe0YeUHPgPYL6mqAjCIKvL62LlIyrQ8IAteoh3MC+4Xb8crXaLGINTcLwYvLwxsHMuC/58hdQ23I4DqnyZGAS3L1IhpX4QvYN6xc7H+ptaiQttweoH5VpMAduSrmSnNmvLOc4g2PvbRES6D95moQ7qk7iX6vpu+PevL5HtQCbB8SFyFLTYWsqZaG+4NrjpIi2hLzUT35meHwrvxG/MsfeqOVlnBfa93VnX75vxOkRFNY56sOtrTJaiVZ+rQUdszAZz3KGPbyrSlz3KxV+ZKxZH6/0oFteAJttXFjWQdICnKGDSo="
     - secure: "sdzU5bPs1w7Nzf3F5Gtk/iq2Kfq3zfLQNGcehLZp1gJXzNf7F6HZLOn6GaXQ/5RVlhqR5nOmzMpVQs88rZv+6PE6YqGMugTxHIQeNmXtGnuEDJSBvGT9Ok8ENxcwKwL93g9SCd9P8mTspxklOsW2Bk3No2+Zlc/aQguBcX44TwYF42KuBU4O7oS6pUg8NjnuQp2zTyhp0ouzyAudatPyu1BLci/3lbx+MehutQw1y4Om6g8tfwf950UONZQdqq9MBluu7yYb1oHkdC1J4qgwdCZkJslWIwQHCH5UE4AW9iVG0qVrLpzBGtV27Kfy/Vf8r2gMYzbiur2h2+zzWSDm8/bk8YLn7u1FBpjGJdJ0pX0ZrZr+hMV2vH3e54NvA5WyRU7tw8mZ1PoDYd/FM5KXIYbscSbSRCqTsbPgyVIHuoOWXeeSe+/Ef1ifZv3HHf6ARfUIWupKfAipxChc7QUMI/HEQ9QPsqgBaooZD9chGsWAgv+8tFxdteqkx4Yh+AuZp1rVykB/9vAamUBebQxy0oeGm65j6X1rksfjAPkfeDYB7L4Ruy7tUwPtvYrAHPoWrf9O0g/qDRsw0vdqp42CjszpIxhuPhVRDx0i0wquqw2LnIU3ejRv2R8d1SecVKBBcVsWLFvc9iR4rNIi5JnxITRtiwL1Xv3Vcgx7WwxExtE="
@@ -32,14 +36,13 @@ cache:
     - $HOME/.cache/pip
     - $HOME/python
     - $HOME/ssl
-    - $HOME/pybuild
 
-matrix:
+jobs:
   include:
     - os: linux
       name: "Linux 64-bit Bionic"
       dist: bionic
-      language: bash
+      language: shell
       env:
          - GAMOS=linux
          - PLATFORM=x86_64
@@ -47,15 +50,35 @@ matrix:
     - os: linux
       name: "Linux 64-bit Xenial"
       dist: xenial
-      language: bash
+      language: shell
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=build
+    - os: linux
+      dist: bionic
+      arch: arm64
+      name: "Linux ARM64 Bionic"
+      language: shell
+      filter_secrets: false
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
+    - os: linux
+      dist: xenial
+      arch: arm64
+      name: "Linux ARM64 Xenial"
+      language: shell
+      filter_secrets: false
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
     - os: linux
       name: "Linux 64-bit Trusty"
       dist: trusty
-      language: bash
+      language: shell
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
@@ -63,47 +86,43 @@ matrix:
     - os: linux
       name: "Linux 64-bit Precise"
       dist: precise
-      language: bash
+      language: shell
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=build
     - os: linux
-      name: "Linux 64-bit - Python 3.5 Source Testing"
+      name: "Python 3.6 Source Testing"
+      dist: bionic
+      language: python
+      python: 3.6
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: linux
+      name: "Python 3.7 Source Testing"
+      dist: bionic
+      language: python
+      python: 3.7
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: linux
+      name: "Python nightly Source Testing"
+      dist: bionic
+      language: python
+      python: nightly
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: linux
+      name: "Python PyPi Source Testing"
       dist: xenial
       language: python
-      python:
-        - "3.5"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: linux
-      name: "Linux 64-bit - Python 3.6 Source Testing"
-      dist: bionic
-      language: python
-      python:
-        - "3.6"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: linux
-      name: "Linux 64-bit - Python 3.8-dev Source Testing"
-      dist: bionic
-      language: python
-      python:
-        - "3.8-dev"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: linux
-      name: "Linux 64-bit - Python nightly Source Testing"
-      dist: bionic
-      language: python
-      python:
-        - "nightly"
+      python: pypy3
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
@@ -127,7 +146,7 @@ matrix:
     - os: osx
       name: "MacOS 10.14"
       language: generic
-      osx_image: xcode11
+      osx_image: xcode11.3
       env:
         - GAMOS=macos
         - PLATFORM=x86_64
@@ -135,7 +154,6 @@ matrix:
     - os: windows
       name: "Windows 64-bit"
       language: shell
-      filter_secrets: false
       env:
         - GAMOS=windows
         - PLATFORM=x86_64
@@ -143,7 +161,6 @@ matrix:
     - os: windows
       name: "Windows 32-bit"
       language: shell
-      filter_secrets: false
       env:
         - GAMOS=windows
         - PLATFORM=x86
@@ -156,12 +173,15 @@ install:
 - source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
 
 script:
+# Discover and run all Python unit tests. Buffer output so that it's not sent to the build log.
+- $python -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
+- touch $gampath/nobrowser.txt
 - $gam version extended
 - $gam version | grep travis # travis should be part of the path (not /tmp or such)
 # determine which Python version GAM is built with and ensure it's at least build version from above.
-- if [ "$VMTYPE" == "build" ]; then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $BUILD_PYTHON_VERSION; fi
+- if [ "$VMTYPE" == "build" ]; then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $MIN_PYTHON_VERSION; fi
 # determine which OpenSSL version GAM is built with and ensure it's at least build version from above.
-- if [ "$VMTYPE" == "build" ]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $BUILD_OPENSSL_VERSION; fi
+- if [ "$VMTYPE" == "build" ]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $MIN_OPENSSL_VERSION; fi
 - if [ "$VMTYPE" == "build" ]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
 - if [ "$VMTYPE" == "build" ]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
 - export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
@@ -203,6 +223,9 @@ script:
 - if [ "$e2e" = true ]; then $gam user $newuser show imap; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user $newuser delegate to ~email; fi
 - if [ "$e2e" = true ]; then $gam user $newuser show delegates; fi
+- if [ "$e2e" = true ]; then $gam user $newuser label "✔ unicode checkmark ✔"; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels > labels.txt; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user importemail subject "Travis import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user insertemail subject "Travis insert $newbase" file gam.py labels INBOX,UNREAD; fi # yep body is gam code
 - if [ "$e2e" = true ]; then $gam user $gam_user sendemail subject "Travis send $gam_user $newbase" file gam.py recipient admin@pdl.jaylee.us; fi
@@ -210,6 +233,7 @@ script:
 - if [ "$e2e" = true ]; then $gam users "$gam_user $newbase-bulkuser-01 $newbase-bulkuser-02 $newbase-bulkuser-03" delete messages query in:anywhere maxtodelete 99999 doit; fi
 - if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-04 $newbase-bulkuser-05 $newbase-bulkuser-06" trash messages query in:anywhere maxtotrash 99999 doit; fi
 - if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-07 $newbase-bulkuser-08 $newbase-bulkuser-09" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit; fi
+- if [ "$e2e" = true ]; then $gam user $newuser delete label --ALL_LABELS--; fi
 - if [ "$e2e" = true ]; then $gam create feature name Whiteboard-$newbase; fi
 - if [ "$e2e" = true ]; then $gam create feature name VC-$newbase; fi
 - if [ "$e2e" = true ]; then $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."; fi
@@ -222,17 +246,14 @@ script:
 - if [ "$e2e" = true ]; then $gam calendar $gam_user add editor $newuser; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user showacl; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi
-- if [ "$e2e" = true ]; then $gam printer register; fi
-- if [ "$e2e" = true ]; then source travis/set_printer_csv_filter.sh; fi
-- if [ "$e2e" = true ]; then $gam print printers > printers.csv; fi
-- if [ "$e2e" = true ]; then unset GAM_CSV_ROW_FILTER; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam printer ~id add USER $newgroup; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam printjob ~id submit https://www.github.com/jay0lee/GAM; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam info printer ~id; fi
-- if [ "$e2e" = true ]; then $gam print printjobs; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam printjob ~id fetch; fi
-- if [ "$e2e" = true ]; then $gam print printjobs | $gam csv - gam printjob ~id delete; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam delete printer ~id; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user addevent summary "Travis test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
+- if [ "$e2e" = true ]; then $gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser; fi
+- if [ "$e2e" = true ]; then $gam create vaulthold matter "Travis matter $newbase" name "Travis hold $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print vaultmatters; fi
+- if [ "$e2e" = true ]; then $gam print vaultholds matter "Travis matter $newbase"; fi
+- if [ "$e2e" = true ]; then $gam create vaultexport matter "Travis matter $newbase" name "Travis export $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam info export id:~~matterId~~ id:~~id~~; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add calendar id:$newresource; fi
 - if [ "$e2e" = true ]; then $gam delete resource $newresource; fi
 - if [ "$e2e" = true ]; then $gam delete feature Whiteboard-$newbase; fi
@@ -242,10 +263,17 @@ script:
 - if [ "$e2e" = true ]; then $gam create alias $newalias user $newuser; fi
 - if [ "$e2e" = true ]; then $gam whatis $newuser; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
+- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam download export id:~~matterId~~ id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter "Travis matter $newbase"; fi
+- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action close; fi
+- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action delete; fi
 - if [ "$e2e" = true ]; then $gam delete user $newuser; fi
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
 - if [ "$e2e" = true ]; then $gam print mobile; fi
 - if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$e2e" = true ]; then $gam report customer todrive; fi
+- if [ "$e2e" = true ]; then $gam report users fulldatarequired accounts,gmail fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
+- if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
 
 before_deploy:
 - export TRAVIS_TAG="preview"
@@ -253,14 +281,13 @@ before_deploy:
 
 deploy:
   provider: releases
-  api_key:
+  token:
     secure: bzambMcQwyv/o5c5GrKGCsZHgE5R85tg8sNFvPfpISz3+uosCjnBXas7wvCKzT75XUFi2ztfbYak6HdKf4sGnNHk0saEicB3slH+ghPyZbYzp76yvvduhFO2nWW3/F01tL+Yfqqt4/q8wFaWGjrC5km+6GLVyB4lWA/Uyu49qKnz02uSwyhBD/VFbO7DOQ65a1iWk9HngyMsu0Oi7HIbSjSLtxTHedNfOf3waW0NivTTxYXiYGX/MCu3GWhgIGj47a+H3A6FcQ/9QWvnKgnoixdgPBUz7kDb7ktsWwQsILPGStgH7iMuG49ZlXdEFmqwifBri2wvzmFEevBGZjHcupy1IGrNFRG+IUGKMotio+OkLHlLjuv7ZJtqCz/Vf5SNFgNyMSanx6jKEUJuYvndVg99IRXmYVwHFwPu5BAcJACpU6C0AfyGmmSqqwxCd46uXL62ynxNFpHuRfOqlDnmCTfZgjOciJSlDDpf+Xz9fF7+oCoeCi3mrcZVFjhd3tT6Oxw5HrsDtm0ZNld1cdLidaq8H6vOFgHMd0A9yNYZzTzXTvpmxzkXT4Zc7s+PYKN6z5fRZ+pJeckUjRXblvVEfs5HFSymavcOc5AkRwxpvOsTQMNmlnaJCBo5UNs0K/rVmRi5cFmaiwTcBCY0kTllOBJ4zWsfq8seiokWwNUNK2g=
   file_glob: true
   overwrite: true
   file: gam-$GAMVERSION-*
   skip_cleanup: true
   draft: true
-  all_branches: true
   on:
     repo: jay0lee/GAM
     condition: $VMTYPE = build

src/GamCommands.txt

@@ -140,6 +140,7 @@ If an item contains spaces, it should be surrounded by ".
 
 <AccessToken> ::= <String>
 <ACLScope> ::= [user:]<EmailAddress>|group:<EmailAddress>|domain[:<DomainName>]|default
+<APIScopeURL> ::= <String>
 <ASPID> ::= <String>
 <BuildingID> ::= <String>|id:<String>
 <CalendarACLRole> ::= editor|freebusy|freebusyreader|owner|reader|writer
@@ -221,6 +222,7 @@ If an item contains spaces, it should be surrounded by ".
 <SchemaName> ::= <String>
 <Section> ::= <String>
 <SerialNumber> ::= <String>
+<ServiceAccountKey> ::= <String>
 <S/MIMEID> ::= <String>	
 <SMTPHostName> ::= <String>
 <StudentItem> ::= <EmailAddress>|<UniqueID>|<String>
@@ -552,6 +554,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
     "'it em' 'it,em' \"it'em\""
 
 <ACLList> ::= "<ACLScope>(,<ACLScope>)*"
+<APIScopeURLList> ::= "<APIScopeURL>(,<APIScopeURL>)*"
 <ASPIDList> ::= "<ASPID>(,<ASPID>)*"
 <CalendarList> ::= "<CalendarItem>(,<CalendarItem>)*"
 <ChatRoomList> ::= "<ChatRoom>(,<ChatRoom>)*"
@@ -590,6 +593,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
 <SKUIDList> ="<SKUID>(,<SKUID>)*"
 <SchemaNameList> ::= "<SchemaName>(,<SchemaName>)*"
 <SerialNumberList> ::= "<SerialNumber>(,<SerialNumber>)*"
+<ServiceAccountKeyList> ::= "<ServiceAccountKey>(,<ServiceAccountKey>)*"
 <TeamDriveIDList> ::= "<TeamDriveID>(,<TeamDriveID>)*"
 <UserFieldNameList> ::= "<UserFieldName>(,<UserFieldName>)*"
 <UserList> ::= "<UserItem>(,<UserItem>)*"
@@ -834,12 +838,17 @@ gam delete project [<EmailAddress>] [gam|<ProjectID>|(filter <String>)]
 gam show projects [<EmailAddress>] [all|gam|<ProjectID>|(filter <String>)]
 gam print projects [<EmailAddress>] [all|gam|<ProjectID>|(filter <String>)] [todrive]
 
+gam rotate sakey|sakeys [retain_none|retain_existing|replace_current]
+         [(algorithm KEY_ALG_RSA_1024|KEY_ALG_RSA_2048)|(localkeysize 1024|2048|4096)]
+gam delete sakey|sakeys <ServiceAccountKeyList>+ [doit]
+gam show sakey|sakeys [all|system|user]
+
 gam oauth|oauth2 create|request [<EmailAddress>]
 gam oauth|oauth2 delete|revoke
 gam oauth|oauth2 info|verify [accesstoken <AccessToken>] [idtoken <IDToken>] [showsecret]
 gam oauth|oauth2 refresh
 
-gam <UserTypeEntity> check serviceaccount
+gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
 gam whatis <EmailItem>
 
@@ -871,6 +880,25 @@ gam update resoldsubscription <CustomerID> <SKUID>
 gam delete resoldsubscription <CustomerID> <SKUID> cancel|downgrade|transfer_to_direct
 gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 
+<ActivityApplicationName> ::=
+	access|accesstransparency|
+	admin|
+	calendar|calendars|
+	chat|
+	drive|doc|docs|
+	enterprisegroups|groupsenterprise|
+	gcp|
+	google+|gplus|
+	group|groups|
+	hangoutsmeet|meet|
+	jamboard|
+	login|logins|
+	mobile|
+	oauthtoken|token|tokens|
+	rules|
+	saml|
+	useraccounts
+
 <ReportsApp> ::=
 	accounts|
 	app_maker|
@@ -890,7 +918,7 @@ gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppL
 	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)] [filter|filters <String>] [fields|parameters <String>]
 gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
 	[fields|parameters <String>]
-gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token [todrive]
+gam report <ActivityApplicationName> [todrive]
 	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
@@ -976,6 +1004,7 @@ The following attributes are equivalent:
 	(end (allday <Date>)|<Time>)|
 	guestscantinviteothers|
 	guestscantseeothers|
+	hangoutsmeet|
 	(id <String>)|
 	(location <String>)|
 	(noreminders| (reminder <Number> email|popup|sms))|
@@ -1070,7 +1099,7 @@ gam print mobile [todrive] [(query <QueryMobile>)|(queries <QueryMobileList>)] [
 	fields <MobileFieldNameList>] [delimiter <Character>] [appslimit <Number>] [listlimit <Number>]
 
 gam create group <EmailAddress> <GroupAttributes>*
-gam update group <GroupItem> [admincreated <Boolean>] [email <EmailAddress>] <GroupAttributes>*
+gam update group <GroupItem> [email <EmailAddress>] <GroupAttributes>*
 gam update group <GroupItem> add [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
 gam update group <GroupItem> delete|remove [owner|manager|member] <UserTypeEntity>
 gam update group <GroupItem> sync [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>

src/auth/__init__.py

@@ -0,0 +1,27 @@
+"""Authentication/Credentials general purpose and convenience methods."""
+
+from . import oauth
+from var import _FN_OAUTH2_TXT
+from var import GC_OAUTH2_TXT
+from var import GC_Values
+
+# TODO: Move logic that determines file name into this module. We should be able
+# to discover the file location without accessing a private member or waiting
+# for a global initialization.
+DEFAULT_OAUTH_STORAGE_FILE = _FN_OAUTH2_TXT
+
+
+def get_admin_credentials_filename():
+  """Gets the name of the file that stores the admin account credentials."""
+  # If the environment globals are loaded, use the set global value. It may have
+  # some custom name in it. Otherwise, just use the default name.
+  if GC_Values[GC_OAUTH2_TXT]:
+    return GC_Values[GC_OAUTH2_TXT]
+  else:
+    return DEFAULT_OAUTH_STORAGE_FILE
+
+
+def get_admin_credentials():
+  """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+  credential_file = get_admin_credentials_filename()
+  return oauth.Credentials.from_credentials_file(credential_file)

src/auth/oauth.py

@@ -0,0 +1,568 @@
+"""OAuth2.0 user credentials."""
+
+import datetime
+import json
+import os
+import re
+import threading
+from urllib.parse import urlencode
+
+from filelock import FileLock
+import google_auth_oauthlib.flow
+import google.oauth2.credentials
+import google.oauth2.id_token
+
+import fileutils
+import transport
+from var import GAM_INFO
+from var import GM_Globals
+from var import GM_WINDOWS
+
+MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
+                                        'browser:\n\n\t{url}\n')
+MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
+MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
+                                             ' visit:\n\n\t{url}\n\nIf your '
+                                             'browser is on a different machine'
+                                             ' then press CTRL+C and create a '
+                                             'file called nobrowser.txt in the '
+                                             'same folder as GAM.\n')
+MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
+                                ' close this browser window and return to GAM.')
+
+
+class CredentialsError(Exception):
+  """Base error class."""
+  pass
+
+
+class InvalidCredentialsFileError(CredentialsError):
+  """Error raised when a file cannot be opened into a credentials object."""
+  pass
+
+
+class EmptyCredentialsFileError(InvalidCredentialsFileError):
+  """Error raised when a credentials file contains no content."""
+  pass
+
+
+class InvalidClientSecretsFileFormatError(CredentialsError):
+  """Error raised when a client secrets file format is invalid."""
+  pass
+
+
+class InvalidClientSecretsFileError(CredentialsError):
+  """Error raised when client secrets file cannot be read."""
+  pass
+
+
+class Credentials(google.oauth2.credentials.Credentials):
+  """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
+
+  DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+  def __init__(self,
+               token,
+               refresh_token=None,
+               id_token=None,
+               token_uri=None,
+               client_id=None,
+               client_secret=None,
+               scopes=None,
+               quota_project_id=None,
+               expiry=None,
+               id_token_data=None,
+               filename=None):
+    """A thread-safe OAuth2.0 credentials object.
+
+    Credentials adds additional utility properties and methods to a
+    standard OAuth2.0 credentials object. When used to store credentials on
+    disk, it implements a file lock to avoid collision during writes.
+
+    Args:
+      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
+        information is provided.
+      refresh_token: String, The OAuth 2.0 refresh token. If specified,
+        credentials can be refreshed.
+      id_token: String, The Open ID Connect ID Token.
+      token_uri: String, The OAuth 2.0 authorization server's token endpoint
+        URI. Must be specified for refresh, can be left as None if the token can
+        not be refreshed.
+      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
+        can be left as None if the token can not be refreshed.
+      client_secret: String, The OAuth 2.0 client secret. Must be specified for
+        refresh, can be left as None if the token can not be refreshed.
+      scopes: Sequence[str], The scopes used to obtain authorization.
+        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
+          not request additional scopes after authorization. The scopes must be
+          derivable from the refresh token if refresh information is provided
+          (e.g. The refresh token scopes are a superset of this or contain a
+          wild card scope like
+            'https://www.googleapis.com/auth/any-api').
+      quota_project_id: String, The project ID used for quota and billing. This
+        project may be different from the project used to create the
+        credentials.
+      expiry: datetime.datetime, The time at which the provided token will
+        expire.
+      id_token_data: Oauth2.0 ID Token data which was previously fetched for
+        this access token against the google.oauth2.id_token library.
+      filename: String, Path to a file that will be used to store the
+        credentials. If provided, a lock file of the same name and a ".lock"
+          extension will be created for concurrency controls. Note: New
+            credentials are not saved to disk until write() or refresh() are
+            called.
+
+    Raises:
+      TypeError: If id_token_data is not the required dict type.
+    """
+    super(Credentials, self).__init__(
+        token=token,
+        refresh_token=refresh_token,
+        id_token=id_token,
+        token_uri=token_uri,
+        client_id=client_id,
+        client_secret=client_secret,
+        scopes=scopes,
+        quota_project_id=quota_project_id)
+
+    # Load data not restored by the super class
+    self.expiry = expiry
+    if id_token_data and not isinstance(id_token_data, dict):
+      raise TypeError(f'Expected type id_token_data dict but received '
+                      f'{type(id_token_data)}')
+    self._id_token_data = id_token_data.copy() if id_token_data else None
+
+    # If a filename is provided, use a lock file to control concurrent access
+    # to the resource. If no filename is provided, use a thread lock that has
+    # the same interface as FileLock in order to simplify the implementation.
+    if filename:
+      # Convert relative paths into absolute
+      self._filename = os.path.abspath(filename)
+      lock_file = os.path.abspath(f'{self._filename}.lock')
+      self._lock = FileLock(lock_file)
+    else:
+      self._filename = None
+      self._lock = _FileLikeThreadLock()
+
+  # Use a property to prevent external mutation of the filename.
+  @property
+  def filename(self):
+    return self._filename
+
+  @classmethod
+  def from_authorized_user_info(cls, info, filename=None):
+    """Generates Credentials from JSON containing authorized user info.
+
+    Args:
+      info: Dict, authorized user info in Google format.
+      filename: String, the filename used to store these credentials on disk. If
+        no filename is provided, the credentials will not be saved to disk.
+
+    Raises:
+      ValueError: If missing fields are detected in the info.
+    """
+    # We need all of these keys
+    keys_needed = set(('client_id', 'client_secret'))
+    # We need 1 or more of these keys
+    keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
+    missing = keys_needed.difference(info.keys())
+    has_one_of = set(info) & keys_need_one_of
+    if missing or not has_one_of:
+      raise ValueError(
+          'Authorized user info was not in the expected format, missing '
+          f'fields {", ".join(missing)} and one of '
+          f'{", ".join(keys_need_one_of)}.')
+
+    expiry = info.get('token_expiry')
+    if expiry:
+      # Convert the raw expiry to datetime
+      expiry = datetime.datetime.strptime(expiry, Credentials.DATETIME_FORMAT)
+    id_token_data = info.get('decoded_id_token')
+
+    # Provide backwards compatibility with field names when loading from JSON.
+    # Some field names may be different, depending on when/how the credentials
+    # were pickled.
+    return cls(
+        token=info.get('token', info.get('auth_token', '')),
+        refresh_token=info.get('refresh_token', ''),
+        id_token=info.get('id_token_jwt', info.get('id_token')),
+        token_uri=info.get('token_uri'),
+        client_id=info['client_id'],
+        client_secret=info['client_secret'],
+        scopes=info.get('scopes'),
+        quota_project_id=info.get('quota_project_id'),
+        expiry=expiry,
+        id_token_data=id_token_data,
+        filename=filename)
+
+  @classmethod
+  def from_google_oauth2_credentials(cls, credentials, filename=None):
+    """Generates Credentials from a google.oauth2.Credentials object."""
+    info = json.loads(credentials.to_json())
+    # Add properties which are not exported with the native to_json() output.
+    info['id_token'] = credentials.id_token
+    if credentials.expiry:
+      info['token_expiry'] = credentials.expiry.strftime(
+          Credentials.DATETIME_FORMAT)
+    info['quota_project_id'] = credentials.quota_project_id
+
+    return cls.from_authorized_user_info(info, filename=filename)
+
+  @classmethod
+  def from_credentials_file(cls, filename):
+    """Generates Credentials from a stored Credentials file.
+
+    The same file will be used to save the credentials when the access token is
+    refreshed.
+
+    Args:
+      filename: String, the name of a file containing JSON credentials to load.
+        The same filename will be used to save credentials back to disk.
+
+    Returns:
+      The credentials loaded from disk.
+
+    Raises:
+      InvalidCredentialsFileError: When the credentials file cannot be opened.
+      EmptyCredentialsFileError: When the provided file contains no credentials.
+    """
+    file_content = fileutils.read_file(
+        filename, continue_on_error=True, display_errors=False)
+    if file_content is None:
+      raise InvalidCredentialsFileError(f'File {filename} could not be opened')
+    info = json.loads(file_content)
+    if not info:
+      raise EmptyCredentialsFileError(
+          f'File {filename} contains no credential data')
+
+    try:
+      # We read the existing data from the passed in file, but we also want to
+      # save future data/tokens in the same place.
+      return cls.from_authorized_user_info(info, filename=filename)
+    except ValueError as e:
+      raise InvalidCredentialsFileError(str(e))
+
+  @classmethod
+  def from_client_secrets(cls,
+                          client_id,
+                          client_secret,
+                          scopes,
+                          access_type='offline',
+                          login_hint=None,
+                          filename=None,
+                          use_console_flow=False):
+    """Runs an OAuth Flow from client secrets to generate credentials.
+
+    Args:
+      client_id: String, The OAuth2.0 Client ID.
+      client_secret: String, The OAuth2.0 Client Secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      filename: String, the path to a file to use to save the credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Returns:
+      Credentials
+    """
+    client_config = {
+        'installed': {
+            'client_id': client_id,
+            'client_secret': client_secret,
+            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
+            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+            'token_uri': 'https://oauth2.googleapis.com/token',
+        }
+    }
+
+    flow = _ShortURLFlow.from_client_config(
+        client_config, scopes, autogenerate_code_verifier=True)
+    flow_kwargs = {'access_type': access_type}
+    if login_hint:
+      flow_kwargs['login_hint'] = login_hint
+
+    # TODO: Move code for browser detection somewhere in this file so that the
+    # messaging about `nobrowser.txt` is co-located with the logic that uses it.
+    if use_console_flow:
+      flow.run_console(
+          authorization_prompt_message=MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
+          authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
+          **flow_kwargs)
+    else:
+      flow.run_local_server(
+          authorization_prompt_message=MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
+          success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
+          **flow_kwargs)
+    return cls.from_google_oauth2_credentials(
+        flow.credentials, filename=filename)
+
+  @classmethod
+  def from_client_secrets_file(cls,
+                               client_secrets_file,
+                               scopes,
+                               access_type='offline',
+                               login_hint=None,
+                               credentials_file=None,
+                               use_console_flow=False):
+    """Runs an OAuth Flow from secrets stored on disk to generate credentials.
+
+    Args:
+      client_secrets_file: String, path to a file containing a client ID and
+        secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      credentials_file: String, the path to a file to use to save the
+        credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Raises:
+      InvalidClientSecretsFileError: If the client secrets file cannot be
+        opened.
+      InvalidClientSecretsFileFormatError: If the client secrets file does not
+        contain the required data or the data is malformed.
+
+    Returns:
+      Credentials
+    """
+    cs_data = fileutils.read_file(
+        client_secrets_file, continue_on_error=True, display_errors=False)
+    if not cs_data:
+      raise InvalidClientSecretsFileError(
+          f'File {client_secrets_file} could not be opened')
+    try:
+      cs_json = json.loads(cs_data)
+      client_id = cs_json['installed']['client_id']
+      # Chop off .apps.googleusercontent.com suffix as it's not needed
+      # and we need to keep things short for the Auth URL.
+      client_id = re.sub(r'\.apps\.googleusercontent\.com$', '', client_id)
+      client_secret = cs_json['installed']['client_secret']
+    except (ValueError, IndexError, KeyError):
+      raise InvalidClientSecretsFileFormatError(
+          f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
+      )
+
+    return cls.from_client_secrets(
+        client_id,
+        client_secret,
+        scopes,
+        access_type=access_type,
+        login_hint=login_hint,
+        filename=credentials_file,
+        use_console_flow=use_console_flow)
+
+  def _fetch_id_token_data(self):
+    """Fetches verification details from Google for the OAuth2.0 token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Raises:
+      CredentialsError: If no id_token is present.
+    """
+    if not self.id_token:
+      raise CredentialsError('Failed to fetch token data. No id_token present.')
+    request = transport.create_request()
+    self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
+        self.id_token, request)
+
+  def get_token_value(self, field):
+    """Retrieves data from the OAuth ID token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Args:
+      field: The name of the key/field to fetch
+
+    Returns:
+      The value associated with the given key or 'Unknown' if the key data can
+      not be found in the access token data.
+    """
+    if not self._id_token_data:
+      self._fetch_id_token_data()
+    # Maintain legacy GAM behavior here to return "Unknown" if the field is
+    # otherwise unpopulated.
+    return self._id_token_data.get(field, 'Unknown')
+
+  def to_json(self, strip=None):
+    """Creates a JSON representation of a Credentials.
+
+    Args:
+        strip: Sequence[str], Optional list of members to exclude from the
+          generated JSON.
+
+    Returns:
+        str: A JSON representation of this instance, suitable to pass to
+             from_json().
+    """
+    expiry = self.expiry.strftime(
+        Credentials.DATETIME_FORMAT) if self.expiry else None
+    prep = {
+        'token': self.token,
+        'refresh_token': self.refresh_token,
+        'token_uri': self.token_uri,
+        'client_id': self.client_id,
+        'client_secret': self.client_secret,
+        'id_token': self.id_token,
+        # Google auth doesn't currently give us scopes back on refresh.
+        # 'scopes': sorted(self.scopes),
+        'token_expiry': expiry,
+        'decoded_id_token': self._id_token_data,
+    }
+
+    # Remove empty entries
+    prep = {k: v for k, v in prep.items() if v is not None}
+
+    # Remove entries that explicitly need to be removed
+    if strip is not None:
+      prep = {k: v for k, v in prep.items() if k not in strip}
+
+    return json.dumps(prep, indent=2, sort_keys=True)
+
+  def refresh(self, request=None):
+    """Refreshes the credential's access token.
+
+    Args:
+      request: google.auth.transport.Request, The object used to make HTTP
+        requests. If not provided, a default request will be used.
+
+    Raises:
+      google.auth.exceptions.RefreshError: If the credentials could not be
+      refreshed.
+    """
+    with self._lock:
+      if request is None:
+        request = transport.create_request()
+      self._locked_refresh(request)
+      # Save the new tokens back to disk, if these credentials are disk-backed.
+      if self._filename:
+        self._locked_write()
+
+  def _locked_refresh(self, request):
+    """Refreshes the credential's access token while the file lock is held."""
+    assert self._lock.is_locked
+    super(Credentials, self).refresh(request)
+
+  def write(self):
+    """Writes credentials to disk."""
+    with self._lock:
+      self._locked_write()
+
+  def _locked_write(self):
+    """Writes credentials to disk while the file lock is held."""
+    assert self._lock.is_locked
+    if not self.filename:
+      # If no filename was provided to the constructor, these credentials cannot
+      # be saved to disk.
+      raise CredentialsError(
+          'The credentials have no associated filename and cannot be saved '
+          'to disk.')
+    fileutils.write_file(self._filename, self.to_json())
+
+  def delete(self):
+    """Deletes all files on disk related to these credentials."""
+    with self._lock:
+      # Only attempt to remove the file if the lock we're using is a FileLock.
+      if isinstance(self._lock, FileLock):
+        os.remove(self._filename)
+        if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
+          os.remove(self._lock.lock_file)
+
+  _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+
+  def revoke(self, http=None):
+    """Revokes this credential's access token with the server.
+
+    Args:
+      http: httplib2.Http compatible object for use as a transport. If no http
+        is provided, a default will be used.
+    """
+    with self._lock:
+      if http is None:
+        http = transport.create_http()
+      params = urlencode({'token': self.refresh_token})
+      revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
+      http.request(revoke_uri, 'GET')
+
+
+class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
+  """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
+
+  URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+  def authorization_url(self, http=None, **kwargs):
+    """Gets a shortened authorization URL."""
+    long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
+    if not http:
+      http = transport.create_http(timeout=10)
+    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
+    try:
+      payload = json.dumps({'long_url': long_url})
+      resp, content = http.request(
+          _ShortURLFlow.URL_SHORTENER_ENDPOINT,
+          'POST',
+          payload,
+          headers=headers)
+    except:
+      return long_url, state
+
+    if resp.status != 200:
+      return long_url, state
+
+    try:
+      if isinstance(content, bytes):
+        content = content.decode()
+      return json.loads(content).get('short_url', long_url), state
+    except:
+      return long_url, state
+
+
+class _FileLikeThreadLock(object):
+  """A threading.lock which has the same interface as filelock.Filelock."""
+
+  def __init__(self):
+    """A shell object that holds a threading.Lock.
+
+    Since we cannot inherit from built-in classes such as threading.Lock, we
+    just use a shell object and maintain a lock inside of it.
+    """
+    self._lock = threading.Lock()
+
+  def __enter__(self, *args, **kwargs):
+    return self._lock.__enter__(*args, **kwargs)
+
+  def __exit__(self, *args, **kwargs):
+    return self._lock.__exit__(*args, **kwargs)
+
+  def acquire(self, **kwargs):
+    return self._lock.acquire(**kwargs)
+
+  def release(self):
+    return self._lock.release()
+
+  @property
+  def is_locked(self):
+    return self._lock.locked()
+
+  @property
+  def lock_file(self):
+    return None

src/auth/oauth_test.py

@@ -0,0 +1,680 @@
+"""Tests for oauth."""
+
+import datetime
+import json
+import os
+import platform
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import google.oauth2.credentials
+
+from auth import oauth
+
+
+class CredentialsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_token = 'fake_token'
+    self.fake_refresh_token = 'fake_refresh_token'
+    self.fake_id_token = 'fake_id_token'
+    self.fake_token_uri = 'https://fake.token.uri'
+    self.fake_client_id = 'fake_client_id'
+    self.fake_client_secret = 'fake_client_secret'
+    self.fake_scopes = [
+        'fake_api.readonly',
+        'fake_other_api.write',
+    ]
+    self.fake_quota_project_id = 'fake_quota_project_id'
+    self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
+    self.fake_filename = 'fake_filename'
+    self.fake_token_data = {
+        'field': 'value',
+        'another-field': 'another-value',
+    }
+    self.info_with_only_required_fields = {
+        'refresh_token': self.fake_refresh_token,
+        'client_id': self.fake_client_id,
+        'client_secret': self.fake_client_secret,
+    }
+    super(CredentialsTest, self).setUp()
+
+  def tearDown(self):
+    # Remove any credential files that may have been created.
+    if os.path.exists(self.fake_filename):
+      os.remove(self.fake_filename)
+    if os.path.exists('%s.lock' % self.fake_filename):
+      os.remove('%s.lock' % self.fake_filename)
+    super(CredentialsTest, self).tearDown()
+
+  def test_from_authorized_user_info_only_required_info(self):
+    creds = oauth.Credentials.from_authorized_user_info(
+        self.info_with_only_required_fields)
+    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+    self.assertEqual(self.fake_client_id, creds.client_id)
+    self.assertEqual(self.fake_client_secret, creds.client_secret)
+    self.assertIsNone(creds.id_token)
+    self.assertIsNone(creds.expiry)
+    self.assertIsNone(creds.filename)
+
+  def test_from_authorized_user_info_all_info_provided(self):
+    info = {
+        'token':
+            self.fake_token,
+        'refresh_token':
+            self.fake_refresh_token,
+        'id_token':
+            self.fake_id_token,
+        'token_uri':
+            self.fake_token_uri,
+        'client_id':
+            self.fake_client_id,
+        'client_secret':
+            self.fake_client_secret,
+        'token_expiry':
+            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+        'id_token_data':
+            self.fake_token_data,
+    }
+    creds = oauth.Credentials.from_authorized_user_info(info)
+    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+    self.assertEqual(self.fake_client_id, creds.client_id)
+    self.assertEqual(self.fake_client_secret, creds.client_secret)
+    self.assertEqual(self.fake_id_token, creds.id_token)
+    self.assertEqual(self.fake_token_uri, creds.token_uri)
+    self.assertEqual(self.fake_token_expiry, creds.expiry)
+    self.assertIsNone(creds.filename)
+
+  def test_from_authorized_user_info_missing_required_info(self):
+    info_with_missing_fields = {'token': self.fake_token}
+    with self.assertRaises(ValueError):
+      oauth.Credentials.from_authorized_user_info(info_with_missing_fields)
+
+  def test_from_authorized_user_info_no_expiry_in_info(self):
+    info_with_no_token_expiry = self.info_with_only_required_fields.copy()
+    self.assertIsNone(info_with_no_token_expiry.get('expiry'))
+    creds = oauth.Credentials.from_authorized_user_info(
+        info_with_no_token_expiry)
+    self.assertIsNone(creds.expiry)
+
+  def test_init_saves_filename(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+  def test_init_loads_decoded_id_token_data(self, mock_verify_token):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data)
+    self.assertEqual(
+        self.fake_token_data.get('field'), creds.get_token_value('field'))
+    # Verify the fetching method was not called, since the token
+    # data was supposed to be loaded from the passed in info.
+    self.assertEqual(mock_verify_token.call_count, 0)
+
+  def test_credentials_uses_file_lock_when_filename_provided(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    self.assertIsInstance(creds._lock, oauth.FileLock)
+    self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
+
+  def test_credentials_uses_thread_lock_when_filename_not_provided(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=None)
+    self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
+    self.assertIsNone(creds.filename)
+
+  def test_from_oauth2credentials(self):
+    google_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    creds = oauth.Credentials.from_google_oauth2_credentials(
+        google_creds, filename=self.fake_filename)
+    self.assertEqual(google_creds.token, creds.token)
+    self.assertEqual(google_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(google_creds.client_id, creds.client_id)
+    self.assertEqual(google_creds.client_secret, creds.client_secret)
+    self.assertEqual(google_creds.id_token, creds.id_token)
+    self.assertEqual(google_creds.expiry, creds.expiry)
+    self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+    self.assertIn('could not be opened', str(e.exception))
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_no_serialized_data_in_file_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({})
+    with self.assertRaises(oauth.EmptyCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_missing_any_token_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({
+        # This data is missing a token key/value pair
+        'client_id': self.fake_client_id,
+        'client_secret': self.fake_client_secret,
+    })
+    with self.assertRaises(oauth.InvalidCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_missing_required_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({
+        # This data is missing a client_secret key/value pair
+        'client_id': self.fake_client_id,
+        'refresh_token': self.fake_refresh_token,
+    })
+    with self.assertRaises(oauth.InvalidCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_console_flow(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        use_console_flow=True)
+    self.assertTrue(mock_flow.return_value.run_console.called)
+    self.assertFalse(mock_flow.return_value.run_local_server.called)
+    self.assertEqual(flow_creds.token, creds.token)
+    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(flow_creds.client_id, creds.client_id)
+    self.assertEqual(flow_creds.client_secret, creds.client_secret)
+    self.assertEqual(flow_creds.id_token, creds.id_token)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_local_server_flow(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        use_console_flow=False)
+    self.assertFalse(mock_flow.return_value.run_console.called)
+    self.assertTrue(mock_flow.return_value.run_local_server.called)
+    self.assertEqual(flow_creds.token, creds.token)
+    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(flow_creds.client_id, creds.client_id)
+    self.assertEqual(flow_creds.client_secret, creds.client_secret)
+    self.assertEqual(flow_creds.id_token, creds.id_token)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_uses_login_hint(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        login_hint='someone@domain.com')
+
+    run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
+    self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
+
+  def test_from_client_secrets_uses_shortened_url_flow(self):
+    with patch.object(oauth._ShortURLFlow, 'from_client_config') as mock_flow:
+      flow_creds = google.oauth2.credentials.Credentials(
+          token=self.fake_token,
+          refresh_token=self.fake_refresh_token,
+          client_id=self.fake_client_id,
+          client_secret=self.fake_client_secret,
+          id_token=self.fake_id_token)
+      mock_flow.return_value.credentials = flow_creds
+      oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                            self.fake_client_secret,
+                                            self.fake_scopes)
+    self.assertTrue(mock_flow.called)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        filename=self.fake_filename)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    with self.assertRaises(oauth.InvalidClientSecretsFileError):
+      oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                 self.fake_scopes)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_client_secrets_file_missing_required_json_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({})
+    with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
+      oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                 self.fake_scopes)
+    self.assertIn('Could not extract Client ID or Client Secret',
+                  str(e.exception))
+
+  @patch.object(oauth.Credentials, 'from_client_secrets')
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_client_secrets_file_strips_domain_from_client_id(
+      self, mock_read_file, mock_creds_from_client_secrets):
+    mock_read_file.return_value = json.dumps({
+        'installed': {
+            'client_id': self.fake_client_id + '.apps.googleusercontent.com',
+            'client_secret': self.fake_client_secret,
+        }
+    })
+
+    oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                               self.fake_scopes)
+    self.assertEqual(self.fake_client_id,
+                     mock_creds_from_client_secrets.call_args[0][0])
+
+  def test_get_token_value_known_token_field(self):
+    token_data = {'known-field': 'known-value'}
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token_data=token_data)
+    self.assertEqual('known-value', creds.get_token_value('known-field'))
+
+  def test_get_token_value_unknown_field_returns_unknown(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token_data=self.fake_token_data)
+    self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
+
+  def test_to_json_contains_all_required_fields(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data,
+        token_uri=self.fake_token_uri,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        scopes=self.fake_scopes,
+        quota_project_id=self.fake_quota_project_id,
+        expiry=self.fake_token_expiry)
+    json_string = creds.to_json()
+    json_data = json.loads(json_string)
+    keys = json_data.keys()
+    self.assertIn('token', keys)
+    self.assertEqual(self.fake_token, json_data['token'])
+    self.assertIn('refresh_token', keys)
+    self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
+    self.assertIn('id_token', keys)
+    self.assertEqual(self.fake_id_token, json_data['id_token'])
+    self.assertIn('token_uri', keys)
+    self.assertEqual(self.fake_token_uri, json_data['token_uri'])
+    self.assertIn('client_id', keys)
+    self.assertEqual(self.fake_client_id, json_data['client_id'])
+    self.assertIn('client_secret', keys)
+    self.assertEqual(self.fake_client_secret, json_data['client_secret'])
+    self.assertNotIn('scopes', keys)  # Scopes are not currently saved
+    self.assertIn('token_expiry', keys)
+    self.assertEqual(
+        self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+        json_data['token_expiry'])
+    self.assertIn('decoded_id_token', keys)
+    self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
+
+  def test_credentials_to_json_and_back(self):
+    original_creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data,
+        token_uri=self.fake_token_uri,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        scopes=self.fake_scopes,
+        quota_project_id=self.fake_quota_project_id,
+        expiry=self.fake_token_expiry)
+    pickled_creds = original_creds.to_json()
+    serialized_json = json.loads(pickled_creds)
+    unpickled_creds = oauth.Credentials.from_authorized_user_info(
+        serialized_json)
+    self.assertEqual(original_creds.token, unpickled_creds.token)
+    self.assertEqual(original_creds.refresh_token,
+                     unpickled_creds.refresh_token)
+    self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
+    self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
+    self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
+    self.assertEqual(original_creds.client_secret,
+                     unpickled_creds.client_secret)
+    self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  def test_refresh_calls_super_refresh(self, mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    request = MagicMock()
+
+    creds.refresh(request)
+    self.assertTrue(mock_super_refresh.called)
+    self.assertEqual(request, mock_super_refresh.call_args[0][0])
+
+  def test_refresh_locks_resource_during_refresh(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    lock = creds._lock
+
+    def check_lock_is_locked(*unused_args, **unused_kwargs):
+      self.assertTrue(lock.is_locked)
+
+    # We need to mock the superclass refresh so it doesn't actually try to
+    # refresh our fake token.
+    # At the same time, we'll make sure the lock is held during the refresh.
+    with patch.object(oauth.google.oauth2.credentials.Credentials,
+                      'refresh') as mock_refresh:
+      mock_refresh.side_effect = check_lock_is_locked
+      creds.refresh(request=MagicMock())
+
+    # Make sure our side effect was actually performed.
+    self.assertTrue(mock_refresh.called)
+    # The lock should be released after refresh
+    self.assertFalse(lock.is_locked)
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  @patch.object(oauth.fileutils, 'write_file')
+  def test_refresh_writes_new_credentials_to_disk_after_refresh(
+      self, mock_write_file, mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+
+    def update_access_token(unused_request):
+      creds.token = 'refreshed_access_token'
+
+    mock_super_refresh.side_effect = update_access_token
+
+    self.assertIsNone(creds.token)
+    creds.refresh(request=MagicMock())
+    self.assertEqual('refreshed_access_token', creds.token,
+                     'Access token was not refreshed')
+    text_written_to_file = mock_write_file.call_args[0][1]
+    self.assertIsNotNone(text_written_to_file, 'Nothing was written to file')
+    saved_json = json.loads(text_written_to_file)
+    self.assertEqual('refreshed_access_token', saved_json['token'],
+                     'Refreshed access token was not saved to disk')
+
+  def test_write_writes_credentials_to_disk(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+
+    self.assertFalse(os.path.exists(self.fake_filename))
+    creds.write()
+    self.assertTrue(os.path.exists(self.fake_filename))
+
+  def test_write_raises_error_when_no_credentials_file_is_set(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+
+    self.assertIsNone(creds.filename)
+    with self.assertRaises(oauth.CredentialsError):
+      creds.write()
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  @patch.object(oauth.fileutils, 'write_file')
+  def test_write_locks_resource_during_write(self, mock_write_file,
+                                             unused_mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    lock = creds._lock
+
+    def check_lock_is_locked(*unused_args, **unused_kwargs):
+      self.assertTrue(creds._lock.is_locked)
+
+    mock_write_file.side_effect = check_lock_is_locked
+
+    self.assertFalse(lock.is_locked)
+    creds.refresh(request=MagicMock())
+    self.assertFalse(lock.is_locked)
+    self.assertTrue(mock_write_file.called)
+
+  def test_delete_removes_credentials_file(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    creds.write()
+    self.assertTrue(os.path.exists(self.fake_filename))
+    creds.delete()
+    self.assertFalse(os.path.exists(self.fake_filename))
+
+  @unittest.skipIf(
+      platform.system() == 'Windows',
+      reason=('On Windows, Filelock deletes the lock file each time the lock '
+              'is released. Delete does not remove it.'))
+  def test_delete_removes_lock_file(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    lock_file = '%s.lock' % creds.filename
+    creds.write()
+    self.assertTrue(os.path.exists(lock_file))
+    creds.delete()
+    self.assertFalse(os.path.exists(lock_file))
+
+  def test_delete_is_noop_when_not_using_filelock(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    self.assertIsNone(creds.filename)
+    creds.delete()  # This should not raise an exception.
+
+  def test_revoke_requests_credential_revoke(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    mock_http = MagicMock()
+
+    creds.revoke(http=mock_http)
+
+    uri = mock_http.request.call_args[0][0]
+    self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
+    params = uri[uri.index('?'):]
+    self.assertIn('token=%s' % creds.refresh_token, params)
+    self.assertEqual('GET', mock_http.request.call_args[0][1])
+
+
+class ShortUrlFlowTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_client_id = 'fake_client_id'
+    self.fake_client_secret = 'fake_client_secret'
+    self.fake_scopes = [
+        'fake_api.readonly',
+        'fake_other_api.write',
+    ]
+    self.fake_client_config = {
+        'installed': {
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
+            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+            'token_uri': 'https://oauth2.googleapis.com/token',
+        }
+    }
+    self.long_url = 'http://example.com/some/long/url'
+    self.short_url = 'http://ex.co/short'
+    super(ShortUrlFlowTest, self).setUp()
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = json.dumps({'short_url': self.short_url})
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.short_url, url)
+    self.assertEqual('fake_state', state)
+
+    # Verify request() was called with the expected arguments.
+    self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
+                     mock_http.request.call_args[0][0])
+    self.assertEqual('POST', mock_http.request.call_args[0][1])
+    self.assertIn(self.long_url, mock_http.request.call_args[0][2])
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_request_error(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_http.request.side_effect = Exception()
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 404  # Use a status that is not 200
+    content = json.dumps({'short_url': self.short_url})
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = None
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = json.dumps({})  # This json content contains no "short-url" key
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/controlflow.py

@@ -0,0 +1,99 @@
+"""Methods related to the central control flow of an application."""
+import random
+import sys
+import time
+
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from var import MESSAGE_INVALID_JSON
+
+
+def system_error_exit(return_code, message):
+  """Raises a system exit with the given return code and message.
+
+  Args:
+    return_code: Int, the return code to yield when the system exits.
+    message: An error message to print before the system exits.
+  """
+  if message:
+    display.print_error(message)
+  sys.exit(return_code)
+
+
+def invalid_argument_exit(argument, command):
+  '''Indicate that the argument is not valid for the command.
+
+  Args:
+    argument: the invalid argument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'{argument} is not a valid argument for "{command}"')
+
+def missing_argument_exit(argument, command):
+  '''Indicate that the argument is missing for the command.
+
+  Args:
+    argument: the missingagrument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'missing argument {argument} for "{command}"')
+
+def expected_argument_exit(name, expected, argument):
+  '''Indicate that the argument does not have an expected value for the command.
+
+  Args:
+    name: the field name
+    expected: the expected values
+    argument: the invalid argument
+  '''
+  system_error_exit(
+      2,
+      f'{name} must be one of {expected}; got {argument}')
+
+def csv_field_error_exit(field_name, field_names):
+  """Raises a system exit when a CSV field is malformed.
+
+  Args:
+    field_name: The CSV field name for which a header does not exist in the
+      existing CSV headers.
+    field_names: The known list of CSV headers.
+  """
+  system_error_exit(
+      2,
+      MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                     ','.join(field_names)))
+
+
+def invalid_json_exit(file_name):
+  """Raises a sysyem exit when invalid JSON content is encountered."""
+  system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+
+
+def wait_on_failure(current_attempt_num,
+                    total_num_retries,
+                    error_message,
+                    error_print_threshold=3):
+  """Executes an exponential backoff-style system sleep.
+
+  Args:
+    current_attempt_num: Int, the current number of retries.
+    total_num_retries: Int, the total number of times the current action will be
+      retried.
+    error_message: String, a message to be displayed that will give more context
+      around why the action is being retried.
+    error_print_threshold: Int, the number of attempts which will have their
+      error messages suppressed. Any current_attempt_num greater than
+      error_print_threshold will print the prescribed error.
+  """
+  wait_on_fail = min(2**current_attempt_num,
+                     60) + float(random.randint(1, 1000)) / 1000
+  if current_attempt_num > error_print_threshold:
+    sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+        f'{int(wait_on_fail)} seconds, Retry: '
+        f'{current_attempt_num}/{total_num_retries}\n'))
+    sys.stderr.flush()
+  time.sleep(wait_on_fail)

src/controlflow_test.py

@@ -0,0 +1,106 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+  def test_system_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(1, 'exit message')
+
+  def test_system_error_exit_raises_systemexit_with_return_code(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertEqual(context_manager.exception.code, 100)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+  def test_csv_field_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+
+  def test_csv_field_error_exit_exits_code_2(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    self.assertEqual(context_manager.exception.code, 2)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('aField', printed_message)
+    self.assertIn('unusedField1', printed_message)
+    self.assertIn('unusedField2', printed_message)
+
+  def test_invalid_json_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+
+  def test_invalid_json_exit_exit_exits_code_17(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.invalid_json_exit('filename')
+    self.assertEqual(context_manager.exception.code, 17)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('filename', printed_message)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+    sleep_calls = mock_sleep.call_args_list
+    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+    total_attempts = 20
+    for attempt in range(1, total_attempts + 1):
+      controlflow.wait_on_failure(
+          attempt,
+          total_attempts,
+          'Attempt #%s' % attempt,
+          # Suppress messages while we make a lot of attempts.
+          error_print_threshold=total_attempts + 1)
+      # Wait time may be between 60 and 61 secs, due to rand addition.
+      self.assertLessEqual(mock_sleep.call_args[0][0], 61)
+
+  # Prevent the system from actually sleeping and thus slowing down the test.
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
+    message = 'An error message to display'
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+    self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_only_prints_after_threshold(self, unused_mock_sleep):
+    total_attempts = 5
+    threshold = 3
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      for attempt in range(1, total_attempts + 1):
+        controlflow.wait_on_failure(
+            attempt,
+            total_attempts,
+            'Attempt #%s' % attempt,
+            error_print_threshold=threshold)
+    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

src/cros-aue-dates.json

@@ -1,256 +0,0 @@
-{
-  "acer ac700": "2016-08-01T00:00:00.000Z",
-  "acer c7 chromebook": "2017-10-01T00:00:00.000Z",
-  "acer c7 chromebook (c710)": "2017-10-01T00:00:00.000Z",
-  "acer c720 chromebook": "2019-06-01T00:00:00.000Z",
-  "acer c740 chromebook": "2019-06-01T00:00:00.000Z",
-  "acer chromebase": "2020-08-01T00:00:00.000Z",
-  "acer chromebase 24": "2021-06-01T00:00:00.000Z",
-  "acer chromebook 11 (c720, c720p)": "2019-06-01T00:00:00.000Z",
-  "acer chromebook 11 (c732, c732t, c732l, c732lt)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 11 (c740)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 11 (c771, c771t)": "2022-11-01T00:00:00.000Z",
-  "acer chromebook 11 (cb3-111, c730, c730e)": "2019-08-01T00:00:00.000Z",
-  "acer chromebook 11 (cb3-131, c735)": "2021-01-01T00:00:00.000Z",
-  "acer chromebook 11 (cb311-8h, cb311-8ht)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 11 n7 (c731, c731t)": "2022-01-01T00:00:00.000Z",
-  "acer chromebook 13 (cb5-311)": "2019-09-01T00:00:00.000Z",
-  "acer chromebook 13 (cb713-1w)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook 13(cb5-311, c810)": "2019-09-01T00:00:00.000Z",
-  "acer chromebook 14 (cb3-431)": "2021-06-01T00:00:00.000Z",
-  "acer chromebook 14 for work (cp5-471)": "2022-11-01T00:00:00.000Z",
-  "acer chromebook 15 (c910 / cb5-571)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 15 (cb3-531)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 15 (cb3-532)": "2021-08-01T00:00:00.000Z",
-  "acer chromebook 15 (cb315-1h,cb315-1ht)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 15 (cb5-571, c910)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 15 (cb515-1h,cb515-1ht)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 311": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 311 (c721, c733, c733u, c733t)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 315": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 315 (cb315-2h)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 512 (c851, c851t)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 514": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 714 (cb714-1w / cb714-1wt)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook 715 (cb715-1w / cb715-1wt)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook r11 (cb5-132t, c738t)": "2021-06-01T00:00:00.000Z",
-  "acer chromebook r13 (cb5-312t)": "2021-09-01T00:00:00.000Z",
-  "acer chromebook spin 11 (cp311-h1, cp311-1hn)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook spin 11 (r751t)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook spin 13 (cp713-1wn)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook spin 15 (cp315)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook spin 311 (r721t)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook spin 511": "2025-06-01T00:00:00.000Z",
-  "acer chromebook spin 511 (r752t, r752tn)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook spin 512 (r851tn)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook tab 10": "2023-08-01T00:00:00.000Z",
-  "acer chromebox": "2019-09-01T00:00:00.000Z",
-  "acer chromebox cxi2": "2020-06-01T00:00:00.000Z",
-  "acer chromebox cxi2 / cxv2": "2020-06-01T00:00:00.000Z",
-  "acer chromebox cxi3": "2024-06-01T00:00:00.000Z",
-  "aopen chromebase commercial": "2020-09-01T00:00:00.000Z",
-  "aopen chromebase mini": "2022-02-01T00:00:00.000Z",
-  "aopen chromebox commercial": "2020-09-01T00:00:00.000Z",
-  "aopen chromebox commercial 2": "2024-06-01T00:00:00.000Z",
-  "aopen chromebox mini": "2022-02-01T00:00:00.000Z",
-  "asi chromebook": "2020-06-01T00:00:00.000Z",
-  "asus chromebit cs10": "2020-11-01T00:00:00.000Z",
-  "asus chromebook c200": "2019-06-01T00:00:00.000Z",
-  "asus chromebook c200ma": "2019-06-01T00:00:00.000Z",
-  "asus chromebook c201pa": "2020-06-01T00:00:00.000Z",
-  "asus chromebook c202sa": "2021-06-01T00:00:00.000Z",
-  "asus chromebook c204": "2025-06-01T00:00:00.000Z",
-  "asus chromebook c213na": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c223": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c300": "2019-08-01T00:00:00.000Z",
-  "asus chromebook c300ma": "2019-08-01T00:00:00.000Z",
-  "asus chromebook c300sa / c301sa": "2021-06-01T00:00:00.000Z",
-  "asus chromebook c403": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c423": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c523": "2023-11-01T00:00:00.000Z",
-  "asus chromebook flip c100pa": "2020-07-01T00:00:00.000Z",
-  "asus chromebook flip c101pa": "2023-08-01T00:00:00.000Z",
-  "asus chromebook flip c213": "2023-11-01T00:00:00.000Z",
-  "asus chromebook flip c214": "2025-06-01T00:00:00.000Z",
-  "asus chromebook flip c302": "2022-11-01T00:00:00.000Z",
-  "asus chromebook flip c434": "2024-06-01T00:00:00.000Z",
-  "asus chromebook tablet ct100": "2023-08-01T00:00:00.000Z",
-  "asus chromebox (cn60)": "2019-09-01T00:00:00.000Z",
-  "asus chromebox 2 (cn62)": "2021-06-01T00:00:00.000Z",
-  "asus chromebox 3": "2024-06-01T00:00:00.000Z",
-  "asus chromebox 3 (cn65)": "2024-06-01T00:00:00.000Z",
-  "asus chromebox cn60": "2019-09-01T00:00:00.000Z",
-  "asus chromebox cn62": "2021-06-01T00:00:00.000Z",
-  "bobicus chromebook 11": "2020-06-01T00:00:00.000Z",
-  "chromebook 11 (c730 / cb3-111)": "2019-08-01T00:00:00.000Z",
-  "chromebook 11 (c735)": "2021-01-01T00:00:00.000Z",
-  "chromebook 15 (cb515 - 1ht / 1h)": "2023-11-01T00:00:00.000Z",
-  "chromebook 311 (c721)": "2025-06-01T00:00:00.000Z",
-  "chromebook pcm-116e": "2020-06-01T00:00:00.000Z",
-  "consumer chromebook": "2020-06-01T00:00:00.000Z",
-  "cr-48": "2015-12-01T00:00:00.000Z",
-  "crambo chromebook": "2020-06-01T00:00:00.000Z",
-  "ctl chromebook j41 / j41t": "2023-11-01T00:00:00.000Z",
-  "ctl chromebook nl7": "2023-11-01T00:00:00.000Z",
-  "ctl chromebook nl7t-360 / nl7tw-360": "2023-11-01T00:00:00.000Z",
-  "ctl chromebook tab tx1": "2023-08-01T00:00:00.000Z",
-  "ctl chromebook tablet tx1 for education": "2023-08-01T00:00:00.000Z",
-  "ctl chromebox cbx1": "2024-06-01T00:00:00.000Z",
-  "ctl j2 / j4 chromebook": "2020-06-01T00:00:00.000Z",
-  "ctl j5 chromebook": "2021-08-01T00:00:00.000Z",
-  "ctl n6 education chromebook": "2020-06-01T00:00:00.000Z",
-  "ctl nl61 chromebook": "2021-08-01T00:00:00.000Z",
-  "dell chromebook 11": "2019-06-01T00:00:00.000Z",
-  "dell chromebook 11 (3120)": "2020-06-01T00:00:00.000Z",
-  "dell chromebook 11 (3180)": "2022-05-01T00:00:00.000Z",
-  "dell chromebook 11 (5190)": "2023-11-01T00:00:00.000Z",
-  "dell chromebook 11 2-in-1 (3189)": "2022-05-01T00:00:00.000Z",
-  "dell chromebook 11 2-in-1 (5190)": "2023-11-01T00:00:00.000Z",
-  "dell chromebook 13 (3380)": "2022-11-01T00:00:00.000Z",
-  "dell chromebook 13 (7310)": "2020-09-01T00:00:00.000Z",
-  "dell chromebook 3100": "2025-06-01T00:00:00.000Z",
-  "dell chromebook 3100 2-in-1": "2025-06-01T00:00:00.000Z",
-  "dell chromebook 3400": "2025-06-01T00:00:00.000Z",
-  "dell chromebox": "2019-09-01T00:00:00.000Z",
-  "dell inspiron chromebook 14 2-in-1 (7486)": "2024-06-01T00:00:00.000Z",
-  "edugear chromebook k": "2020-06-01T00:00:00.000Z",
-  "edugear chromebook m": "2020-06-01T00:00:00.000Z",
-  "edugear chromebook r": "2020-06-01T00:00:00.000Z",
-  "edugear cmt chromebook": "2021-08-01T00:00:00.000Z",
-  "edxis chromebook": "2020-06-01T00:00:00.000Z",
-  "edxis education chromebook": "2020-06-01T00:00:00.000Z",
-  "epik 11.6\" chromebook  elb1101": "2020-06-01T00:00:00.000Z",
-  "google chromebook pixel": "2018-06-01T00:00:00.000Z",
-  "google chromebook pixel (2015)": "2020-06-01T00:00:00.000Z",
-  "google cr-48": "2015-12-01T00:00:00.000Z",
-  "google pixel slate": "2024-06-01T00:00:00.000Z",
-  "google pixelbook": "2024-06-01T00:00:00.000Z",
-  "haier chromebook 11": "2020-06-01T00:00:00.000Z",
-  "haier chromebook 11 c": "2021-08-01T00:00:00.000Z",
-  "haier chromebook 11 g2": "2020-09-01T00:00:00.000Z",
-  "haier chromebook 11e": "2020-06-01T00:00:00.000Z",
-  "hexa chromebook pi": "2020-06-01T00:00:00.000Z",
-  "hisense chromebook 11": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 1100-1199 / hp chromebook 11 g1": "2018-10-01T00:00:00.000Z",
-  "hp chromebook 11 2000-2099 / hp chromebook 11 g2": "2019-06-01T00:00:00.000Z",
-  "hp chromebook 11 2100-2199 / hp chromebook 11 g3": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 2200-2299 / hp chromebook 11 g4/g4 ee": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 g1": "2018-10-01T00:00:00.000Z",
-  "hp chromebook 11 g2": "2019-06-01T00:00:00.000Z",
-  "hp chromebook 11 g3": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 g4/g4 ee": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 g5": "2021-07-01T00:00:00.000Z",
-  "hp chromebook 11 g5 / hp chromebook 11-vxxx": "2021-07-01T00:00:00.000Z",
-  "hp chromebook 11 g5 ee": "2022-01-01T00:00:00.000Z",
-  "hp chromebook 11 g6 ee": "2023-11-01T00:00:00.000Z",
-  "hp chromebook 11 g7 ee": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 11a g6 ee": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 13 g1": "2022-11-01T00:00:00.000Z",
-  "hp chromebook 14": "2019-06-01T00:00:00.000Z",
-  "hp chromebook 14 / hp chromebook 14 g5": "2023-11-01T00:00:00.000Z",
-  "hp chromebook 14 ak000-099 / hp chromebook 14 g4": "2021-09-01T00:00:00.000Z",
-  "hp chromebook 14 db0000-db0999": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 14 g3": "2019-10-01T00:00:00.000Z",
-  "hp chromebook 14 g4": "2021-09-01T00:00:00.000Z",
-  "hp chromebook 14 g5": "2023-11-01T00:00:00.000Z",
-  "hp chromebook 14 x000-x999 / hp chromebook 14 g3": "2019-10-01T00:00:00.000Z",
-  "hp chromebook 14a g5": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 15 g1": "2024-06-01T00:00:00.000Z",
-  "hp chromebook x2 ": "2024-06-01T00:00:00.000Z",
-  "hp chromebook x360 11 g1 ee": "2023-11-01T00:00:00.000Z",
-  "hp chromebook x360 11 g2 ee": "2025-06-01T00:00:00.000Z",
-  "hp chromebook x360 14": "2024-06-01T00:00:00.000Z",
-  "hp chromebook x360 14 g1": "2024-06-01T00:00:00.000Z",
-  "hp chromebox cb1-(000-099) / hp chromebox g1/ hp chromebox for meetings": "2019-09-01T00:00:00.000Z",
-  "hp chromebox g1": "2019-09-01T00:00:00.000Z",
-  "hp chromebox g2": "2024-06-01T00:00:00.000Z",
-  "hp pavilion chromebook 14": "2018-02-01T00:00:00.000Z",
-  "jp sa couto chromebook": "2020-06-01T00:00:00.000Z",
-  "lava xolo chromebook": "2020-06-01T00:00:00.000Z",
-  "lenovo 100e chromebook": "2023-11-01T00:00:00.000Z",
-  "lenovo 100e chromebook 2nd gen": "2025-06-01T00:00:00.000Z",
-  "lenovo 100e chromebook 2nd gen mtk": "2025-06-01T00:00:00.000Z",
-  "lenovo 100s chromebook": "2020-09-01T00:00:00.000Z",
-  "lenovo 14e chromebook": "2025-06-01T00:00:00.000Z",
-  "lenovo 300e chromebook": "2025-06-01T00:00:00.000Z",
-  "lenovo 300e chromebook 2nd gen": "2025-06-01T00:00:00.000Z",
-  "lenovo 300e chromebook 2nd gen mtk": "2025-06-01T00:00:00.000Z",
-  "lenovo 500e chromebook": "2023-11-01T00:00:00.000Z",
-  "lenovo 500e chromebook 2nd gen": "2025-06-01T00:00:00.000Z",
-  "lenovo chromebook c330": "2022-06-01T00:00:00.000Z",
-  "lenovo chromebook s330": "2022-06-01T00:00:00.000Z",
-  "lenovo flex 11 chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo ideapad c330 chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo ideapad s330 chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo n20 chromebook": "2019-06-01T00:00:00.000Z",
-  "lenovo n21 chromebook": "2020-06-01T00:00:00.000Z",
-  "lenovo n22 chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo n23 chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo n23 yoga chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo n42 chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo thinkcentre chromebox": "2020-06-01T00:00:00.000Z",
-  "lenovo thinkpad 11e 3rd gen chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo thinkpad 11e 4th gen chromebook": "2023-11-01T00:00:00.000Z",
-  "lenovo thinkpad 11e chromebook": "2019-06-01T00:00:00.000Z",
-  "lenovo thinkpad 11e chromebook (4th gen)/lenovo thinkpad yoga 11e chromebook (4th gen)": "2023-11-01T00:00:00.000Z",
-  "lenovo thinkpad 13": "2022-11-01T00:00:00.000Z",
-  "lenovo thinkpad x131e chromebook": "2018-06-01T00:00:00.000Z",
-  "lenovo yoga c630 chromebook": "2024-06-01T00:00:00.000Z",
-  "lg chromebase (22cb25s)": "2020-06-01T00:00:00.000Z",
-  "lg chromebase (22cv241)": "2019-06-01T00:00:00.000Z",
-  "lumos education chromebook": "2020-06-01T00:00:00.000Z",
-  "m&a chromebook": "2020-06-01T00:00:00.000Z",
-  "mecer chromebook": "2020-06-01T00:00:00.000Z",
-  "mecer v2 chromebook": "2021-08-01T00:00:00.000Z",
-  "medion chromebook akoya s2013 ": "2020-06-01T00:00:00.000Z",
-  "medion chromebook s2015": "2020-06-01T00:00:00.000Z",
-  "multilaser chromebook m11c": "2021-08-01T00:00:00.000Z",
-  "ncomputing chromebook cx100": "2020-06-01T00:00:00.000Z",
-  "ncomputing chromebook cx110": "2020-06-01T00:00:00.000Z",
-  "nexian chromebook 11.6\"": "2020-06-01T00:00:00.000Z",
-  "pcmerge chromebook al116": "2023-11-01T00:00:00.000Z",
-  "pcmerge chromebookpcm-116e/pcm-116eb": "2020-06-01T00:00:00.000Z",
-  "pcmerge chromebookpcm-116t-432b": "2021-08-01T00:00:00.000Z",
-  "poin2 chromebook 11": "2020-06-01T00:00:00.000Z",
-  "poin2 chromebook 11c": "2022-11-01T00:00:00.000Z",
-  "poin2 chromebook 14": "2022-03-01T00:00:00.000Z",
-  "positivo chromebook c216b": "2021-08-01T00:00:00.000Z",
-  "positivo chromebook ch1190": "2020-06-01T00:00:00.000Z",
-  "promethean chromebox": "2024-06-01T00:00:00.000Z",
-  "prowise 11.6\" entry line chromebook": "2020-06-01T00:00:00.000Z",
-  "prowise chromebook eduline": "2023-11-01T00:00:00.000Z",
-  "prowise chromebook entryline": "2020-06-01T00:00:00.000Z",
-  "prowise chromebook proline": "2021-08-01T00:00:00.000Z",
-  "prowise proline chromebook": "2021-08-01T00:00:00.000Z",
-  "rgs education chromebook": "2020-06-01T00:00:00.000Z",
-  "samsung chromebook": "2018-07-01T00:00:00.000Z",
-  "samsung chromebook - xe303": "2018-07-01T00:00:00.000Z",
-  "samsung chromebook 2 11": "2019-06-01T00:00:00.000Z",
-  "samsung chromebook 2 11 - xe500c12": "2020-06-01T00:00:00.000Z",
-  "samsung chromebook 2 13": "2019-06-01T00:00:00.000Z",
-  "samsung chromebook 3": "2021-06-01T00:00:00.000Z",
-  "samsung chromebook plus": "2023-08-01T00:00:00.000Z",
-  "samsung chromebook plus (lte)": "2024-06-01T00:00:00.000Z",
-  "samsung chromebook plus (v2)": "2024-06-01T00:00:00.000Z",
-  "samsung chromebook pro": "2022-11-01T00:00:00.000Z",
-  "samsung chromebook series 5": "2016-06-01T00:00:00.000Z",
-  "samsung chromebook series 5 550": "2017-05-01T00:00:00.000Z",
-  "samsung chromebox series 3": "2018-03-01T00:00:00.000Z",
-  "sector 5 e1 rugged chromebook": "2020-06-01T00:00:00.000Z",
-  "sector 5 e3 chromebook": "2023-11-01T00:00:00.000Z",
-  "senkatel c1101 chromebook": "2020-06-01T00:00:00.000Z",
-  "thinkpad 11e chromebook 3rd gen (yoga/clamshell)": "2021-06-01T00:00:00.000Z",
-  "thinkpad 13 chromebook": "2022-11-01T00:00:00.000Z",
-  "toshiba chromebook": "2019-06-01T00:00:00.000Z",
-  "toshiba chromebook 2": "2020-06-01T00:00:00.000Z",
-  "toshiba chromebook 2 (2015 edition)": "2020-09-01T00:00:00.000Z",
-  "true idc chromebook": "2020-06-01T00:00:00.000Z",
-  "true idc chromebook 11": "2020-06-01T00:00:00.000Z",
-  "videonet chromebook": "2020-06-01T00:00:00.000Z",
-  "videonet chromebook bl10": "2020-06-01T00:00:00.000Z",
-  "viewsonic nmp660 chromebox": "2024-06-01T00:00:00.000Z",
-  "viglen chromebook 11": "2020-06-01T00:00:00.000Z",
-  "viglen chromebook 11c": "2023-11-01T00:00:00.000Z",
-  "viglen chromebook 360": "2021-08-01T00:00:00.000Z",
-  "xolo chromebook": "2020-06-01T00:00:00.000Z"
-}

src/display.py

@@ -0,0 +1,234 @@
+"""Methods related to display of information to the user."""
+
+import csv
+import io
+import sys
+import webbrowser
+
+import dateutil
+import googleapiclient.http
+
+#TODO: get rid of these hacks
+import __main__
+from var import *
+import controlflow
+import gapi
+
+
+def current_count(i, count):
+  return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
+
+def current_count_nl(i, count):
+  return f' ({i}/{count})\n' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
+
+def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
+  fields = fieldsChoiceMap[fieldName.lower()]
+  if isinstance(fields, list):
+    fieldsList.extend(fields)
+  else:
+    fieldsList.append(fields)
+
+# Write a CSV file
+def add_titles_to_csv_file(addTitles, titles):
+  for title in addTitles:
+    if title not in titles:
+      titles.append(title)
+
+def add_row_titles_to_csv_file(row, csvRows, titles):
+  csvRows.append(row)
+  for title in row:
+    if title not in titles:
+      titles.append(title)
+
+# fieldName is command line argument
+# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
+#ARGUMENT_TO_PROPERTY_MAP = {
+#  u'admincreated': [u'adminCreated'],
+#  u'aliases': [u'aliases', u'nonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles, titles):
+  for ftList in fieldNameMap[fieldName]:
+    if ftList not in fieldsTitles:
+      fieldsList.append(ftList)
+      fieldsTitles[ftList] = ftList
+      add_titles_to_csv_file([ftList], titles)
+
+# fieldName is command line argument
+# fieldNameTitleMap maps fieldName to API field name and CSV file header
+#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
+#  u'admincreated': [u'adminCreated', u'Admin_Created'],
+#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList, fieldsTitles, titles):
+  ftList = fieldNameTitleMap[fieldName]
+  for i in range(0, len(ftList), 2):
+    if ftList[i] not in fieldsTitles:
+      fieldsList.append(ftList[i])
+      fieldsTitles[ftList[i]] = ftList[i+1]
+      add_titles_to_csv_file([ftList[i+1]], titles)
+
+def sort_csv_titles(firstTitle, titles):
+  restoreTitles = []
+  for title in firstTitle:
+    if title in titles:
+      titles.remove(title)
+      restoreTitles.append(title)
+  titles.sort()
+  for title in restoreTitles[::-1]:
+    titles.insert(0, title)
+
+def QuotedArgumentList(items):
+  return ' '.join([item if item and (item.find(' ') == -1) and (item.find(',') == -1) else '"'+item+'"' for item in items])
+
+def write_csv_file(csvRows, titles, list_type, todrive):
+  def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
+    if not rowDate or not isinstance(rowDate, str):
+      return False
+    try:
+      rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
+      if dateMode:
+        rowDate = datetime.datetime(rowTime.year, rowTime.month, rowTime.day).isoformat()+'Z'
+    except ValueError:
+      rowDate = NEVER_TIME
+    if op == '<':
+      return rowDate < filterDate
+    if op == '<=':
+      return rowDate <= filterDate
+    if op == '>':
+      return rowDate > filterDate
+    if op == '>=':
+      return rowDate >= filterDate
+    if op == '!=':
+      return rowDate != filterDate
+    return rowDate == filterDate
+
+  def rowCountFilterMatch(rowCount, op, filterCount):
+    if isinstance(rowCount, str):
+      if not rowCount.isdigit():
+        return False
+      rowCount = int(rowCount)
+    elif not isinstance(rowCount, int):
+      return False
+    if op == '<':
+      return rowCount < filterCount
+    if op == '<=':
+      return rowCount <= filterCount
+    if op == '>':
+      return rowCount > filterCount
+    if op == '>=':
+      return rowCount >= filterCount
+    if op == '!=':
+      return rowCount != filterCount
+    return rowCount == filterCount
+  def rowBooleanFilterMatch(rowBoolean, filterBoolean):
+    if not isinstance(rowBoolean, bool):
+      return False
+    return rowBoolean == filterBoolean
+
+  def headerFilterMatch(title):
+    for filterStr in GC_Values[GC_CSV_HEADER_FILTER]:
+      if filterStr.match(title):
+        return True
+    return False
+
+  if GC_Values[GC_CSV_ROW_FILTER]:
+    for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
+      if column not in titles:
+        sys.stderr.write(f'WARNING: Row filter column "{column}" is not in output columns\n')
+        continue
+      if filterVal[0] == 'regex':
+        csvRows = [row for row in csvRows if filterVal[1].search(str(row.get(column, '')))]
+      elif filterVal[0] == 'notregex':
+        csvRows = [row for row in csvRows if not filterVal[1].search(str(row.get(column, '')))]
+      elif filterVal[0] in ['date', 'time']:
+        csvRows = [row for row in csvRows if rowDateTimeFilterMatch(filterVal[0] == 'date', row.get(column, ''), filterVal[1], filterVal[2])]
+      elif filterVal[0] == 'count':
+        csvRows = [row for row in csvRows if rowCountFilterMatch(row.get(column, 0), filterVal[1], filterVal[2])]
+      else: #boolean
+        csvRows = [row for row in csvRows if rowBooleanFilterMatch(row.get(column, False), filterVal[1])]
+  if GC_Values[GC_CSV_HEADER_FILTER]:
+    titles = [t for t in titles if headerFilterMatch(t)]
+    if not titles:
+      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER\n')
+      return
+  csv.register_dialect('nixstdout', lineterminator='\n')
+  if todrive:
+    write_to = io.StringIO()
+  else:
+    write_to = sys.stdout
+  writer = csv.DictWriter(write_to, fieldnames=titles, dialect='nixstdout', extrasaction='ignore', quoting=csv.QUOTE_MINIMAL)
+  try:
+    writer.writerow(dict((item, item) for item in writer.fieldnames))
+    writer.writerows(csvRows)
+  except IOError as e:
+    controlflow.system_error_exit(6, e)
+  if todrive:
+    admin_email = __main__._getValueFromOAuth('email')
+    _, drive = __main__.buildDrive3GAPIObject(admin_email)
+    if not drive:
+      print(f'''\nGAM is not authorized to create Drive files. Please run:
+gam user {admin_email} check serviceaccount
+and follow recommend steps to authorize GAM for Drive access.''')
+      sys.exit(5)
+    result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
+    columns = len(titles)
+    rows = len(csvRows)
+    cell_count = rows * columns
+    data_size = len(write_to.getvalue())
+    max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
+    if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
+      print(f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}')
+      mimeType = 'text/csv'
+    else:
+      mimeType = MIMETYPE_GA_SPREADSHEET
+    body = {'description': QuotedArgumentList(sys.argv),
+            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
+            'mimeType': mimeType}
+    result = gapi.call(drive.files(), 'create', fields='webViewLink',
+                       body=body,
+                       media_body=googleapiclient.http.MediaInMemoryUpload(write_to.getvalue().encode(),
+                                                                           mimetype='text/csv'))
+    file_url = result['webViewLink']
+    if GC_Values[GC_NO_BROWSER]:
+      msg_txt = f'Drive file uploaded to:\n {file_url}'
+      msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
+      __main__.send_email(msg_subj, msg_txt)
+      print(msg_txt)
+    else:
+      webbrowser.open(file_url)
+
+def print_error(message):
+  """Prints a one-line error message to stderr in a standard format."""
+  sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
+
+
+def print_warning(message):
+  """Prints a one-line warning message to stderr in a standard format."""
+  sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
+
+def print_json(object_value, spacing=''):
+  """Prints Dict or Array to screen in clean human-readable format.."""
+  if isinstance(object_value, list):
+    if len(object_value) == 1 and isinstance(object_value[0], (str, int, bool)):
+      sys.stdout.write(f'{object_value[0]}\n')
+      return
+    if spacing:
+      sys.stdout.write('\n')
+    for i, a_value in enumerate(object_value):
+      if isinstance(a_value, (str, int, bool)):
+        sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
+      else:
+        sys.stdout.write(f' {spacing}{i+1}) ')
+        print_json(a_value, f' {spacing}')
+  elif isinstance(object_value, dict):
+    for key in ['kind', 'etag', 'etags']:
+      object_value.pop(key, None)
+    for another_object, another_value in object_value.items():
+      sys.stdout.write(f' {spacing}{another_object}: ')
+      print_json(another_value, f' {spacing}')
+  else:
+    sys.stdout.write(f'{object_value}\n')

src/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+import display
+from var import ERROR_PREFIX
+from var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+  def test_print_error_prints_to_stderr(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  def test_print_error_prints_error_prefix(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(ERROR_PREFIX), printed_message.find(message),
+        'The error prefix does not appear before the error message')
+
+  def test_print_error_ends_message_with_newline(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The error message does not end in a newline.')
+
+  def test_print_warning_prints_to_stderr(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  def test_print_warning_prints_error_prefix(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(WARNING_PREFIX), printed_message.find(message),
+        'The warning prefix does not appear before the error message')
+
+  def test_print_warning_ends_message_with_newline(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The warning message does not end in a newline.')

src/fileutils.py

@@ -0,0 +1,179 @@
+"""Common file operations."""
+
+import io
+import os
+import sys
+
+import controlflow
+import display
+from var import GM_Globals
+from var import GM_SYS_ENCODING
+from var import UTF8_SIG
+
+
+def _open_file(filename, mode, encoding=None, newline=None):
+  """Opens a file with no error handling."""
+  # Determine which encoding to use
+  if 'b' in mode:
+    encoding = None
+  elif not encoding:
+    encoding = GM_Globals[GM_SYS_ENCODING]
+  elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
+    encoding = UTF8_SIG
+
+  return open(
+      os.path.expanduser(filename), mode, newline=newline, encoding=encoding)
+
+
+def open_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              strip_utf_bom=False):
+  """Opens a file.
+
+  Args:
+    filename: String, the name of the file to open, or '-' to use stdin/stdout,
+      to read/write, depending on the mode param, respectively.
+    mode: String, the common file mode to open the file with. Default is read.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    strip_utf_bom: Boolean, True if the file being opened should seek past the
+      UTF Byte Order Mark before being returned.
+        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+
+  Returns:
+    The opened file.
+  """
+  try:
+    if filename == '-':
+      # Read from stdin, rather than a file
+      if 'r' in mode:
+        return io.StringIO(str(sys.stdin.read()))
+      return sys.stdout
+
+    # Open a file on disk
+    f = _open_file(filename, mode, newline=newline, encoding=encoding)
+    if strip_utf_bom:
+      utf_bom = u'\ufeff'
+      has_bom = False
+
+      if 'b' in mode:
+        has_bom = f.read(3).decode('UTF-8') == utf_bom
+      elif f.encoding and not f.encoding.lower().startswith('utf'):
+        # Convert UTF BOM into ISO-8859-1 via Bytes
+        utf8_bom_bytes = utf_bom.encode('UTF-8')
+        iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
+            'iso-8859-1')
+        has_bom = f.read(3).encode('iso-8859-1', 'replace') == iso_8859_1_bom
+      else:
+        has_bom = f.read(1) == utf_bom
+
+      if not has_bom:
+        f.seek(0)
+
+    return f
+
+  except IOError as e:
+    controlflow.system_error_exit(6, e)
+
+
+def close_file(f, force_flush=False):
+  """Closes a file.
+
+  Args:
+    f: The file to close
+    force_flush: Flush file to disk emptying Python and OS caches. See:
+       https://stackoverflow.com/a/13762137/1503886
+
+  Returns:
+    Boolean, True if the file was successfully closed. False if an error
+        was encountered while closing.
+  """
+  if force_flush:
+    f.flush()
+    os.fsync(f.fileno())
+  try:
+    f.close()
+    return True
+  except IOError as e:
+    display.print_error(e)
+    return False
+
+
+def read_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              continue_on_error=False,
+              display_errors=True):
+  """Reads a file from disk.
+
+  Args:
+    filename: String, the path of the file to open from disk, or "-" to read
+      from stdin.
+    mode: String, the mode in which to open the file.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    The contents of the file, or stdin if filename == "-". Returns None if
+    an error is encountered and continue_on_errors is True.
+  """
+  try:
+    if filename == '-':
+      # Read from stdin, rather than a file.
+      return str(sys.stdin.read())
+
+    with _open_file(filename, mode, newline=newline, encoding=encoding) as f:
+      return f.read()
+
+  except IOError as e:
+    if continue_on_error:
+      if display_errors:
+        display.print_warning(e)
+      return None
+    controlflow.system_error_exit(6, e)
+  except (LookupError, UnicodeDecodeError, UnicodeError) as e:
+    controlflow.system_error_exit(2, str(e))
+
+
+def write_file(filename,
+               data,
+               mode='w',
+               continue_on_error=False,
+               display_errors=True):
+  """Writes data to a file.
+
+  Args:
+    filename: String, the path of the file to write to disk.
+    data: Serializable data to write to the file.
+    mode: String, the mode in which to open the file and write to it.
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    Boolean, True if the write operation succeeded, or False if not.
+  """
+  try:
+    with _open_file(filename, mode) as f:
+      f.write(data)
+    return True
+
+  except IOError as e:
+    if continue_on_error:
+      if display_errors:
+        display.print_error(e)
+      return False
+    else:
+      controlflow.system_error_exit(6, e)

src/fileutils_test.py

@@ -0,0 +1,234 @@
+"""Tests for fileutils."""
+
+import io
+import os
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import fileutils
+
+
+class FileutilsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_path = '/some/path/to/file'
+    super(FileutilsTest, self).setUp()
+
+  @patch.object(fileutils.sys, 'stdin')
+  def test_open_file_stdin(self, mock_stdin):
+    mock_stdin.read.return_value = 'some stdin content'
+    f = fileutils.open_file('-', mode='r')
+    self.assertIsInstance(f, fileutils.io.StringIO)
+    self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
+
+  def test_open_file_stdout(self):
+    f = fileutils.open_file('-', mode='w')
+    self.assertEqual(fileutils.sys.stdout, f)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_opens_correct_path(self, mock_open):
+    f = fileutils.open_file(self.fake_path)
+    self.assertEqual(self.fake_path, mock_open.call_args[0][0])
+    self.assertEqual(mock_open.return_value, f)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_expands_user_file_path(self, mock_open):
+    file_path = '~/some/path/containing/tilde/shortcut/to/home'
+    fileutils.open_file(file_path)
+    opened_path = mock_open.call_args[0][0]
+    home_path = os.environ.get('HOME')
+    self.assertIsNotNone(home_path)
+    self.assertIn(home_path, opened_path)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_opens_correct_mode(self, mock_open):
+    fileutils.open_file(self.fake_path)
+    self.assertEqual('r', mock_open.call_args[0][1])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_encoding_for_binary(self, mock_open):
+    fileutils.open_file(self.fake_path, mode='b')
+    self.assertIsNone(mock_open.call_args[1]['encoding'])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_default_system_encoding(self, mock_open):
+    fileutils.open_file(self.fake_path)
+    self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
+                     mock_open.call_args[1]['encoding'])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_utf8_encoding_specified(self, mock_open):
+    fileutils.open_file(self.fake_path, encoding='UTF-8')
+    self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
+
+  def test_open_file_strips_utf_bom_in_utf(self):
+    bom_prefixed_data = u'\ufefffoobar'
+    fake_file = io.StringIO(bom_prefixed_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      self.assertEqual('foobar', f.read())
+
+  def test_open_file_strips_utf_bom_in_non_utf(self):
+    bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
+
+    # We need to trick the method under test into believing that a StringIO
+    # instance is a file with an encoding. Since StringIO does not usually have,
+    # an encoding, we'll mock it and add our own encoding, but send the other
+    # methods in use (read and seek) back to the real StringIO object.
+    real_stringio = io.StringIO(bom_prefixed_data)
+    mock_file = MagicMock(spec=io.StringIO)
+    mock_file.read.side_effect = real_stringio.read
+    mock_file.seek.side_effect = real_stringio.seek
+    mock_file.encoding = 'iso-8859-1'
+
+    mock_open = MagicMock(spec=open, return_value=mock_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      self.assertEqual('foobar', f.read())
+
+  def test_open_file_strips_utf_bom_in_binary(self):
+    bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
+    fake_file = io.BytesIO(bom_prefixed_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, mode='rb', strip_utf_bom=True)
+      self.assertEqual(b'foobar', f.read())
+
+  def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
+    no_bom_data = 'This data has no BOM'
+    fake_file = io.StringIO(no_bom_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      # Since there was no opening BOM, we should be back at the beginning of
+      # the file.
+      self.assertEqual(fake_file.tell(), 0)
+      self.assertEqual(f.read(), no_bom_data)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_exits_on_io_error(self, mock_open):
+    mock_open.side_effect = IOError('Fake IOError')
+    with self.assertRaises(SystemExit) as context:
+      fileutils.open_file(self.fake_path)
+    self.assertEqual(context.exception.code, 6)
+
+  def test_close_file_closes_file_successfully(self):
+    mock_file = MagicMock()
+    self.assertTrue(fileutils.close_file(mock_file))
+    self.assertEqual(mock_file.close.call_count, 1)
+
+  def test_close_file_with_error(self):
+    mock_file = MagicMock()
+    mock_file.close.side_effect = IOError()
+    self.assertFalse(fileutils.close_file(mock_file))
+    self.assertEqual(mock_file.close.call_count, 1)
+
+  @patch.object(fileutils.sys, 'stdin')
+  def test_read_file_from_stdin(self, mock_stdin):
+    mock_stdin.read.return_value = 'some stdin content'
+    self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_default_params(self, mock_open_file):
+    fake_content = 'some fake content'
+    mock_open_file.return_value.__enter__().read.return_value = fake_content
+    self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
+    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+    self.assertEqual(mock_open_file.call_args[0][1], 'r')
+    self.assertIsNone(mock_open_file.call_args[1]['newline'])
+
+  @patch.object(fileutils.display, 'print_warning')
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_continues_on_errors_without_displaying(
+      self, mock_open_file, mock_print_warning):
+    mock_open_file.side_effect = IOError()
+    contents = fileutils.read_file(
+        self.fake_path, continue_on_error=True, display_errors=False)
+    self.assertIsNone(contents)
+    self.assertFalse(mock_print_warning.called)
+
+  @patch.object(fileutils.display, 'print_warning')
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_displays_errors(self, mock_open_file, mock_print_warning):
+    mock_open_file.side_effect = IOError()
+    fileutils.read_file(
+        self.fake_path, continue_on_error=True, display_errors=True)
+    self.assertTrue(mock_print_warning.called)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_6_when_continue_on_error_is_false(
+      self, mock_open_file):
+    mock_open_file.side_effect = IOError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path, continue_on_error=False)
+    self.assertEqual(context.exception.code, 6)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
+    mock_open_file.return_value.__enter__().read.side_effect = LookupError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
+    mock_open_file.return_value.__enter__().read.side_effect = UnicodeError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
+    fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0, 1,
+                                           'testing only')
+    mock_open_file.return_value.__enter__().read.side_effect = fake_decode_error
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_writes_data_to_file(self, mock_open_file):
+    fake_data = 'some fake data'
+    fileutils.write_file(self.fake_path, fake_data)
+    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+    self.assertEqual(mock_open_file.call_args[0][1], 'w')
+
+    opened_file = mock_open_file.return_value.__enter__()
+    self.assertTrue(opened_file.write.called)
+    self.assertEqual(opened_file.write.call_args[0][0], fake_data)
+
+  @patch.object(fileutils.display, 'print_error')
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_continues_on_errors_without_displaying(
+      self, mock_open_file, mock_print_error):
+    mock_open_file.side_effect = IOError()
+    status = fileutils.write_file(
+        self.fake_path,
+        'foo data',
+        continue_on_error=True,
+        display_errors=False)
+    self.assertFalse(status)
+    self.assertFalse(mock_print_error.called)
+
+  @patch.object(fileutils.display, 'print_error')
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
+    mock_open_file.side_effect = IOError()
+    fileutils.write_file(
+        self.fake_path, 'foo data', continue_on_error=True, display_errors=True)
+    self.assertTrue(mock_print_error.called)
+
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_exits_code_6_when_continue_on_error_is_false(
+      self, mock_open_file):
+    mock_open_file.side_effect = IOError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.write_file(self.fake_path, 'foo data', continue_on_error=False)
+    self.assertEqual(context.exception.code, 6)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gam-install.sh

@@ -10,6 +10,7 @@ OPTIONS:
    -d      Directory where gam folder will be installed. Default is \$HOME/bin/
    -a      Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m".
    -o      OS we are running (linux, macos). Default is to detect your OS with "uname -s".
+   -b      OS version. Default is to detect on MacOS and Linux.
    -l      Just upgrade GAM to latest version. Skips project creation and auth.
    -p      Profile update (true, false). Should script add gam command to environment. Default is true.
    -u      Admin user email address to use with GAM. Default is to prompt.
@@ -21,20 +22,23 @@ EOF
 target_dir="$HOME/bin"
 gamarch=$(uname -m)
 gamos=$(uname -s)
+osversion=""
 update_profile=true
 upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.23 2.19 2.15"
+gam_glibc_vers="2.27 2.23 2.19 2.15"
+gam_macos_vers="10.14.6 10.13.6 10.12.6"
 
-while getopts "hd:a:o:lp:u:r:v:" OPTION
+while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
      case $OPTION in
          h) usage; exit;;
          d) target_dir="$OPTARG";;
          a) gamarch="$OPTARG";;
          o) gamos="$OPTARG";;
+         b) osversion="$OPTARG";;
          l) upgrade_only=true;;
          p) update_profile="$OPTARG";;
          u) adminuser="$OPTARG";;
@@ -48,7 +52,7 @@ done
 target_dir=${target_dir%/}
 
 update_profile() {
-	[ -f "$1" ] || return 1
+	[ $2 -eq 1 ] || [ -f "$1" ] || return 1
 
 	grep -F "$alias_line" "$1" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
@@ -79,13 +83,26 @@ echo -e '\x1B[0m'
 
 version_gt()
 {
-test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+# MacOS < 10.13 doesn't support sort -V
+echo "" | sort -V > /dev/null 2>&1
+vsort_failed=$?
+if [ "${1}" = "${2}" ]; then
+  true
+elif (( $vsort_failed != 0 )); then
+  false
+else
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+fi
 }
 
 case $gamos in
   [lL]inux)
     gamos="linux"
-    this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    if [ "$osversion" == "" ]; then
+      this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    else
+      this_glibc_ver=$osversion
+    fi
     echo "This Linux distribution uses glibc $this_glibc_ver"
     useglibc="legacy"
     for gam_glibc_ver in $gam_glibc_vers; do
@@ -97,24 +114,33 @@ case $gamos in
     done
     case $gamarch in
       x86_64) gamfile="linux-x86_64-$useglibc.tar.xz";;
-      i?86) gamfile="linux-i686.tar.xz";;
-      arm|armv7l) gamfile="linux-armv7l.tar.xz";;
-      arm64|aarch64) gamfile="linux-aarch64.tar.xz";;
+      arm64|aarch64) gamfile="linux-arm64-$useglibc.tar.xz";;
       *)
-        echo_red "ERROR: this installer currently only supports i386, x86_64, arm and arm64 Linux. Looks like you're running on $gamarch. Exiting."
+        echo_red "ERROR: this installer currently only supports x86_64 and arm64 Linux. Looks like you're running on $gamarch. Exiting."
         exit
     esac
     ;;
   [Mm]ac[Oo][sS]|[Dd]arwin)
-    osver=$(sw_vers -productVersion | awk -F'.' '{print $2}')
-    if (( $osver < 13 )); then
-      echo_red "ERROR: GAM currently requires MacOS 10.13 or newer. You are running MacOS 10.$osver. Please upgrade." 
-      exit
-    else
-      echo_green "Good, you're running MacOS 10.$osver..."
-    fi
     gamos="macos"
-    gamfile="macos-x86_64.tar.xz"
+    if [ "$osversion" == "" ]; then
+      this_macos_ver=$(sw_vers -productVersion)
+    else
+      this_macos_ver=$osversion
+    fi
+    echo "You are running MacOS $this_macos_ver"
+    use_macos_ver=""
+    for gam_macos_ver in $gam_macos_vers; do
+      if version_gt $this_macos_ver $gam_macos_ver; then
+        use_macos_ver="MacOS$gam_macos_ver"
+        echo_green "Using GAM compiled on $use_macos_ver"
+        break
+      fi
+    done
+    if [ "$use_macos_ver" == "" ]; then
+      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
+      exit
+    fi
+    gamfile="macos-x86_64-$use_macos_ver.tar.xz"
     ;;
   *)
     echo_red "Sorry, this installer currently only supports Linux and MacOS. Looks like you're runnning on $gamos. Exiting."
@@ -161,11 +187,16 @@ try:
 except KeyError:
   print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
-pycmd="python"
+pycmd="python3"
 $pycmd -V >/dev/null 2>&1
 rc=$?
 if (( $rc != 0 )); then
-  pycmd="python3"
+  pycmd="python"
+fi
+$pycmd -V >/dev/null 2>&1
+rc=$?
+if (( $rc != 0 )); then
+  pycmd="python2"
 fi
 $pycmd -V >/dev/null 2>&1
 rc=$?
@@ -203,9 +234,21 @@ else
   echo_green "Finished extracting GAM archive."
 fi
 
+# Update profile to add gam command
+if [ "$update_profile" = true ]; then
+  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
+  if [ "$gamos" == "linux" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0
+  elif [ "$gamos" == "macos" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0 || update_profile "$HOME/.profile" 1
+  fi
+else
+  echo_yellow "skipping profile update."
+fi
+
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
-  "$target_dir/gam/gam" version
+  "$target_dir/gam/gam" version extended
   rc=$?
   if (( $rc != 0 )); then
     echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -216,18 +259,6 @@ if [ "$upgrade_only" = true ]; then
   exit
 fi
 
-# Update profile to add gam command
-if [ "$update_profile" = true ]; then
-  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
-  if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile"
-  elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile"
-  fi
-else
-  echo_yellow "skipping profile update."
-fi
-
 while true; do
   read -p "Can you run a full browser on this machine? (usually Y for MacOS, N for Linux if you SSH into this machine) " yn
   case $yn in
@@ -328,7 +359,7 @@ while $project_created; do
 done
 
 echo_green "Here's information about your new GAM installation:"
-"$target_dir/gam/gam" version
+"$target_dir/gam/gam" version extended
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."

src/gapi/__init__.py

@@ -0,0 +1,328 @@
+"""Methods related to execution of GAPI requests."""
+
+import sys
+
+import googleapiclient.errors
+import google.auth.exceptions
+import httplib2
+
+import controlflow
+import display
+from gapi import errors
+import transport
+from var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+                 GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
+                 MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
+                 MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
+
+
+def call(service,
+         function,
+         silent_errors=False,
+         soft_errors=False,
+         throw_reasons=None,
+         retry_reasons=None,
+         **kwargs):
+  """Executes a single request on a Google service function.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    silent_errors: Bool, If True, error messages are suppressed when
+      encountered.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A response object for the corresponding Google API call.
+  """
+  if throw_reasons is None:
+    throw_reasons = []
+  if retry_reasons is None:
+    retry_reasons = []
+
+  method = getattr(service, function)
+  retries = 10
+  parameters = dict(
+      list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+  for n in range(1, retries + 1):
+    try:
+      return method(**parameters).execute()
+    except googleapiclient.errors.HttpError as e:
+      http_status, reason, message = errors.get_gapi_error_detail(
+          e,
+          soft_errors=soft_errors,
+          silent_errors=silent_errors,
+          retry_on_http_error=n < 3)
+      if http_status == -1:
+        # The error detail indicated that we should retry this request
+        # We'll refresh credentials and make another pass
+        service._http.request.credentials.refresh(transport.create_http())
+        continue
+      if http_status == 0:
+        return None
+
+      is_known_error_reason = reason in [r.value for r in errors.ErrorReason]
+      if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
+        if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
+          raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
+              message)
+        raise e
+      if (n != retries) and (is_known_error_reason and errors.ErrorReason(
+          reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
+        controlflow.wait_on_failure(n, retries, reason)
+        continue
+      if soft_errors:
+        display.print_error(f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}')
+        return None
+      controlflow.system_error_exit(
+          int(http_status), f'{http_status}: {message} - {reason}')
+    except google.auth.exceptions.RefreshError as e:
+      handle_oauth_token_error(
+          e, soft_errors or
+          errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+      if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
+        raise errors.GapiServiceNotAvailableError(str(e))
+      display.print_error(f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
+      return None
+    except ValueError as e:
+      if hasattr(service._http, 'cache') and service._http.cache is not None:
+        service._http.cache = None
+        continue
+      controlflow.system_error_exit(4, str(e))
+    except (httplib2.ServerNotFoundError, RuntimeError) as e:
+      if n != retries:
+        service._http.connections = {}
+        controlflow.wait_on_failure(n, retries, str(e))
+        continue
+      controlflow.system_error_exit(4, str(e))
+    except TypeError as e:
+      controlflow.system_error_exit(4, str(e))
+
+
+def get_items(service,
+              function,
+              items='items',
+              throw_reasons=None,
+              retry_reasons=None,
+              **kwargs):
+  """Gets a single page of items from a Google service function that is paged.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the service
+      method's response object.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    The list of items in the first page of a response.
+  """
+  results = call(
+      service,
+      function,
+      throw_reasons=throw_reasons,
+      retry_reasons=retry_reasons,
+      **kwargs)
+  if results:
+    return results.get(items, [])
+  return []
+
+
+def _get_max_page_size_for_api_call(service, function, **kwargs):
+  """Gets the maximum number of results supported for a single API call.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of the service method to check for max page size.
+    **kwargs: Additional params that will be passed to the request method.
+
+  Returns:
+    Int, A value from discovery if it exists, otherwise value from
+        MAX_RESULTS_API_EXCEPTIONS, otherwise None
+  """
+  method = getattr(service, function)
+  api_id = method(**kwargs).methodId
+  for resource in service._rootDesc.get('resources', {}).values():
+    for a_method in resource.get('methods', {}).values():
+      if a_method.get('id') == api_id:
+        if not a_method.get('parameters') or a_method['parameters'].get(
+            'pageSize') or not a_method['parameters'].get('maxResults'):
+          # Make sure API call supports maxResults. For now we don't care to
+          # set pageSize since all known pageSize API calls have
+          # default pageSize == max pageSize.
+          return None
+        known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
+        max_results = a_method['parameters']['maxResults'].get(
+            'maximum', known_api_max)
+        return {'maxResults': max_results}
+
+  return None
+
+
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+def got_total_items_msg(items, eol):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+def got_total_items_first_last_msg(items):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}'+'\n'
+
+def get_all_pages(service,
+                  function,
+                  items='items',
+                  page_message=None,
+                  message_attribute=None,
+                  soft_errors=False,
+                  throw_reasons=None,
+                  retry_reasons=None,
+                  **kwargs):
+  """Aggregates and returns all pages of a Google service function response.
+
+  All pages of items are aggregated and returned as a single list.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the method's
+      response object. The items in this field will be aggregated across all
+      pages and returned.
+    page_message: String, a message to be displayed to the user during paging.
+      Template strings allow for dynamic content to be inserted during paging.
+        Supported template strings:
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
+            pages.
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
+            display a unique property of the first item in the current page.
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
+            display a unique property of the last item in the current page.
+    message_attribute: String, the name of a signature field within a single
+      returned item which identifies that unique item. This field is used with
+      `page_message` to templatize a paging status message.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A list of all items received from all paged responses.
+  """
+  if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
+    page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
+    if page_key:
+      kwargs.update(page_key)
+  all_items = []
+  page_token = None
+  total_items = 0
+  while True:
+    page = call(
+        service,
+        function,
+        soft_errors=soft_errors,
+        throw_reasons=throw_reasons,
+        retry_reasons=retry_reasons,
+        pageToken=page_token,
+        **kwargs)
+    if page:
+      page_token = page.get('nextPageToken')
+      page_items = page.get(items, [])
+      num_page_items = len(page_items)
+      total_items += num_page_items
+      all_items.extend(page_items)
+    else:
+      page_token = None
+      num_page_items = 0
+
+    # Show a paging message to the user that indicates paging progress
+    if page_message:
+      show_message = page_message.replace(TOTAL_ITEMS_MARKER, str(total_items))
+      if message_attribute:
+        first_item = page_items[0] if num_page_items > 0 else {}
+        last_item = page_items[-1] if num_page_items > 1 else first_item
+        show_message = show_message.replace(FIRST_ITEM_MARKER, str(first_item.get(message_attribute, '')))
+        show_message = show_message.replace(LAST_ITEM_MARKER, str(last_item.get(message_attribute, '')))
+      sys.stderr.write('\r')
+      sys.stderr.flush()
+      sys.stderr.write(show_message)
+
+    if not page_token:
+      # End the paging status message and return all items.
+      if page_message and (page_message[-1] != '\n'):
+        sys.stderr.write('\r\n')
+        sys.stderr.flush()
+      return all_items
+
+
+# TODO: Make this private once all execution related items that use this method
+# have been brought into this file
+def handle_oauth_token_error(e, soft_errors):
+  """On a token error, exits the application and writes a message to stderr.
+
+  Args:
+    e: google.auth.exceptions.RefreshError, The error to handle.
+    soft_errors: Boolean, if True, suppresses any applicable errors and instead
+      returns to the caller.
+  """
+  token_error = str(e).replace('.', '')
+  if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
+      'Invalid response'):
+    if soft_errors:
+      return
+    if not GM_Globals[GM_CURRENT_API_USER]:
+      display.print_error(
+          MESSAGE_API_ACCESS_DENIED.format(
+              GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+              ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+      controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
+    else:
+      controlflow.system_error_exit(
+          19,
+          MESSAGE_SERVICE_NOT_APPLICABLE.format(
+              GM_Globals[GM_CURRENT_API_USER]))
+  controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
+
+def get_enum_values_minus_unspecified(values):
+  return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gapi/__init___test.py

@@ -0,0 +1,502 @@
+"""Tests for gapi."""
+
+import json
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import gapi
+from gapi import errors
+
+
+def create_http_error(status, reason, message):
+  """Creates a HttpError object similar to most Google API Errors.
+
+  Args:
+    status: Int, the error's HTTP response status number.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    googleapiclient.errors.HttpError
+  """
+  response = {
+      'status': status,
+      'content-type': 'application/json',
+  }
+  content = {
+      'error': {
+          'code': status,
+          'errors': [{
+              'reason': str(reason),
+              'message': message,
+          }]
+      }
+  }
+  content_bytes = json.dumps(content).encode('UTF-8')
+  return gapi.googleapiclient.errors.HttpError(response, content_bytes)
+
+
+class GapiTest(unittest.TestCase):
+
+  def setUp(self):
+    SetGlobalVariables()
+    self.mock_service = MagicMock()
+    self.mock_method_name = 'mock_method'
+    self.mock_method = getattr(self.mock_service, self.mock_method_name)
+
+    self.simple_3_page_response = [
+        {
+            'items': [{
+                'position': 'page1,item1'
+            }, {
+                'position': 'page1,item2'
+            }, {
+                'position': 'page1,item3'
+            }],
+            'nextPageToken': 'page2'
+        },
+        {
+            'items': [{
+                'position': 'page2,item1'
+            }, {
+                'position': 'page2,item2'
+            }, {
+                'position': 'page2,item3'
+            }],
+            'nextPageToken': 'page3'
+        },
+        {
+            'items': [{
+                'position': 'page3,item1'
+            }, {
+                'position': 'page3,item2'
+            }, {
+                'position': 'page3,item3'
+            }],
+        },
+    ]
+    self.empty_items_response = {'items': []}
+
+    super(GapiTest, self).setUp()
+
+  def test_call_returns_basic_200_response(self):
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, self.mock_method().execute.return_value)
+
+  def test_call_passes_target_method_params(self):
+    gapi.call(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    self.assertEqual(self.mock_method.call_count, 1)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(method_kwargs.get('my_param_1'), 1)
+    self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+  @patch.object(gapi.errors, 'get_gapi_error_detail')
+  def test_call_retries_with_soft_errors(self, mock_error_detail):
+    mock_error_detail.return_value = (-1, 'aReason', 'some message')
+
+    # Make the request fail first, then return the proper response on the retry.
+    fake_http_error = create_http_error(403, 'aReason', 'unused message')
+    fake_200_response = MagicMock()
+    self.mock_method.return_value.execute.side_effect = [
+        fake_http_error, fake_200_response
+    ]
+
+    response = gapi.call(
+        self.mock_service, self.mock_method_name, soft_errors=True)
+    self.assertEqual(response, fake_200_response)
+    self.assertEqual(
+        self.mock_service._http.request.credentials.refresh.call_count, 1)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+
+  def test_call_throws_for_provided_reason(self):
+    throw_reason = errors.ErrorReason.USER_NOT_FOUND
+    fake_http_error = create_http_error(404, throw_reason, 'forced throw')
+    self.mock_method.return_value.execute.side_effect = fake_http_error
+
+    gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
+    with self.assertRaises(gam_exception):
+      gapi.call(
+          self.mock_service,
+          self.mock_method_name,
+          throw_reasons=[throw_reason])
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_request_for_default_retry_reasons(
+      self, mock_wait_on_failure):
+
+    # Test using one of the default retry reasons
+    default_throw_reason = errors.ErrorReason.BACKEND_ERROR
+    self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
+
+    fake_http_error = create_http_error(404, default_throw_reason, 'message')
+    fake_200_response = MagicMock()
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_http_error, fake_200_response
+    ]
+
+    response = gapi.call(
+        self.mock_service, self.mock_method_name, retry_reasons=[])
+    self.assertEqual(response, fake_200_response)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+    # Make sure a backoff technique was used for retry.
+    self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_requests_for_provided_retry_reasons(
+      self, unused_mock_wait_on_failure):
+
+    retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
+    fake_retrieable_error1 = create_http_error(400, retry_reason1,
+                                               'Forced Error 1')
+    retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
+    fake_retrieable_error2 = create_http_error(400, retry_reason2,
+                                               'Forced Error 2')
+    non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
+    fake_non_retriable_error = create_http_error(
+        400, non_retriable_reason,
+        'This error should not cause the request to be retried')
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_retrieable_error1, fake_retrieable_error2, fake_non_retriable_error
+    ]
+
+    with self.assertRaises(SystemExit):
+      # The third call should raise the SystemExit when non_retriable_error is
+      # raised.
+      gapi.call(
+          self.mock_service,
+          self.mock_method_name,
+          retry_reasons=[retry_reason1, retry_reason2])
+
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
+
+  def test_call_exits_on_oauth_token_error(self):
+    # An error with any OAUTH2_TOKEN_ERROR
+    fake_token_error = gapi.google.auth.exceptions.RefreshError(
+        errors.OAUTH2_TOKEN_ERRORS[0])
+    self.mock_method.return_value.execute.side_effect = fake_token_error
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_exits_on_nonretriable_error(self):
+    error_reason = 'unknownReason'
+    fake_http_error = create_http_error(500, error_reason,
+                                        'Testing unretriable errors')
+    self.mock_method.return_value.execute.side_effect = fake_http_error
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_exits_on_request_valueerror(self):
+    self.mock_method.return_value.execute.side_effect = ValueError()
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_clears_bad_http_cache_on_request_failure(self):
+    self.mock_service._http.cache = 'something that is not None'
+    fake_200_response = MagicMock()
+    self.mock_method.return_value.execute.side_effect = [
+        ValueError(), fake_200_response
+    ]
+
+    self.assertIsNotNone(self.mock_service._http.cache)
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, fake_200_response)
+    # Assert the cache was cleared
+    self.assertIsNone(self.mock_service._http.cache)
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_requests_with_backoff_on_servernotfounderror(
+      self, mock_wait_on_failure):
+    fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
+    fake_200_response = MagicMock()
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_servernotfounderror, fake_200_response
+    ]
+
+    http_connections = self.mock_service._http.connections
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, fake_200_response)
+    # HTTP cached connections should be cleared on receiving this error
+    self.assertNotEqual(http_connections, self.mock_service._http.connections)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+    # Make sure a backoff technique was used for retry.
+    self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+  def test_get_items_calls_correct_service_function(self):
+    gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertTrue(self.mock_method.called)
+
+  def test_get_items_returns_one_page(self):
+    fake_response = {'items': [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertEqual(page, fake_response['items'])
+
+  def test_get_items_non_default_page_field_name(self):
+    field_name = 'things'
+    fake_response = {field_name: [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_items(
+        self.mock_service, self.mock_method_name, items=field_name)
+    self.assertEqual(page, fake_response[field_name])
+
+  def test_get_items_passes_additional_kwargs_to_service(self):
+    gapi.get_items(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    self.assertEqual(self.mock_method.call_count, 1)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(1, method_kwargs.get('my_param_1'))
+    self.assertEqual(2, method_kwargs.get('my_param_2'))
+
+  def test_get_items_returns_empty_list_when_no_items_returned(self):
+    non_items_response = {'noItemsInThisResponse': {}}
+    self.mock_method.return_value.execute.return_value = non_items_response
+    page = gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertIsInstance(page, list)
+    self.assertEqual(0, len(page))
+
+  def test_get_all_pages_returns_all_items(self):
+    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
+    page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
+    page_3 = {'items': ['3-1', '3-2', '3-3']}
+    self.mock_method.return_value.execute.side_effect = [page_1, page_2, page_3]
+    response_items = gapi.get_all_pages(self.mock_service,
+                                        self.mock_method_name)
+    self.assertListEqual(response_items,
+                         page_1['items'] + page_2['items'] + page_3['items'])
+
+  def test_get_all_pages_includes_next_pagetoken_in_request(self):
+    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
+    page_2 = {'items': ['2-1', '2-2', '2-3']}
+    self.mock_method.return_value.execute.side_effect = [page_1, page_2]
+
+    gapi.get_all_pages(self.mock_service, self.mock_method_name, pageSize=100)
+    self.assertEqual(self.mock_method.call_count, 2)
+    call_2_kwargs = self.mock_method.call_args_list[1][1]
+    self.assertIn('pageToken', call_2_kwargs)
+    self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
+
+  def test_get_all_pages_uses_default_max_page_size(self):
+    sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
+    sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
+    self.mock_method.return_value.methodId = sample_api_id
+    self.mock_service._rootDesc = {
+        'resources': {
+            'someResource': {
+                'methods': {
+                    'someMethod': {
+                        'id': sample_api_id,
+                        'parameters': {
+                            'maxResults': {
+                                'maximum': sample_api_max_results
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+    gapi.get_all_pages(self.mock_service, self.mock_method_name)
+    request_method_kwargs = self.mock_method.call_args[1]
+    self.assertIn('maxResults', request_method_kwargs)
+    self.assertEqual(request_method_kwargs['maxResults'],
+                     gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
+
+  def test_get_all_pages_max_page_size_overrided(self):
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+    gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, pageSize=123456)
+    request_method_kwargs = self.mock_method.call_args[1]
+    self.assertIn('pageSize', request_method_kwargs)
+    self.assertEqual(123456, request_method_kwargs['pageSize'])
+
+  def test_get_all_pages_prints_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    self.assertIn(paging_message, messages_written)
+
+  def test_get_all_pages_prints_paging_message_inline(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+
+    # Make sure a return carriage was written between two pages
+    paging_message_call_positions = [
+        i for i, message in enumerate(messages_written)
+        if message == paging_message
+    ]
+    self.assertGreater(len(paging_message_call_positions), 1)
+    printed_between_page_messages = messages_written[
+        paging_message_call_positions[0]:paging_message_call_positions[1]]
+    self.assertIn('\r', printed_between_page_messages)
+
+  def test_get_all_pages_ends_paging_message_with_newline(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    last_page_message_index = len(
+        messages_written) - messages_written[::-1].index(paging_message)
+    last_carriage_return_index = len(
+        messages_written) - messages_written[::-1].index('\r\n')
+    self.assertGreater(last_carriage_return_index, last_page_message_index)
+
+  def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'Total number of items discovered: %%total_items%%'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_item_count = len(self.simple_3_page_response[0]['items'])
+    page_1_message = paging_message.replace('%%total_items%%',
+                                            str(page_1_item_count))
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_item_count = len(self.simple_3_page_response[1]['items'])
+    page_2_message = paging_message.replace(
+        '%%total_items%%', str(page_1_item_count + page_2_item_count))
+    self.assertIn(page_2_message, messages_written)
+
+    page_3_item_count = len(self.simple_3_page_response[2]['items'])
+    page_3_message = paging_message.replace(
+        '%%total_items%%',
+        str(page_1_item_count + page_2_item_count + page_3_item_count))
+    self.assertIn(page_3_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%total_items', message)
+
+  def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'First item in page: %%first_item%%'
+
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_message = paging_message.replace(
+        '%%first_item%%',
+        self.simple_3_page_response[0]['items'][0]['position'])
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_message = paging_message.replace(
+        '%%first_item%%',
+        self.simple_3_page_response[1]['items'][0]['position'])
+    self.assertIn(page_2_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%first_item', message)
+
+  def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'Last item in page: %%last_item%%'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_message = paging_message.replace(
+        '%%last_item%%',
+        self.simple_3_page_response[0]['items'][-1]['position'])
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_message = paging_message.replace(
+        '%%last_item%%',
+        self.simple_3_page_response[1]['items'][-1]['position'])
+    self.assertIn(page_2_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%last_item', message)
+
+  def test_get_all_pages_prints_all_attributes_in_paging_message(self):
+    pass
+
+  def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+    gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(method_kwargs.get('my_param_1'), 1)
+    self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+  @patch.object(gapi, 'call')
+  def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
+    throw_for = MagicMock()
+    retry_for = MagicMock()
+    mock_call.return_value = self.empty_items_response
+    gapi.get_all_pages(
+        self.mock_service,
+        self.mock_method_name,
+        throw_reasons=throw_for,
+        retry_reasons=retry_for)
+    method_kwargs = mock_call.call_args[1]
+    self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
+    self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
+
+  def test_get_all_pages_non_default_items_field_name(self):
+    field_name = 'things'
+    fake_response = {field_name: [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, items=field_name)
+    self.assertEqual(page, fake_response[field_name])
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gapi/calendar.py

@@ -0,0 +1,893 @@
+import csv
+import sys
+import uuid
+
+# TODO: get rid of these hacks
+import __main__
+from var import *
+
+import controlflow
+import display
+import fileutils
+import gapi
+import utils
+
+
+def normalizeCalendarId(calname, checkPrimary=False):
+    if checkPrimary and calname.lower() == 'primary':
+        return calname
+    if not GC_Values[GC_DOMAIN]:
+        GC_Values[GC_DOMAIN] = __main__._getValueFromOAuth('hd')
+    return __main__.convertUIDtoEmailAddress(calname,
+                                             email_types=['user', 'resource'])
+
+
+def buildCalendarGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+    return (calendarId, __main__.buildGAPIServiceObject('calendar',
+                                                        calendarId))
+
+
+def buildCalendarDataGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+
+    # Try to impersonate the calendar owner. If we fail, fall back to using
+    # admin for authentication. Resource calendars cannot be impersonated,
+    # so we need to access them as the admin.
+    cal = None
+    if not calname.endswith('.calendar.google.com'):
+        cal = __main__.buildGAPIServiceObject('calendar', calendarId, False)
+    if cal is None:
+        _, cal = buildCalendarGAPIObject(__main__._getValueFromOAuth('email'))
+    return (calendarId, cal)
+
+def printShowACLs(csvFormat):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    toDrive = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if csvFormat and myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            action = ['showacl', 'printacl'][csvFormat]
+            message = f"gam calendar <email> {action}"
+            controlflow.invalid_argument_exit(sys.argv[i], message)
+    acls = gapi.get_all_pages(
+        cal.acl(), 'list', 'items', calendarId=calendarId)
+    i = 0
+    if csvFormat:
+        titles = []
+        rows = []
+    else:
+        count = len(acls)
+    for rule in acls:
+        i += 1
+        if csvFormat:
+            row = utils.flatten_json(rule, None)
+            for key in row:
+                if key not in titles:
+                    titles.append(key)
+            rows.append(row)
+        else:
+            formatted_acl = formatACLRule(rule)
+            current_count = display.current_count(i, count)
+            print(f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
+    if csvFormat:
+        display.write_csv_file(
+            rows, titles, f'{calendarId} Calendar ACLs', toDrive)
+
+
+def _getCalendarACLScope(i, body):
+    body['scope'] = {}
+    myarg = sys.argv[i].lower()
+    body['scope']['type'] = myarg
+    i += 1
+    if myarg in ['user', 'group']:
+        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
+            sys.argv[i], noUid=True)
+        i += 1
+    elif myarg == 'domain':
+        if i < len(sys.argv) and \
+           sys.argv[i].lower().replace('_', '') != 'sendnotifications':
+            body['scope']['value'] = sys.argv[i].lower()
+            i += 1
+        else:
+            body['scope']['value'] = GC_Values[GC_DOMAIN]
+    elif myarg != 'default':
+        body['scope']['type'] = 'user'
+        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
+            myarg, noUid=True)
+    return i
+
+
+CALENDAR_ACL_ROLES_MAP = {
+    'editor': 'writer',
+    'freebusy': 'freeBusyReader',
+    'freebusyreader': 'freeBusyReader',
+    'owner': 'owner',
+    'read': 'reader',
+    'reader': 'reader',
+    'writer': 'writer',
+    'none': 'none',
+}
+
+
+def addACL(function):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    myarg = sys.argv[4].lower().replace('_', '')
+    if myarg not in CALENDAR_ACL_ROLES_MAP:
+        controlflow.expected_argument_exit(
+            "Role", ", ".join(CALENDAR_ACL_ROLES_MAP), myarg)
+    body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]}
+    i = _getCalendarACLScope(5, body)
+    sendNotifications = True
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'sendnotifications':
+            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam calendar <email> {function.lower()}")
+    print(f'Calendar: {calendarId}, {function} ACL: {formatACLRule(body)}')
+    gapi.call(cal.acl(), 'insert', calendarId=calendarId,
+              body=body, sendNotifications=sendNotifications)
+
+
+def delACL():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    if sys.argv[4].lower() == 'id':
+        ruleId = sys.argv[5]
+        print(f'Removing rights for {ruleId} to {calendarId}')
+        gapi.call(cal.acl(), 'delete', calendarId=calendarId, ruleId=ruleId)
+    else:
+        body = {'role': 'none'}
+        _getCalendarACLScope(5, body)
+        print(f'Calendar: {calendarId}, Delete ACL: {formatACLScope(body)}')
+        gapi.call(cal.acl(), 'insert', calendarId=calendarId,
+                  body=body, sendNotifications=False)
+
+
+def wipeData():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    gapi.call(cal.calendars(), 'clear', calendarId=calendarId)
+
+
+def printEvents():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    q = showDeleted = showHiddenInvitations = timeMin = \
+        timeMax = timeZone = updatedMin = None
+    toDrive = False
+    titles = []
+    csvRows = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            q = sys.argv[i+1]
+            i += 2
+        elif myarg == 'includedeleted':
+            showDeleted = True
+            i += 1
+        elif myarg == 'includehidden':
+            showHiddenInvitations = True
+            i += 1
+        elif myarg == 'after':
+            timeMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'before':
+            timeMax = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i+1]
+            i += 2
+        elif myarg == 'updated':
+            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam calendar <email> printevents")
+    page_message = gapi.got_total_items_msg(f'Events for {calendarId}', '')
+    results = gapi.get_all_pages(cal.events(), 'list', 'items',
+                                 page_message=page_message,
+                                 calendarId=calendarId, q=q,
+                                 showDeleted=showDeleted,
+                                 showHiddenInvitations=showHiddenInvitations,
+                                 timeMin=timeMin, timeMax=timeMax,
+                                 timeZone=timeZone,
+                                 updatedMin=updatedMin)
+    for result in results:
+        row = {'calendarId': calendarId}
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(result, flattened=row), csvRows, titles)
+    display.sort_csv_titles(['calendarId', 'id', 'summary', 'status'], titles)
+    display.write_csv_file(csvRows, titles, 'Calendar Events', toDrive)
+
+
+def formatACLScope(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]})'
+    return f'(Scope: {rule["scope"]["type"]})'
+
+
+def formatACLRule(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]}, ' \
+               f'Role: {rule["role"]})'
+    return f'(Scope: {rule["scope"]["type"]}, Role: {rule["role"]})'
+
+
+def getSendUpdates(myarg, i, cal):
+    if myarg == 'notifyattendees':
+        sendUpdates = 'all'
+        i += 1
+    elif myarg == 'sendnotifications':
+        sendUpdates = 'all' if __main__.getBoolean(sys.argv[i+1], myarg) else 'none'
+        i += 2
+    else:  # 'sendupdates':
+        sendUpdatesMap = {}
+        for val in cal._rootDesc['resources']['events']['methods']['delete'][
+                'parameters']['sendUpdates']['enum']:
+            sendUpdatesMap[val.lower()] = val
+        sendUpdates = sendUpdatesMap.get(sys.argv[i+1].lower(), False)
+        if not sendUpdates:
+            controlflow.expected_argument_exit(
+                "sendupdates", ", ".join(sendUpdatesMap), sys.argv[i+1])
+        i += 2
+    return (sendUpdates, i)
+
+
+def moveOrDeleteEvent(moveOrDelete):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    sendUpdates = None
+    doit = False
+    kwargs = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg in ['id', 'eventid']:
+            eventId = sys.argv[i+1]
+            i += 2
+        elif myarg in ['query', 'eventquery']:
+            controlflow.system_error_exit(
+                2, f'query is no longer supported for {moveOrDelete}event. ' \
+                   f'Use "gam calendar <email> printevents query <query> | ' \
+                   f'gam csv - gam {moveOrDelete}event id ~id" instead.')
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        elif moveOrDelete == 'move' and myarg == 'destination':
+            kwargs['destination'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam calendar <email> {moveOrDelete}event")
+    if doit:
+        print(f' going to {moveOrDelete} eventId {eventId}')
+        gapi.call(cal.events(), moveOrDelete, calendarId=calendarId,
+                  eventId=eventId, sendUpdates=sendUpdates, **kwargs)
+    else:
+        print(
+            f' would {moveOrDelete} eventId {eventId}. Add doit to command ' \
+            f'to actually {moveOrDelete} event')
+
+
+def infoEvent():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    eventId = sys.argv[4]
+    result = gapi.call(cal.events(), 'get',
+                       calendarId=calendarId, eventId=eventId)
+    display.print_json(result)
+
+
+def addOrUpdateEvent(action):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    # only way for non-Google calendars to get updates is via email
+    timeZone = None
+    kwargs = {}
+    body = {}
+    if action == 'add':
+        i = 4
+        func = 'insert'
+    else:
+        eventId = sys.argv[4]
+        kwargs = {'eventId': eventId}
+        i = 5
+        func = 'patch'
+        requires_full_update = ['attendee', 'optionalattendee',
+                                'removeattendee', 'replacedescription']
+        for arg in sys.argv[i:]:
+            if arg.replace('_', '').lower() in requires_full_update:
+                func = 'update'
+                body = gapi.call(cal.events(), 'get',
+                                 calendarId=calendarId, eventId=eventId)
+                break
+    sendUpdates, body = getEventAttributes(i, calendarId, cal, body, action)
+    result = gapi.call(cal.events(), func, conferenceDataVersion=1,
+                       supportsAttachments=True, calendarId=calendarId,
+                       sendUpdates=sendUpdates, body=body, fields='id',
+                       **kwargs)
+    print(f'Event {result["id"]} {action} finished')
+
+
+def _remove_attendee(attendees, remove_email):
+    return [attendee for attendee in attendees
+            if not attendee['email'].lower() == remove_email]
+
+
+def getEventAttributes(i, calendarId, cal, body, action):
+    # Default to external only so non-Google
+    # calendars are notified of changes
+    sendUpdates = 'externalOnly'
+    action = 'update' if body else 'add'
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg == 'attendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({'email': sys.argv[i+1]})
+            i += 2
+        elif myarg == 'removeattendee' and action == 'update':
+            remove_email = sys.argv[i+1].lower()
+            body['attendees'] = _remove_attendee(body['attendees'],
+                                                 remove_email)
+            i += 2
+        elif myarg == 'optionalattendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append(
+                {'email': sys.argv[i+1], 'optional': True})
+            i += 2
+        elif myarg == 'anyonecanaddself':
+            body['anyoneCanAddSelf'] = True
+            i += 1
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'replacedescription':
+            search = sys.argv[i+1]
+            replace = sys.argv[i+2]
+            body['description'] = re.sub(search, replace, body['description'])
+            i += 3
+        elif myarg == 'start':
+            if sys.argv[i+1].lower() == 'allday':
+                body['start'] = {'date': __main__.getYYYYMMDD(sys.argv[i+2])}
+                i += 3
+            else:
+                start_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                body['start'] = {'dateTime': start_time}
+                i += 2
+        elif myarg == 'end':
+            if sys.argv[i+1].lower() == 'allday':
+                body['end'] = {'date': __main__.getYYYYMMDD(sys.argv[i+2])}
+                i += 3
+            else:
+                end_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                body['end'] = {'dateTime': end_time}
+                i += 2
+        elif myarg == 'guestscantinviteothers':
+            body['guestsCanInviteOthers'] = False
+            i += 1
+        elif myarg == 'guestscaninviteothers':
+            body['guestsCanInviteTohters'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscaninviteothers')
+            i += 2
+        elif myarg == 'guestscantseeothers':
+            body['guestsCanSeeOtherGuests'] = False
+            i += 1
+        elif myarg == 'guestscanseeothers':
+            body['guestsCanSeeOtherGuests'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscanseeothers')
+            i += 2
+        elif myarg == 'guestscanmodify':
+            body['guestsCanModify'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscanmodify')
+            i += 2
+        elif myarg == 'id':
+            if action == 'update':
+                controlflow.invalid_argument_exit(
+                    'id', 'gam calendar <calendar> updateevent')
+            body['id'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'available':
+            body['transparency'] = 'transparent'
+            i += 1
+        elif myarg == 'transparency':
+            validTransparency = ['opaque', 'transparent']
+            if sys.argv[i+1].lower() in validTransparency:
+                body['transparency'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    'transparency',
+                    ", ".join(validTransparency), sys.argv[i+1])
+            i += 2
+        elif myarg == 'visibility':
+            validVisibility = ['default', 'public', 'private']
+            if sys.argv[i+1].lower() in validVisibility:
+                body['visibility'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    "visibility", ", ".join(validVisibility), sys.argv[i+1])
+            i += 2
+        elif myarg == 'tentative':
+            body['status'] = 'tentative'
+            i += 1
+        elif myarg == 'status':
+            validStatus = ['confirmed', 'tentative', 'cancelled']
+            if sys.argv[i+1].lower() in validStatus:
+                body['status'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    'visibility', ', '.join(validStatus), sys.argv[i+1])
+            i += 2
+        elif myarg == 'source':
+            body['source'] = {'title': sys.argv[i+1], 'url': sys.argv[i+2]}
+            i += 3
+        elif myarg == 'noreminders':
+            body['reminders'] = {'useDefault': False}
+            i += 1
+        elif myarg == 'reminder':
+            minutes = \
+            __main__.getInteger(sys.argv[i+1], myarg, minVal=0,
+                                maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+            reminder = {'minutes': minutes, 'method': sys.argv[i+2]}
+            body.setdefault(
+                'reminders', {'overrides': [], 'useDefault': False})
+            body['reminders']['overrides'].append(reminder)
+            i += 3
+        elif myarg == 'recurrence':
+            body.setdefault('recurrence', [])
+            body['recurrence'].append(sys.argv[i+1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i+1]
+            i += 2
+        elif myarg == 'privateproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['private'][sys.argv[i+1]] = sys.argv[i+2]
+            i += 3
+        elif myarg == 'sharedproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['shared'][sys.argv[i+1]] = sys.argv[i+2]
+            i += 3
+        elif myarg == 'colorindex':
+            body['colorId'] = __main__.getInteger(
+                sys.argv[i+1], myarg, CALENDAR_EVENT_MIN_COLOR_INDEX,
+                CALENDAR_EVENT_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'hangoutsmeet':
+            body['conferenceData'] = {'createRequest': {
+                'requestId': f'{str(uuid.uuid4())}'}}
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {action}event')
+    if ('recurrence' in body) and (('start' in body) or ('end' in body)):
+        if not timeZone:
+            timeZone = gapi.call(cal.calendars(), 'get',
+                                 calendarId=calendarId,
+                                 fields='timeZone')['timeZone']
+        if 'start' in body:
+            body['start']['timeZone'] = timeZone
+        if 'end' in body:
+            body['end']['timeZone'] = timeZone
+    return (sendUpdates, body)
+
+
+def modifySettings():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'timezone':
+            body['timeZone'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam calendar <email> modify")
+    gapi.call(cal.calendars(), 'patch', calendarId=calendarId, body=body)
+
+
+def changeAttendees(users):
+    do_it = True
+    i = 5
+    allevents = False
+    start_date = end_date = None
+    while len(sys.argv) > i:
+        myarg = sys.argv[i].lower()
+        if myarg == 'csv':
+            csv_file = sys.argv[i+1]
+            i += 2
+        elif myarg == 'dryrun':
+            do_it = False
+            i += 1
+        elif myarg == 'start':
+            start_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'end':
+            end_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'allevents':
+            allevents = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam <users> update calattendees")
+    attendee_map = {}
+    f = fileutils.open_file(csv_file)
+    csvFile = csv.reader(f)
+    for row in csvFile:
+        attendee_map[row[0].lower()] = row[1].lower()
+    fileutils.close_file(f)
+    for user in users:
+        sys.stdout.write(f'Checking user {user}\n')
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        page_token = None
+        while True:
+            events_page = gapi.call(cal.events(), 'list', calendarId=user,
+                                    pageToken=page_token, timeMin=start_date,
+                                    timeMax=end_date, showDeleted=False,
+                                    showHiddenInvitations=False)
+            print(f'Got {len(events_page.get("items", []))}')
+            for event in events_page.get('items', []):
+                if event['status'] == 'cancelled':
+                    # print u' skipping cancelled event'
+                    continue
+                try:
+                    event_summary = event['summary']
+                except (KeyError, UnicodeEncodeError, UnicodeDecodeError):
+                    event_summary = event['id']
+                try:
+                    organizer = event['organizer']['email'].lower()
+                    if not allevents and organizer != user:
+                        #print(f' skipping not-my-event {event_summary}')
+                        continue
+                except KeyError:
+                    pass  # no email for organizer
+                needs_update = False
+                try:
+                    for attendee in event['attendees']:
+                        try:
+                            if attendee['email'].lower() in attendee_map:
+                                old_email = attendee['email'].lower()
+                                new_email = attendee_map[attendee['email'].lower(
+                                )]
+                                print(f' SWITCHING attendee {old_email} to ' \
+                                    f'{new_email} for {event_summary}')
+                                event['attendees'].remove(attendee)
+                                event['attendees'].append({'email': new_email})
+                                needs_update = True
+                        except KeyError:  # no email for that attendee
+                            pass
+                except KeyError:
+                    continue  # no attendees
+                if needs_update:
+                    body = {}
+                    body['attendees'] = event['attendees']
+                    print(f'UPDATING {event_summary}')
+                    if do_it:
+                        gapi.call(cal.events(), 'patch', calendarId=user,
+                                  eventId=event['id'],
+                                  sendNotifications=False, body=body)
+                    else:
+                        print(' not pulling the trigger.')
+                # else:
+                #  print(f' no update needed for {event_summary}')
+            try:
+                page_token = events_page['nextPageToken']
+            except KeyError:
+                break
+
+
+def deleteCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    for user in users:
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        gapi.call(cal.calendarList(), 'delete',
+                  soft_errors=True, calendarId=calendarId)
+
+
+CALENDAR_REMINDER_MAX_MINUTES = 40320
+
+CALENDAR_MIN_COLOR_INDEX = 1
+CALENDAR_MAX_COLOR_INDEX = 24
+
+CALENDAR_EVENT_MIN_COLOR_INDEX = 1
+CALENDAR_EVENT_MAX_COLOR_INDEX = 11
+
+
+def getCalendarAttributes(i, body, function):
+    colorRgbFormat = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'selected':
+            body['selected'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg == 'hidden':
+            body['hidden'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg == 'summary':
+            body['summaryOverride'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'colorindex':
+            body['colorId'] = __main__.getInteger(
+                sys.argv[i+1], myarg, minVal=CALENDAR_MIN_COLOR_INDEX,
+                maxVal=CALENDAR_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'backgroundcolor':
+            body['backgroundColor'] = __main__.getColor(sys.argv[i+1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'foregroundcolor':
+            body['foregroundColor'] = __main__.getColor(sys.argv[i+1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'reminder':
+            body.setdefault('defaultReminders', [])
+            method = sys.argv[i+1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_REMINDER_METHODS:
+                    controlflow.expected_argument_exit("Method", ", ".join(
+                        CALENDAR_REMINDER_METHODS+CLEAR_NONE_ARGUMENT), method)
+                minutes = __main__.getInteger(
+                    sys.argv[i+2], myarg, minVal=0,
+                    maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+                body['defaultReminders'].append(
+                    {'method': method, 'minutes': minutes})
+                i += 3
+            else:
+                i += 2
+        elif myarg == 'notification':
+            body.setdefault('notificationSettings', {'notifications': []})
+            method = sys.argv[i+1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_NOTIFICATION_METHODS:
+                    controlflow.expected_argument_exit("Method", ", ".join(
+                        CALENDAR_NOTIFICATION_METHODS+CLEAR_NONE_ARGUMENT), method)
+                eventType = sys.argv[i+2].lower()
+                if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP:
+                    controlflow.expected_argument_exit("Event", ", ".join(
+                        CALENDAR_NOTIFICATION_TYPES_MAP), eventType)
+                notice = {'method': method,
+                          'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]}
+                body['notificationSettings']['notifications'].append(notice)
+                i += 3
+            else:
+                i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam {function} calendar")
+    return colorRgbFormat
+
+
+def addCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    body = {'id': calendarId, 'selected': True, 'hidden': False}
+    colorRgbFormat = getCalendarAttributes(6, body, 'add')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f'Subscribing {user} to calendar {calendarId}{current_count}')
+        gapi.call(cal.calendarList(), 'insert', soft_errors=True,
+                  body=body, colorRgbFormat=colorRgbFormat)
+
+
+def updateCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    body = {}
+    colorRgbFormat = getCalendarAttributes(6, body, 'update')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f"Updating {user}'s subscription to calendar ' \
+              f'{calendarId}{current_count}")
+        calId = calendarId if calendarId != 'primary' else user
+        gapi.call(cal.calendarList(), 'patch', soft_errors=True,
+                  calendarId=calId, body=body, colorRgbFormat=colorRgbFormat)
+
+
+def _showCalendar(userCalendar, j, jcount):
+    current_count = display.current_count(j, jcount)
+    summary = userCalendar.get("summaryOverride", userCalendar["summary"])
+    print(f'  Calendar: {userCalendar["id"]}{current_count}')
+    print(f'    Summary: {summary}')
+    print(f'    Description: {userCalendar.get("description", "")}')
+    print(f'    Access Level: {userCalendar["accessRole"]}')
+    print(f'    Timezone: {userCalendar["timeZone"]}')
+    print(f'    Location: {userCalendar.get("location", "")}')
+    print(f'    Hidden: {userCalendar.get("hidden", "False")}')
+    print(f'    Selected: {userCalendar.get("selected", "False")}')
+    print(f'    Color ID: {userCalendar["colorId"]}, ' \
+          f'Background Color: {userCalendar["backgroundColor"]}, ' \
+          f'Foreground Color: {userCalendar["foregroundColor"]}')
+    print(f'    Default Reminders:')
+    for reminder in userCalendar.get('defaultReminders', []):
+        print(f'      Method: {reminder["method"]}, ' \
+              f'Minutes: {reminder["minutes"]}')
+    print('    Notifications:')
+    if 'notificationSettings' in userCalendar:
+        notifications = userCalendar['notificationSettings'].get(
+            'notifications', [])
+        for notification in notifications:
+            print(f'      Method: {notification["method"]}, ' \
+                  f'Type: {notification["type"]}')
+
+
+def infoCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.call(cal.calendarList(), 'get',
+                           soft_errors=True,
+                           calendarId=calendarId)
+        if result:
+            print(f'User: {user}, Calendar:{display.current_count(i, count)}')
+            _showCalendar(result, 1, 1)
+
+
+def printShowCalendars(users, csvFormat):
+    if csvFormat:
+        todrive = False
+        titles = []
+        csvRows = []
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, f"gam <users> {['show', 'print'][csvFormat]} calendars")
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.get_all_pages(
+            cal.calendarList(), 'list', 'items', soft_errors=True)
+        jcount = len(result)
+        if not csvFormat:
+            print(f'User: {user}, Calendars:{display.current_count(i, count)}')
+            if jcount == 0:
+                continue
+            j = 0
+            for userCalendar in result:
+                j += 1
+                _showCalendar(userCalendar, j, jcount)
+        else:
+            if jcount == 0:
+                continue
+            for userCalendar in result:
+                row = {'primaryEmail': user}
+                display.add_row_titles_to_csv_file(utils.flatten_json(
+                    userCalendar, flattened=row), csvRows, titles)
+    if csvFormat:
+        display.sort_csv_titles(['primaryEmail', 'id'], titles)
+        display.write_csv_file(csvRows, titles, 'Calendars', todrive)
+
+
+def showCalSettings(users):
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        feed = gapi.get_all_pages(
+            cal.settings(), 'list', 'items', soft_errors=True)
+        if feed:
+            current_count = display.current_count(i, count)
+            print(f'User: {user}, Calendar Settings:{current_count}')
+            settings = {}
+            for setting in feed:
+                settings[setting['id']] = setting['value']
+            for attr, value in sorted(settings.items()):
+                print(f'  {attr}: {value}')
+
+
+def transferSecCals(users):
+    target_user = sys.argv[5]
+    remove_source_user = sendNotifications = True
+    i = 6
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'keepuser':
+            remove_source_user = False
+            i += 1
+        elif myarg == 'sendnotifications':
+            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam <users> transfer seccals")
+    if remove_source_user:
+        target_user, target_cal = buildCalendarGAPIObject(target_user)
+        if not target_cal:
+            return
+    for user in users:
+        user, source_cal = buildCalendarGAPIObject(user)
+        if not source_cal:
+            continue
+        calendars = gapi.get_all_pages(source_cal.calendarList(), 'list',
+                                       'items', soft_errors=True,
+                                       minAccessRole='owner', showHidden=True,
+                                       fields='items(id),nextPageToken')
+        for calendar in calendars:
+            calendarId = calendar['id']
+            if calendarId.find('@group.calendar.google.com') != -1:
+                body = {'role': 'owner',
+                        'scope': {'type': 'user', 'value': target_user}}
+                gapi.call(source_cal.acl(), 'insert', calendarId=calendarId,
+                          body=body, sendNotifications=sendNotifications)
+                if remove_source_user:
+                    body = {'role': 'none',
+                            'scope': {'type': 'user', 'value': user}}
+                    gapi.call(target_cal.acl(), 'insert',
+                              calendarId=calendarId, body=body,
+                              sendNotifications=sendNotifications)

src/gapi/directory/__init__.py

@@ -0,0 +1,5 @@
+import __main__
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('directory')

src/gapi/directory/cros.py

@@ -0,0 +1,795 @@
+import datetime
+
+from var import *
+import __main__
+import controlflow
+import display
+import fileutils
+import gapi
+import gapi.directory
+import utils
+
+
+def doUpdateCros():
+    cd = gapi.directory.buildGAPIObject()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    update_body = {}
+    action_body = {}
+    orgUnitPath = None
+    ack_wipe = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'user':
+            update_body['annotatedUser'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            update_body['annotatedLocation'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'notes':
+            update_body['notes'] = sys.argv[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg in ['tag', 'asset', 'assetid']:
+            update_body['annotatedAssetId'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['ou', 'org']:
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'action':
+            action = sys.argv[i+1].lower().replace('_', '').replace('-', '')
+            deprovisionReason = None
+            if action in ['deprovisionsamemodelreplace',
+                          'deprovisionsamemodelreplacement']:
+                action = 'deprovision'
+                deprovisionReason = 'same_model_replacement'
+            elif action in ['deprovisiondifferentmodelreplace',
+                            'deprovisiondifferentmodelreplacement']:
+                action = 'deprovision'
+                deprovisionReason = 'different_model_replacement'
+            elif action in ['deprovisionretiringdevice']:
+                action = 'deprovision'
+                deprovisionReason = 'retiring_device'
+            elif action not in ['disable', 'reenable']:
+                controlflow.system_error_exit(2, f'expected action of ' \
+                    f'deprovision_same_model_replace, ' \
+                    f'deprovision_different_model_replace, ' \
+                    f'deprovision_retiring_device, disable or reenable,'
+                    f' got {action}')
+            action_body = {'action': action}
+            if deprovisionReason:
+                action_body['deprovisionReason'] = deprovisionReason
+            i += 2
+        elif myarg == 'acknowledgedevicetouchrequirement':
+            ack_wipe = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam update cros")
+    i = 0
+    count = len(devices)
+    if action_body:
+        if action_body['action'] == 'deprovision' and not ack_wipe:
+            print(f'WARNING: Refusing to deprovision {count} devices because '
+                  'acknowledge_device_touch_requirement not specified. ' \
+                  'Deprovisioning a device means the device will have to ' \
+                  'be physically wiped and re-enrolled to be managed by ' \
+                  'your domain again. This requires physical access to ' \
+                  'the device and is very time consuming to perform for ' \
+                  'each device. Please add ' \
+                  '"acknowledge_device_touch_requirement" to the GAM ' \
+                  'command if you understand this and wish to proceed ' \
+                  'with the deprovision. Please also be aware that ' \
+                  'deprovisioning can have an effect on your device ' \
+                  'license count. See ' \
+                  'https://support.google.com/chrome/a/answer/3523633 '\
+                  'for full details.')
+            sys.exit(3)
+        for deviceId in devices:
+            i += 1
+            cur_count = __main__.currentCount(i, count)
+            print(f' performing action {action} for {deviceId}{cur_count}')
+            gapi.call(cd.chromeosdevices(), function='action',
+                      customerId=GC_Values[GC_CUSTOMER_ID],
+                      resourceId=deviceId, body=action_body)
+    else:
+        if update_body:
+            for deviceId in devices:
+                i += 1
+                current_count = __main__.currentCount(i, count)
+                print(f' updating {deviceId}{current_count}')
+                gapi.call(cd.chromeosdevices(), 'update',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          deviceId=deviceId, body=update_body)
+        if orgUnitPath:
+            # split moves into max 50 devices per batch
+            for l in range(0, len(devices), 50):
+                move_body = {'deviceIds': devices[l:l+50]}
+                print(f' moving {len(move_body["deviceIds"])} devices to ' \
+                      f'{orgUnitPath}')
+                gapi.call(cd.chromeosdevices(), 'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath, body=move_body)
+
+
+def doGetCrosInfo():
+    cd = gapi.directory.buildGAPIObject()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    downloadfile = None
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    projection = None
+    fieldsList = []
+    noLists = False
+    startDate = endDate = None
+    listLimit = 0
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nolists':
+            noLists = True
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=-1)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            fieldsList = []
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg])
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i+1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field])
+                    if field in CROS_ACTIVE_TIME_RANGES_ARGUMENTS + \
+                                CROS_DEVICE_FILES_ARGUMENTS + \
+                                CROS_RECENT_USERS_ARGUMENTS:
+                        projection = 'FULL'
+                        noLists = False
+                else:
+                    controlflow.invalid_argument_exit(
+                        field, "gam info cros fields")
+            i += 2
+        elif myarg == 'downloadfile':
+            downloadfile = sys.argv[i+1]
+            if downloadfile.lower() == 'latest':
+                downloadfile = downloadfile.lower()
+            i += 2
+        elif myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i+1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam info cros")
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = ','.join(set(fieldsList)).replace('.', '/')
+    else:
+        fields = None
+    i = 0
+    device_count = len(devices)
+    for deviceId in devices:
+        i += 1
+        cros = gapi.call(cd.chromeosdevices(), 'get',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         deviceId=deviceId, projection=projection,
+                         fields=fields)
+        print(f'CrOS Device: {deviceId} ({i} of {device_count})')
+        if 'notes' in cros:
+            cros['notes'] = cros['notes'].replace('\n', '\\n')
+        if 'autoUpdateExpiration' in cros:
+            cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                cros['autoUpdateExpiration'])
+        _checkTPMVulnerability(cros)
+        for up in CROS_SCALAR_PROPERTY_PRINT_ORDER:
+            if up in cros:
+                if isinstance(cros[up], str):
+                    print(f'  {up}: {cros[up]}')
+                else:
+                    sys.stdout.write(f'  {up}:')
+                    display.print_json(cros[up], '  ')
+        if not noLists:
+            activeTimeRanges = _filterTimeRanges(
+                cros.get('activeTimeRanges', []), startDate, endDate)
+            lenATR = len(activeTimeRanges)
+            if lenATR:
+                print('  activeTimeRanges')
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    active_date = activeTimeRange["date"]
+                    active_time = activeTimeRange["activeTime"]
+                    duration = utils.formatMilliSeconds(active_time)
+                    minutes = active_time // 60000
+                    print(f'    date: {active_date}')
+                    print(f'      activeTime: {active_time}')
+                    print(f'      duration: {duration}')
+                    print(f'      minutes: {minutes}')
+            recentUsers = cros.get('recentUsers', [])
+            lenRU = len(recentUsers)
+            if lenRU:
+                print('  recentUsers')
+                num_ranges = min(lenRU, listLimit or lenRU)
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get("email")
+                    if not useremail:
+                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    print(f'    type: {recentUser["type"]}')
+                    print(f'      email: {useremail}')
+            deviceFiles = _filterCreateReportTime(
+                cros.get('deviceFiles', []), 'createTime', startDate, endDate)
+            lenDF = len(deviceFiles)
+            if lenDF:
+                num_ranges = min(lenDF, listLimit or lenDF)
+                print('  deviceFiles')
+                for deviceFile in deviceFiles[:num_ranges]:
+                    device_type = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    print(f'    {device_type}: {create_time}')
+            if downloadfile:
+                deviceFiles = cros.get('deviceFiles', [])
+                lenDF = len(deviceFiles)
+                if lenDF:
+                    if downloadfile == 'latest':
+                        deviceFile = deviceFiles[-1]
+                    else:
+                        for deviceFile in deviceFiles:
+                            if deviceFile['createTime'] == downloadfile:
+                                break
+                        else:
+                            print(f'ERROR: file {downloadfile} not ' \
+                                  f'available to download.')
+                            deviceFile = None
+                    if deviceFile:
+                        created = deviceFile["createTime"]
+                        downloadfile = f'cros-logs-{deviceId}-{created}.zip'
+                        downloadfilename = os.path.join(targetFolder,
+                                                        downloadfile)
+                        dl_url = deviceFile['downloadUrl']
+                        _, content = cd._http.request(dl_url)
+                        fileutils.write_file(downloadfilename, content,
+                                             mode='wb',
+                                             continue_on_error=True)
+                        print(f'Downloaded: {downloadfilename}')
+                elif downloadfile:
+                    print('ERROR: no files to download.')
+            cpuStatusReports = _filterCreateReportTime(
+                cros.get('cpuStatusReports', []),
+                'reportTime',
+                startDate,
+                endDate)
+            lenCSR = len(cpuStatusReports)
+            if lenCSR:
+                print('  cpuStatusReports')
+                num_ranges = min(lenCSR, listLimit or lenCSR)
+                for cpuStatusReport in cpuStatusReports[:num_ranges]:
+                    print(f'    reportTime: {cpuStatusReport["reportTime"]}')
+                    print('      cpuTemperatureInfo')
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        temp_label = tempInfo['label'].strip()
+                        temperature = tempInfo['temperature']
+                        print(f'        {temp_label}: {temperature}')
+                    pct_info = cpuStatusReport["cpuUtilizationPercentageInfo"]
+                    util = ",".join([str(x) for x in pct_info])
+                    print(f'      cpuUtilizationPercentageInfo: {util}')
+            diskVolumeReports = cros.get('diskVolumeReports', [])
+            lenDVR = len(diskVolumeReports)
+            if lenDVR:
+                print('  diskVolumeReports')
+                print('    volumeInfo')
+                num_ranges = min(lenDVR, listLimit or lenDVR)
+                for diskVolumeReport in diskVolumeReports[:num_ranges]:
+                    volumeInfo = diskVolumeReport['volumeInfo']
+                    for volume in volumeInfo:
+                        vid = volume['volumeId']
+                        vstorage_free = volume['storageFree']
+                        vstorage_total = volume['storageTotal']
+                        print(f'      volumeId: {vid}')
+                        print(f'        storageFree: {vstorage_free}')
+                        print(f'        storageTotal: {vstorage_total}')
+            systemRamFreeReports = _filterCreateReportTime(
+                cros.get('systemRamFreeReports', []),
+                'reportTime', startDate, endDate)
+            lenSRFR = len(systemRamFreeReports)
+            if lenSRFR:
+                print('  systemRamFreeReports')
+                num_ranges = min(lenSRFR, listLimit or lenSRFR)
+                for systemRamFreeReport in systemRamFreeReports[:num_ranges]:
+                    report_time = systemRamFreeReport["reportTime"]
+                    free_info = systemRamFreeReport["systemRamFreeInfo"]
+                    free_ram = ",".join(free_info)
+                    print(f'    reportTime: {report_time}')
+                    print(f'      systemRamFreeInfo: {free_ram}')
+
+
+def doPrintCrosActivity():
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    titles = ['deviceId', 'annotatedAssetId',
+              'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    csvRows = []
+    fieldsList = ['deviceId', 'annotatedAssetId',
+                  'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    startDate = endDate = None
+    selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False
+    listLimit = 0
+    delimiter = ','
+    orgUnitPath = None
+    queries = [None]
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectActiveTimeRanges = True
+            i += 1
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectDeviceFiles = True
+            i += 1
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectRecentUsers = True
+            i += 1
+        elif myarg == 'both':
+            selectActiveTimeRanges = selectRecentUsers = True
+            i += 1
+        elif myarg == 'all':
+            selectActiveTimeRanges = selectDeviceFiles = True
+            selectRecentUsers = True
+            i += 1
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print crosactivity")
+    if not selectActiveTimeRanges and \
+       not selectDeviceFiles and \
+       not selectRecentUsers:
+        selectActiveTimeRanges = selectRecentUsers = True
+    if selectRecentUsers:
+        fieldsList.append('recentUsers')
+        display.add_titles_to_csv_file(['recentUsers.email', ], titles)
+    if selectActiveTimeRanges:
+        fieldsList.append('activeTimeRanges')
+        titles_to_add = ['activeTimeRanges.date',
+                         'activeTimeRanges.duration',
+                         'activeTimeRanges.minutes']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    if selectDeviceFiles:
+        fieldsList.append('deviceFiles')
+        titles_to_add = ['deviceFiles.type', 'deviceFiles.createTime']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    fields = f'nextPageToken,chromeosdevices({",".join(fieldsList)})'
+    for query in queries:
+        __main__.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection='FULL',
+                                      fields=fields, orgUnitPath=orgUnitPath)
+        for cros in all_cros:
+            row = {}
+            skip_attribs = ['recentUsers', 'activeTimeRanges', 'deviceFiles']
+            for attrib in cros:
+                if attrib not in skip_attribs:
+                    row[attrib] = cros[attrib]
+            if selectActiveTimeRanges:
+                activeTimeRanges = _filterTimeRanges(
+                    cros.get('activeTimeRanges', []), startDate, endDate)
+                lenATR = len(activeTimeRanges)
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['activeTimeRanges.date'] = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    newrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    newrow['activeTimeRanges.minutes'] = \
+                        activeTimeRange['activeTime']//60000
+                    csvRows.append(new_row)
+            if selectRecentUsers:
+                recentUsers = cros.get('recentUsers', [])
+                lenRU = len(recentUsers)
+                num_ranges = min(lenRU, listLimit or lenRU)
+                recent_users = []
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get("email")
+                    if not useremail:
+                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    recent_users.append(useremail)
+                row['recentUsers.email'] = delimiter.join(recent_users)
+                csvRows.append(row)
+            if selectDeviceFiles:
+                deviceFiles = _filterCreateReportTime(
+                    cros.get('deviceFiles', []),
+                    'createTime', startDate, endDate)
+                lenDF = len(deviceFiles)
+                num_ranges = min(lenDF, listLimit or lenDF)
+                for deviceFile in deviceFiles[:num_ranges]:
+                    new_row = row.copy()
+                    new_row['deviceFiles.type'] = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    new_row['deviceFiles.createTime'] = create_time
+                    csvRows.append(new_row)
+    display.write_csv_file(csvRows, titles, 'CrOS Activity', todrive)
+
+
+def _checkTPMVulnerability(cros):
+    if 'tpmVersionInfo' in cros and \
+       'firmwareVersion' in cros['tpmVersionInfo']:
+        firmware_version = cros['tpmVersionInfo']['firmwareVersion']
+        if firmware_version in CROS_TPM_VULN_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'VULNERABLE'
+        elif firmware_version in CROS_TPM_FIXED_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'UPDATED'
+        else:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'NOT IMPACTED'
+
+
+def doPrintCrosDevices():
+    def _getSelectedLists(myarg):
+        if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectedLists['activeTimeRanges'] = True
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectedLists['recentUsers'] = True
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectedLists['deviceFiles'] = True
+        elif myarg in CROS_CPU_STATUS_REPORTS_ARGUMENTS:
+            selectedLists['cpuStatusReports'] = True
+        elif myarg in CROS_DISK_VOLUME_REPORTS_ARGUMENTS:
+            selectedLists['diskVolumeReports'] = True
+        elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS:
+            selectedLists['systemRamFreeReports'] = True
+
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    display.add_field_to_csv_file(
+        'deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles)
+    projection = orderBy = sortOrder = orgUnitPath = None
+    queries = [None]
+    noLists = sortHeaders = False
+    selectedLists = {}
+    startDate = endDate = None
+    listLimit = 0
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'nolists':
+            noLists = True
+            selectedLists = {}
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i+1].lower().replace('_', '')
+            validOrderBy = ['location', 'user', 'lastsync',
+                            'notes', 'serialnumber', 'status', 'supportenddate']
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit(
+                    "orderby", ", ".join(validOrderBy), orderBy)
+            if orderBy == 'location':
+                orderBy = 'annotatedLocation'
+            elif orderBy == 'user':
+                orderBy = 'annotatedUser'
+            elif orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'serialnumber':
+                orderBy = 'serialNumber'
+            elif orderBy == 'supportenddate':
+                orderBy = 'supportEndDate'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            sortHeaders = True
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            sortHeaders = True
+            fieldsList = []
+            i += 1
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in CROS_LISTS_ARGUMENTS:
+            _getSelectedLists(myarg)
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_fields_list(
+                myarg, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i+1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_LISTS_ARGUMENTS:
+                    _getSelectedLists(field)
+                elif field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    display.add_field_to_fields_list(
+                        field, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+                else:
+                    controlflow.invalid_argument_exit(
+                        field, "gam print cros fields")
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam print cros")
+    if selectedLists:
+        noLists = False
+        projection = 'FULL'
+        for selectList in selectedLists:
+            display.add_field_to_fields_list(
+                selectList, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = f'nextPageToken,chromeosdevices({",".join(set(fieldsList))})'.replace(
+            '.', '/')
+    else:
+        fields = None
+    for query in queries:
+        __main__.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                      'chromeosdevices',
+                                      page_message=page_message, query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection=projection,
+                                      orgUnitPath=orgUnitPath,
+                                      orderBy=orderBy, sortOrder=sortOrder,
+                                      fields=fields)
+        for cros in all_cros:
+            _checkTPMVulnerability(cros)
+        if not noLists and not selectedLists:
+            for cros in all_cros:
+                if 'notes' in cros:
+                    cros['notes'] = cros['notes'].replace('\n', '\\n')
+                if 'autoUpdateExpiration' in cros:
+                    cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                        cros['autoUpdateExpiration'])
+                for cpuStatusReport in cros.get('cpuStatusReports', []):
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        tempInfo['label'] = tempInfo['label'].strip()
+                display.add_row_titles_to_csv_file(utils.flatten_json(
+                    cros, listLimit=listLimit), csvRows, titles)
+            continue
+        for cros in all_cros:
+            if 'notes' in cros:
+                cros['notes'] = cros['notes'].replace('\n', '\\n')
+            if 'autoUpdateExpiration' in cros:
+                cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                    cros['autoUpdateExpiration'])
+            row = {}
+            for attrib in cros:
+                if attrib not in set(['kind', 'etag', 'tpmVersionInfo',
+                                      'recentUsers', 'activeTimeRanges',
+                                      'deviceFiles', 'cpuStatusReports',
+                                      'diskVolumeReports',
+                                      'systemRamFreeReports']):
+                    row[attrib] = cros[attrib]
+            if selectedLists.get('activeTimeRanges'):
+                timergs = cros.get('activeTimeRanges', [])
+            else:
+                timergs = []
+            activeTimeRanges = _filterTimeRanges(timergs, startDate, endDate)
+            if selectedLists.get('recentUsers'):
+                recentUsers = cros.get('recentUsers', [])
+            else:
+                recentUsers = []
+            if selectedLists.get('deviceFiles'):
+                device_files = cros.get('deviceFiles', [])
+            else:
+                device_files = []
+            deviceFiles = _filterCreateReportTime(device_files, 'createTime',
+                                                  startDate, endDate)
+            if selectedLists.get('cpuStatusReports'):
+                cpu_reports = cros.get('cpuStatusReports', [])
+            else:
+                cpu_reports = []
+            cpuStatusReports = _filterCreateReportTime(cpu_reports,
+                                                       'reportTime',
+                                                       startDate, endDate)
+            if selectedLists.get('diskVolumeReports'):
+                diskVolumeReports = cros.get('diskVolumeReports', [])
+            else:
+                diskVolumeReports = []
+            if selectedLists.get('systemRamFreeReports'):
+                ram_reports = cros.get('systemRamFreeReports', [])
+            else:
+                ram_reports = []
+            systemRamFreeReports = _filterCreateReportTime(ram_reports,
+                                                           'reportTime',
+                                                           startDate,
+                                                           endDate)
+            if noLists or (not activeTimeRanges and \
+                           not recentUsers and \
+                           not deviceFiles and \
+                           not cpuStatusReports and \
+                           not diskVolumeReports and \
+                           not systemRamFreeReports):
+                display.add_row_titles_to_csv_file(row, csvRows, titles)
+                continue
+            lenATR = len(activeTimeRanges)
+            lenRU = len(recentUsers)
+            lenDF = len(deviceFiles)
+            lenCSR = len(cpuStatusReports)
+            lenDVR = len(diskVolumeReports)
+            lenSRFR = len(systemRamFreeReports)
+            max_len = max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR)
+            for i in range(min(max_len, listLimit or max_len)):
+                nrow = row.copy()
+                if i < lenATR:
+                    nrow['activeTimeRanges.date'] = \
+                        activeTimeRanges[i]['date']
+                    nrow['activeTimeRanges.activeTime'] = \
+                        str(activeTimeRanges[i]['activeTime'])
+                    active_time = activeTimeRanges[i]['activeTime']
+                    nrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    nrow['activeTimeRanges.minutes'] = active_time // 60000
+                if i < lenRU:
+                    nrow['recentUsers.type'] = recentUsers[i]['type']
+                    nrow['recentUsers.email'] = recentUsers[i].get('email')
+                    if not nrow['recentUsers.email']:
+                        if nrow['recentUsers.type'] == 'USER_TYPE_UNMANAGED':
+                            nrow['recentUsers.email'] = 'UnmanagedUser'
+                        else:
+                            nrow['recentUsers.email'] = 'Unknown'
+                if i < lenDF:
+                    nrow['deviceFiles.type'] = deviceFiles[i]['type']
+                    nrow['deviceFiles.createTime'] = \
+                        deviceFiles[i]['createTime']
+                if i < lenCSR:
+                    nrow['cpuStatusReports.reportTime'] = \
+                        cpuStatusReports[i]['reportTime']
+                    tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo',
+                                                        [])
+                    for tempInfo in tempInfos:
+                        temperature = tempInfo['temperature']
+                        label = tempInfo["label"].strip()
+                        base = 'cpuStatusReports.cpuTemperatureInfo.'
+                        nrow[f'{base}{label}'] = tempInfo['temperature']
+                    cpu_field = 'cpuUtilizationPercentageInfo'
+                    cpu_reports = cpuStatusReports[i][cpu_field]
+                    cpu_pcts = [str(x) for x in cpu_reports]
+                    nrow[f'cpuStatusReports.{cpu_field}'] = ','.join(cpu_pcts)
+                if i < lenDVR:
+                    volumeInfo = diskVolumeReports[i]['volumeInfo']
+                    j = 0
+                    vfield = 'diskVolumeReports.volumeInfo.'
+                    for volume in volumeInfo:
+                        nrow[f'{vfield}{j}.volumeId'] = \
+                            volume['volumeId']
+                        nrow[f'{vfield}{j}.storageFree'] = \
+                            volume['storageFree']
+                        nrow[f'{vfield}{j}.storageTotal'] = \
+                            volume['storageTotal']
+                        j += 1
+                if i < lenSRFR:
+                    nrow['systemRamFreeReports.reportTime'] = \
+                        systemRamFreeReports[i]['reportTime']
+                    ram_reports = systemRamFreeReports[i]['systemRamFreeInfo']
+                    ram_info = [str(x) for x in ram_reports]
+                    nrow['systenRamFreeReports.systemRamFreeInfo'] = \
+                        ','.join(ram_info)
+                display.add_row_titles_to_csv_file(nrow, csvRows, titles)
+    if sortHeaders:
+        display.sort_csv_titles(['deviceId', ], titles)
+    display.write_csv_file(csvRows, titles, 'CrOS', todrive)
+
+
+def getCrOSDeviceEntity(i, cd):
+    myarg = sys.argv[i].lower()
+    if myarg == 'cros_sn':
+        return i+2, __main__.getUsersToModify('cros_sn', sys.argv[i+1])
+    if myarg == 'query':
+        return i+2, __main__.getUsersToModify('crosquery', sys.argv[i+1])
+    if myarg[:6] == 'query:':
+        query = sys.argv[i][6:]
+        if query[:12].lower() == 'orgunitpath:':
+            kwargs = {'orgUnitPath': query[12:]}
+        else:
+            kwargs = {'query': query}
+        fields = 'nextPageToken,chromeosdevices(deviceId)'
+        devices = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                     'chromeosdevices',
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     fields=fields, **kwargs)
+        return i+1, [device['deviceId'] for device in devices]
+    return i+1, sys.argv[i].replace(',', ' ').split()
+
+
+def _getFilterDate(dateStr):
+    return datetime.datetime.strptime(dateStr, YYYYMMDD_FORMAT)
+
+
+def _filterTimeRanges(activeTimeRanges, startDate, endDate):
+    if startDate is None and endDate is None:
+        return activeTimeRanges
+    filteredTimeRanges = []
+    for timeRange in activeTimeRanges:
+        activityDate = datetime.datetime.strptime(
+            timeRange['date'], YYYYMMDD_FORMAT)
+        if ((startDate is None) or \
+            (activityDate >= startDate)) and \
+           ((endDate is None) or \
+            (activityDate <= endDate)):
+            filteredTimeRanges.append(timeRange)
+    return filteredTimeRanges
+
+
+def _filterCreateReportTime(items, timeField, startTime, endTime):
+    if startTime is None and endTime is None:
+        return items
+    filteredItems = []
+    time_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    for item in items:
+        timeValue = datetime.datetime.strptime(item[timeField], time_format)
+        if ((startTime is None) or \
+            (timeValue >= startTime)) and \
+           ((endTime is None) or \
+            (timeValue <= endTime)):
+            filteredItems.append(item)
+    return filteredItems

src/gapi/directory/customer.py

@@ -0,0 +1,113 @@
+import datetime
+
+from var import *
+import controlflow
+import gapi
+import gapi.directory
+import gapi.reports
+
+
+def doGetCustomerInfo():
+    cd = gapi.directory.buildGAPIObject()
+    customer_info = gapi.call(cd.customers(), 'get',
+                              customerKey=GC_Values[GC_CUSTOMER_ID])
+    print(f'Customer ID: {customer_info["id"]}')
+    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    result = gapi.call(cd.domains(), 'get', customer=customer_info['id'],
+                       domainName=customer_info['customerDomain'],
+                       fields='verified')
+    print(f'Primary Domain Verified: {result["verified"]}')
+    # If customer has changed primary domain customerCreationTime is date
+    # of current primary being added, not customer create date.
+    # We should also get all domains and use oldest date
+    customer_creation = customer_info['customerCreationTime']
+    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    oldest = datetime.datetime.strptime(customer_creation, date_format)
+    domains = gapi.get_items(cd.domains(), 'list', 'domains',
+                             customer=GC_Values[GC_CUSTOMER_ID],
+                             fields='domains(creationTime)')
+    for domain in domains:
+        creation_timestamp = int(domain['creationTime'])/1000
+        domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
+        if domain_creation < oldest:
+            oldest = domain_creation
+    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
+    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    print(f'Default Language: {customer_language}')
+    if 'postalAddress' in customer_info:
+        print('Address:')
+        for field in ADDRESS_FIELDS_PRINT_ORDER:
+            if field in customer_info['postalAddress']:
+                print(f' {field}: {customer_info["postalAddress"][field]}')
+    if 'phoneNumber' in customer_info:
+        print(f'Phone: {customer_info["phoneNumber"]}')
+    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    user_counts_map = {
+        'accounts:num_users': 'Total Users',
+        'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',
+        'accounts:gsuite_basic_used_licenses': 'G Suite Basic Users',
+        'accounts:gsuite_enterprise_total_licenses': 'G Suite Enterprise ' \
+        'Licenses',
+        'accounts:gsuite_enterprise_used_licenses': 'G Suite Enterprise ' \
+        'Users',
+        'accounts:gsuite_unlimited_total_licenses': 'G Suite Business ' \
+        'Licenses',
+        'accounts:gsuite_unlimited_used_licenses': 'G Suite Business Users'
+    }
+    parameters = ','.join(list(user_counts_map))
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    rep = gapi.reports.buildGAPIObject()
+    usage = None
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    while True:
+        try:
+            usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                                       'usageReports',
+                                       throw_reasons=throw_reasons,
+                                       customerId=customerId, date=tryDate,
+                                       parameters=parameters)
+            break
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = gapi.reports._adjust_date(str(e))
+    if not usage:
+        print('No user count data available.')
+        return
+    print(f'User counts as of {tryDate}:')
+    for item in usage[0]['parameters']:
+        api_name = user_counts_map.get(item['name'])
+        api_value = int(item.get('intValue', 0))
+        if api_name and api_value:
+            print(f'  {api_name}: {api_value:,}')
+
+
+def doUpdateCustomer():
+    cd = gapi.directory.buildGAPIObject()
+    body = {}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ADDRESS_FIELDS_ARGUMENT_MAP:
+            body.setdefault('postalAddress', {})
+            arg = ADDRESS_FIELDS_ARGUMENT_MAP[myarg]
+            body['postalAddress'][arg] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['adminsecondaryemail', 'alternateemail']:
+            body['alternateEmail'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['phone', 'phonenumber']:
+            body['phoneNumber'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'language':
+            body['language'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam update customer")
+    if not body:
+        controlflow.system_error_exit(2, 'no arguments specified for "gam '
+                                         'update customer"')
+    gapi.call(cd.customers(), 'patch', customerKey=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print('Updated customer')

src/gapi/directory/resource.py

@@ -0,0 +1,487 @@
+import sys
+import uuid
+
+import __main__
+from var import *
+import controlflow
+import display
+import gapi.directory
+import utils
+
+
+def printBuildings():
+    to_drive = False
+    cd = gapi.directory.buildGAPIObject()
+    titles = []
+    csvRows = []
+    fieldsList = ['buildingId']
+    # buildings.list() currently doesn't support paging
+    # but should soon, attempt to use it now so we
+    # won't break when it's turned on.
+    fields = 'nextPageToken,buildings(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Building']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        # Allows shorter arguments like "name" instead of "buildingname"
+        elif 'building'+myarg in possible_fields:
+            fieldsList.append(possible_fields['building'+myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print buildings")
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    for building in buildings:
+        building.pop('etags', None)
+        building.pop('etag', None)
+        building.pop('kind', None)
+        if 'buildingId' in building:
+            building['buildingId'] = f'id:{building["buildingId"]}'
+        if 'floorNames' in building:
+            building['floorNames'] = ','.join(building['floorNames'])
+        building = utils.flatten_json(building)
+        for item in building:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(building)
+    display.sort_csv_titles('buildingId', titles)
+    display.write_csv_file(csvRows, titles, 'Buildings', to_drive)
+
+
+def printResourceCalendars():
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    query = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg == 'allfields':
+            fieldsList = []
+            fieldsTitles = {}
+            titles = []
+            for field in RESCAL_ALLFIELDS:
+                display.add_field_to_csv_file(field,
+                                              RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                              fieldsList, fieldsTitles,
+                                              titles)
+            i += 1
+        elif myarg in RESCAL_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_csv_file(myarg,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print resources")
+    if not fieldsList:
+        for field in RESCAL_DFLTFIELDS:
+            display.add_field_to_csv_file(field,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+    fields = f'nextPageToken,items({",".join(set(fieldsList))})'
+    if 'buildingId' in fieldsList:
+        display.add_field_to_csv_file('buildingName', {'buildingName': [
+            'buildingName', ]}, fieldsList, fieldsTitles, titles)
+    __main__.printGettingAllItems('Resource Calendars', None)
+    page_message = gapi.got_total_items_first_last_msg('Resource Calendars')
+    resources = gapi.get_all_pages(cd.resources().calendars(), 'list',
+                                   'items', page_message=page_message,
+                                   message_attribute='resourceId',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   query=query, fields=fields)
+    for resource in resources:
+        if 'featureInstances' in resource:
+            features = [a_feature['feature']['name'] for \
+                a_feature in resource['featureInstances']]
+            resource['featureInstances'] = ','.join(features)
+        if 'buildingId' in resource:
+            resource['buildingName'] = getBuildingNameById(
+                cd, resource['buildingId'])
+            resource['buildingId'] = f'id:{resource["buildingId"]}'
+        resUnit = {}
+        for field in fieldsList:
+            resUnit[fieldsTitles[field]] = resource.get(field, '')
+        csvRows.append(resUnit)
+    display.sort_csv_titles(
+        ['resourceId', 'resourceName', 'resourceEmail'], titles)
+    display.write_csv_file(csvRows, titles, 'Resources', todrive)
+
+
+RESCAL_DFLTFIELDS = ['id', 'name', 'email',]
+RESCAL_ALLFIELDS = ['id', 'name', 'email', 'description', 'type',
+                    'buildingid', 'category', 'capacity', 'features', 'floor',
+                    'floorsection', 'generatedresourcename',
+                    'uservisibledescription',]
+
+RESCAL_ARGUMENT_TO_PROPERTY_MAP = {
+    'description': ['resourceDescription'],
+    'building': ['buildingId', ],
+    'buildingid': ['buildingId', ],
+    'capacity': ['capacity', ],
+    'category': ['resourceCategory', ],
+    'email': ['resourceEmail'],
+    'feature': ['featureInstances', ],
+    'features': ['featureInstances', ],
+    'floor': ['floorName', ],
+    'floorname': ['floorName', ],
+    'floorsection': ['floorSection', ],
+    'generatedresourcename': ['generatedResourceName', ],
+    'id': ['resourceId'],
+    'name': ['resourceName'],
+    'type': ['resourceType'],
+    'userdescription': ['userVisibleDescription', ],
+    'uservisibledescription': ['userVisibleDescription', ],
+}
+
+
+def printFeatures():
+    to_drive = False
+    cd = gapi.directory.buildGAPIObject()
+    titles = []
+    csvRows = []
+    fieldsList = ['name']
+    fields = 'nextPageToken,features(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Feature']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        elif 'feature'+myarg in possible_fields:
+            fieldsList.append(possible_fields['feature'+myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print features")
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    features = gapi.get_all_pages(cd.resources().features(), 'list',
+                                  'features',
+                                  customer=GC_Values[GC_CUSTOMER_ID],
+                                  fields=fields)
+    for feature in features:
+        feature.pop('etags', None)
+        feature.pop('etag', None)
+        feature.pop('kind', None)
+        feature = utils.flatten_json(feature)
+        for item in feature:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(feature)
+    display.sort_csv_titles('name', titles)
+    display.write_csv_file(csvRows, titles, 'Features', to_drive)
+
+
+def _getBuildingAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'id':
+            body['buildingId'] = args[i+1]
+            i += 2
+        elif myarg == 'name':
+            body['buildingName'] = args[i+1]
+            i += 2
+        elif myarg in ['lat', 'latitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['latitude'] = args[i+1]
+            i += 2
+        elif myarg in ['long', 'lng', 'longitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['longitude'] = args[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = args[i+1]
+            i += 2
+        elif myarg == 'floors':
+            body['floorNames'] = args[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, "gam create|update building")
+    return body
+
+
+def createBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    body = {'floorNames': ['1'],
+            'buildingId': str(uuid.uuid4()),
+            'buildingName': sys.argv[3]}
+    body = _getBuildingAttributes(sys.argv[4:], body)
+    print(f'Creating building {body["buildingId"]}...')
+    gapi.call(cd.resources().buildings(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def _makeBuildingIdNameMap(cd):
+    fields = 'nextPageToken,buildings(buildingId,buildingName)'
+    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {}
+    GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {}
+    for building in buildings:
+        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][building['buildingId']
+                                               ] = building['buildingName']
+        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building['buildingName']
+                                               ] = building['buildingId']
+
+
+def getBuildingByNameOrId(cd, which_building, minLen=1):
+    if not which_building or \
+       (minLen == 0 and which_building in ['id:', 'uid:']):
+        if minLen == 0:
+            return ''
+        controlflow.system_error_exit(3, 'Building id/name is empty')
+    cg = UID_PATTERN.match(which_building)
+    if cg:
+        return cg.group(1)
+    if GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] is None:
+        _makeBuildingIdNameMap(cd)
+    # Exact name match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID]:
+        return GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][which_building]
+    # No exact name match, check for case insensitive name matches
+    which_building_lower = which_building.lower()
+    ci_matches = []
+    for buildingName, buildingId in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID].items():
+        if buildingName.lower() == which_building_lower:
+            ci_matches.append(
+                {'buildingName': buildingName, 'buildingId': buildingId})
+    # One match, return ID
+    if len(ci_matches) == 1:
+        return ci_matches[0]['buildingId']
+    # No or multiple name matches, try ID
+    # Exact ID match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        return which_building
+    # No exact ID match, check for case insensitive id match
+    for buildingId in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        # Match, return ID
+        if buildingId.lower() == which_building_lower:
+            return buildingId
+    # Multiple name  matches
+    if len(ci_matches) > 1:
+        message = 'Multiple buildings with same name:\n'
+        for building in ci_matches:
+            message += f'  Name:{building["buildingName"]}  ' \
+                       f'id:{building["buildingId"]}\n'
+        message += '\nPlease specify building name by exact case or by id.'
+        controlflow.system_error_exit(3, message)
+    # No matches
+    else:
+        controlflow.system_error_exit(3, f'No such building {which_building}')
+
+
+def getBuildingNameById(cd, buildingId):
+    if GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] is None:
+        _makeBuildingIdNameMap(cd)
+    return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, 'UNKNOWN')
+
+
+def updateBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    body = _getBuildingAttributes(sys.argv[4:])
+    print(f'Updating building {buildingId}...')
+    gapi.call(cd.resources().buildings(), 'patch',
+              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId,
+              body=body)
+
+
+def getBuildingInfo():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    building = gapi.call(cd.resources().buildings(), 'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         buildingId=buildingId)
+    if 'buildingId' in building:
+        building['buildingId'] = f'id:{building["buildingId"]}'
+    if 'floorNames' in building:
+        building['floorNames'] = ','.join(building['floorNames'])
+    if 'buildingName' in building:
+        sys.stdout.write(building.pop('buildingName'))
+    display.print_json(building)
+
+
+def deleteBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    print(f'Deleting building {buildingId}...')
+    gapi.call(cd.resources().buildings(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId)
+
+
+def _getFeatureAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = args[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, "gam create|update feature")
+    return body
+
+
+def createFeature():
+    cd = gapi.directory.buildGAPIObject()
+    body = _getFeatureAttributes(sys.argv[3:])
+    print(f'Creating feature {body["name"]}...')
+    gapi.call(cd.resources().features(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def updateFeature():
+    # update does not work for name and name is only field to be updated
+    # if additional writable fields are added to feature in the future
+    # we'll add support for update as well as rename
+    cd = gapi.directory.buildGAPIObject()
+    oldName = sys.argv[3]
+    body = {'newName': sys.argv[5:]}
+    print(f'Updating feature {oldName}...')
+    gapi.call(cd.resources().features(), 'rename',
+              customer=GC_Values[GC_CUSTOMER_ID], oldName=oldName,
+              body=body)
+
+
+def deleteFeature():
+    cd = gapi.directory.buildGAPIObject()
+    featureKey = sys.argv[3]
+    print(f'Deleting feature {featureKey}...')
+    gapi.call(cd.resources().features(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], featureKey=featureKey)
+
+
+def _getResourceCalendarAttributes(cd, args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['resourceName'] = args[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['resourceDescription'] = args[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'type':
+            body['resourceType'] = args[i+1]
+            i += 2
+        elif myarg in ['building', 'buildingid']:
+            body['buildingId'] = getBuildingByNameOrId(
+                cd, args[i+1], minLen=0)
+            i += 2
+        elif myarg in ['capacity']:
+            body['capacity'] = __main__.getInteger(args[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg in ['feature', 'features']:
+            features = args[i+1].split(',')
+            body['featureInstances'] = []
+            for feature in features:
+                instance = {'feature': {'name': feature}}
+                body['featureInstances'].append(instance)
+            i += 2
+        elif myarg in ['floor', 'floorname']:
+            body['floorName'] = args[i+1]
+            i += 2
+        elif myarg in ['floorsection']:
+            body['floorSection'] = args[i+1]
+            i += 2
+        elif myarg in ['category']:
+            body['resourceCategory'] = args[i+1].upper()
+            if body['resourceCategory'] == 'ROOM':
+                body['resourceCategory'] = 'CONFERENCE_ROOM'
+            i += 2
+        elif myarg in ['uservisibledescription', 'userdescription']:
+            body['userVisibleDescription'] = args[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                args[i], "gam create|update resource")
+    return body
+
+
+def createResourceCalendar():
+    cd = gapi.directory.buildGAPIObject()
+    body = {'resourceId': sys.argv[3],
+            'resourceName': sys.argv[4]}
+    body = _getResourceCalendarAttributes(cd, sys.argv[5:], body)
+    print(f'Creating resource {body["resourceId"]}...')
+    gapi.call(cd.resources().calendars(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def updateResourceCalendar():
+    cd = gapi.directory.buildGAPIObject()
+    resId = sys.argv[3]
+    body = _getResourceCalendarAttributes(cd, sys.argv[4:])
+    # Use patch since it seems to work better.
+    # update requires name to be set.
+    gapi.call(cd.resources().calendars(), 'patch',
+              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId,
+              body=body, fields='')
+    print(f'updated resource {resId}')
+
+
+def getResourceCalendarInfo():
+    cd = gapi.directory.buildGAPIObject()
+    resId = sys.argv[3]
+    resource = gapi.call(cd.resources().calendars(), 'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         calendarResourceId=resId)
+    if 'featureInstances' in resource:
+        features = []
+        for a_feature in resource.pop('featureInstances'):
+            features.append(a_feature['feature']['name'])
+        resource['features'] = ', '.join(features)
+    if 'buildingId' in resource:
+        resource['buildingName'] = getBuildingNameById(
+            cd, resource['buildingId'])
+        resource['buildingId'] = f'id:{resource["buildingId"]}'
+    display.print_json(resource)
+
+
+def deleteResourceCalendar():
+    resId = sys.argv[3]
+    cd = gapi.directory.buildGAPIObject()
+    print(f'Deleting resource calendar {resId}')
+    gapi.call(cd.resources().calendars(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId)

src/gapi/errors.py

@@ -0,0 +1,362 @@
+"""GAPI and OAuth Token related errors methods."""
+
+from enum import Enum
+import json
+
+import controlflow
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import UTF8
+
+
+class GapiAbortedError(Exception):
+  pass
+
+
+class GapiAuthErrorError(Exception):
+  pass
+
+
+class GapiBadGatewayError(Exception):
+  pass
+
+
+class GapiBadRequestError(Exception):
+  pass
+
+
+class GapiConditionNotMetError(Exception):
+  pass
+
+
+class GapiCyclicMembershipsNotAllowedError(Exception):
+  pass
+
+
+class GapiDomainCannotUseApisError(Exception):
+  pass
+
+
+class GapiDomainNotFoundError(Exception):
+  pass
+
+
+class GapiDuplicateError(Exception):
+  pass
+
+
+class GapiFailedPreconditionError(Exception):
+  pass
+
+
+class GapiForbiddenError(Exception):
+  pass
+
+
+class GapiGatewayTimeoutError(Exception):
+  pass
+
+
+class GapiGroupNotFoundError(Exception):
+  pass
+
+
+class GapiInvalidError(Exception):
+  pass
+
+
+class GapiInvalidArgumentError(Exception):
+  pass
+
+
+class GapiInvalidMemberError(Exception):
+  pass
+
+
+class GapiMemberNotFoundError(Exception):
+  pass
+
+
+class GapiNotFoundError(Exception):
+  pass
+
+
+class GapiNotImplementedError(Exception):
+  pass
+
+
+class GapiPermissionDeniedError(Exception):
+  pass
+
+
+class GapiResourceNotFoundError(Exception):
+  pass
+
+
+class GapiServiceNotAvailableError(Exception):
+  pass
+
+
+class GapiUserNotFoundError(Exception):
+  pass
+
+
+# GAPI Error Reasons
+class ErrorReason(Enum):
+  """The reason why a non-200 HTTP response was returned from a GAPI."""
+  ABORTED = 'aborted'
+  AUTH_ERROR = 'authError'
+  BACKEND_ERROR = 'backendError'
+  BAD_GATEWAY = 'badGateway'
+  BAD_REQUEST = 'badRequest'
+  CONDITION_NOT_MET = 'conditionNotMet'
+  CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
+  DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
+  DOMAIN_NOT_FOUND = 'domainNotFound'
+  DUPLICATE = 'duplicate'
+  FAILED_PRECONDITION = 'failedPrecondition'
+  FORBIDDEN = 'forbidden'
+  FOUR_O_THREE = '403'
+  GATEWAY_TIMEOUT = 'gatewayTimeout'
+  GROUP_NOT_FOUND = 'groupNotFound'
+  INTERNAL_ERROR = 'internalError'
+  INVALID = 'invalid'
+  INVALID_ARGUMENT = 'invalidArgument'
+  INVALID_MEMBER = 'invalidMember'
+  MEMBER_NOT_FOUND = 'memberNotFound'
+  NOT_FOUND = 'notFound'
+  NOT_IMPLEMENTED = 'notImplemented'
+  PERMISSION_DENIED = 'permissionDenied'
+  QUOTA_EXCEEDED = 'quotaExceeded'
+  RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
+  RESOURCE_NOT_FOUND = 'resourceNotFound'
+  SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+  SERVICE_LIMIT = 'serviceLimit'
+  SYSTEM_ERROR = 'systemError'
+  USER_NOT_FOUND = 'userNotFound'
+  USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+  FOUR_TWO_NINE = '429'
+  DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
+
+  def __str__(self):
+    return str(self.value)
+
+
+# Common sets of GAPI error reasons
+DEFAULT_RETRY_REASONS = [
+  ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
+  ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
+  ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT,
+  ErrorReason.INTERNAL_ERROR, ErrorReason.FOUR_TWO_NINE,
+  ]
+GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
+GROUP_GET_THROW_REASONS = [
+  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
+  ErrorReason.BAD_REQUEST
+  ]
+GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
+MEMBERS_THROW_REASONS = [
+  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
+  ErrorReason.FORBIDDEN
+  ]
+MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
+
+# A map of GAPI error reasons to the corresponding GAM Python Exception
+ERROR_REASON_TO_EXCEPTION = {
+  ErrorReason.ABORTED:
+    GapiAbortedError,
+  ErrorReason.AUTH_ERROR:
+    GapiAuthErrorError,
+  ErrorReason.BAD_GATEWAY:
+    GapiBadGatewayError,
+  ErrorReason.BAD_REQUEST:
+    GapiBadRequestError,
+  ErrorReason.CONDITION_NOT_MET:
+    GapiConditionNotMetError,
+  ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
+    GapiCyclicMembershipsNotAllowedError,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS:
+    GapiDomainCannotUseApisError,
+  ErrorReason.DOMAIN_NOT_FOUND:
+    GapiDomainNotFoundError,
+  ErrorReason.DUPLICATE:
+    GapiDuplicateError,
+  ErrorReason.FAILED_PRECONDITION:
+    GapiFailedPreconditionError,
+  ErrorReason.FORBIDDEN:
+    GapiForbiddenError,
+  ErrorReason.GATEWAY_TIMEOUT:
+    GapiGatewayTimeoutError,
+  ErrorReason.GROUP_NOT_FOUND:
+    GapiGroupNotFoundError,
+  ErrorReason.INVALID:
+    GapiInvalidError,
+  ErrorReason.INVALID_ARGUMENT:
+    GapiInvalidArgumentError,
+  ErrorReason.INVALID_MEMBER:
+    GapiInvalidMemberError,
+  ErrorReason.MEMBER_NOT_FOUND:
+    GapiMemberNotFoundError,
+  ErrorReason.NOT_FOUND:
+    GapiNotFoundError,
+  ErrorReason.NOT_IMPLEMENTED:
+    GapiNotImplementedError,
+  ErrorReason.PERMISSION_DENIED:
+    GapiPermissionDeniedError,
+  ErrorReason.RESOURCE_NOT_FOUND:
+    GapiResourceNotFoundError,
+  ErrorReason.SERVICE_NOT_AVAILABLE:
+    GapiServiceNotAvailableError,
+  ErrorReason.USER_NOT_FOUND:
+    GapiUserNotFoundError,
+  }
+
+# OAuth Token Errors
+OAUTH2_TOKEN_ERRORS = [
+  'access_denied',
+  'access_denied: Requested client not authorized',
+  'internal_failure: Backend Error',
+  'internal_failure: None',
+  'invalid_grant',
+  'invalid_grant: Bad Request',
+  'invalid_grant: Invalid email or User ID',
+  'invalid_grant: Not a valid email',
+  'invalid_grant: Invalid JWT: No valid verifier found for issuer',
+  'invalid_request: Invalid impersonation prn email address',
+  'unauthorized_client: Client is unauthorized to retrieve access tokens '
+  'using this method',
+  'unauthorized_client: Client is unauthorized to retrieve access tokens '
+  'using this method, or client not authorized for any of the scopes '
+  'requested',
+  'unauthorized_client: Unauthorized client or scope in request',
+  ]
+
+
+def _create_http_error_dict(status_code, reason, message):
+  """Creates a basic error dict similar to most Google API Errors.
+
+  Args:
+    status_code: Int, the error's HTTP response status code.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    dict
+  """
+  return {
+    'error': {
+      'code': status_code,
+      'errors': [{
+        'reason': str(reason),
+        'message': message,
+        }]
+      }
+  }
+
+
+def get_gapi_error_detail(e,
+                          soft_errors=False,
+                          silent_errors=False,
+                          retry_on_http_error=False):
+  """Extracts error detail from a non-200 GAPI Response.
+
+  Args:
+    e: googleapiclient.HttpError, The HTTP Error received.
+    soft_errors: Boolean, If true, causes error messages to be surpressed,
+      rather than sending them to stderr.
+    silent_errors: Boolean, If true, suppresses and ignores any errors from
+      being displayed
+    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
+      code, indicating that the request can be retried. TODO: Remove this param,
+        as it seems to be outside the scope of this method.
+
+  Returns:
+    A tuple containing the HTTP Response code, GAPI error reason, and error
+        message.
+  """
+  try:
+    error = json.loads(e.content.decode(UTF8))
+  except ValueError:
+    error_content = e.content.decode(UTF8) if isinstance(e.content,
+                                                         bytes) else e.content
+    if (e.resp['status'] == '503') and (
+        error_content == 'Quota exceeded for the current request'):
+      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
+    if (e.resp['status'] == '403') and (
+        error_content.startswith('Request rate higher than configured')):
+      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
+    if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
+      return (e.resp['status'], ErrorReason.BAD_GATEWAY.value, error_content)
+    if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
+      return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value, error_content)
+    if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
+      error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
+                                      'Domain not found')
+    elif (e.resp['status'] == '400') and (
+        'InvalidSsoSigningKey' in error_content):
+      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                      'InvalidSsoSigningKey')
+    elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
+      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                      'UnknownError')
+    elif retry_on_http_error:
+      return (-1, None, None)
+    elif soft_errors:
+      if not silent_errors:
+        display.print_error(error_content)
+      return (0, None, None)
+    else:
+      controlflow.system_error_exit(5, error_content)
+    # END: ValueError catch
+
+  if 'error' in error:
+    http_status = error['error']['code']
+    try:
+      message = error['error']['errors'][0]['message']
+    except KeyError:
+      message = error['error']['message']
+  else:
+    if 'error_description' in error:
+      if error['error_description'] == 'Invalid Value':
+        message = error['error_description']
+        http_status = 400
+        error = _create_http_error_dict(400, ErrorReason.INVALID.value, message)
+      else:
+        controlflow.system_error_exit(4, str(error))
+    else:
+      controlflow.system_error_exit(4, str(error))
+
+  # Extract the error reason
+  try:
+    reason = error['error']['errors'][0]['reason']
+    if reason == 'notFound':
+      if 'userKey' in message:
+        reason = ErrorReason.USER_NOT_FOUND.value
+      elif 'groupKey' in message:
+        reason = ErrorReason.GROUP_NOT_FOUND.value
+      elif 'memberKey' in message:
+        reason = ErrorReason.MEMBER_NOT_FOUND.value
+      elif 'Domain not found' in message:
+        reason = ErrorReason.DOMAIN_NOT_FOUND.value
+      elif 'Resource Not Found' in message:
+        reason = ErrorReason.RESOURCE_NOT_FOUND.value
+    elif reason == 'invalid':
+      if 'userId' in message:
+        reason = ErrorReason.USER_NOT_FOUND.value
+      elif 'memberKey' in message:
+        reason = ErrorReason.INVALID_MEMBER.value
+    elif reason == 'failedPrecondition':
+      if 'Bad Request' in message:
+        reason = ErrorReason.BAD_REQUEST.value
+      elif 'Mail service not enabled' in message:
+        reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
+    elif reason == 'required':
+      if 'memberKey' in message:
+        reason = ErrorReason.MEMBER_NOT_FOUND.value
+    elif reason == 'conditionNotMet':
+      if 'Cyclic memberships not allowed' in message:
+        reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
+  except KeyError:
+    reason = f'{http_status}'
+  return (http_status, reason, message)

src/gapi/errors_test.py

@@ -0,0 +1,209 @@
+"""Python unit tests for gapi.errors"""
+
+import json
+
+import unittest
+from unittest.mock import patch
+
+import googleapiclient.errors
+from gapi import errors
+
+
+def create_simple_http_error(status, reason, message):
+  content = errors._create_http_error_dict(status, reason, message)
+  return create_http_error(status, content)
+
+
+def create_http_error(status, content):
+  response = {
+      'status': status,
+      'content-type': 'application/json',
+  }
+  content_as_bytes = json.dumps(content).encode('UTF-8')
+  return googleapiclient.errors.HttpError(response, content_as_bytes)
+
+
+class ErrorsTest(unittest.TestCase):
+
+  def test_get_gapi_error_detail_quota_exceeded(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_invalid_domain(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_invalid_signing_key(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_unknown_error(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_retry_http_error(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_prints_soft_errors(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_exits_on_unrecoverable_errors(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_quota_exceeded_for_current_request(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_quota_exceeded_high_request_rate(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_extracts_user_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: userKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: userKey.')
+
+  def test_get_gapi_error_extracts_group_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: groupKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: groupKey.')
+
+  def test_get_gapi_error_extracts_member_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: memberKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: memberKey.')
+
+  def test_get_gapi_error_extracts_domain_not_found(self):
+    err = create_simple_http_error(404, 'notFound', 'Domain not found.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
+    self.assertEqual(message, 'Domain not found.')
+
+  def test_get_gapi_error_extracts_generic_resource_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: unknownResource.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: unknownResource.')
+
+  def test_get_gapi_error_extracts_invalid_userid(self):
+    err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+    self.assertEqual(message, 'Invalid Input: userId')
+
+  def test_get_gapi_error_extracts_invalid_member(self):
+    err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
+    self.assertEqual(message, 'Invalid Input: memberKey')
+
+  def test_get_gapi_error_extracts_bad_request(self):
+    err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
+    self.assertEqual(message, 'Bad Request')
+
+  def test_get_gapi_error_extracts_service_not_available(self):
+    err = create_simple_http_error(400, 'failedPrecondition',
+                                   'Mail service not enabled')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
+    self.assertEqual(message, 'Mail service not enabled')
+
+  def test_get_gapi_error_extracts_required_member_not_found(self):
+    err = create_simple_http_error(400, 'required',
+                                   'Missing required field: memberKey')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+    self.assertEqual(message, 'Missing required field: memberKey')
+
+  def test_get_gapi_error_extracts_cyclic_memberships_error(self):
+    err = create_simple_http_error(400, 'conditionNotMet',
+                                   'Cyclic memberships not allowed')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason,
+                     errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
+    self.assertEqual(message, 'Cyclic memberships not allowed')
+
+  def test_get_gapi_error_extracts_single_error_with_message(self):
+    status_code = 999
+    response = {'status': status_code}
+    # This error does not have an "errors" key describing each error.
+    content = {'error': {'code': status_code, 'message': 'unknown error'}}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, status_code)
+    self.assertEqual(reason, str(status_code))
+    self.assertEqual(message, content['error']['message'])
+
+  def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
+      self):
+    status_code = 999
+    response = {'status': status_code}
+    # This error only has an error_description_field and an unknown description.
+    content = {'error_description': 'something errored'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    with self.assertRaises(SystemExit) as context:
+      errors.get_gapi_error_detail(err)
+    self.assertEqual(4, context.exception.code)
+
+  def test_get_gapi_error_exits_on_invalid_error_description(self):
+    status_code = 400
+    response = {'status': status_code}
+    content = {'error_description': 'Invalid Value'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, status_code)
+    self.assertEqual(reason, errors.ErrorReason.INVALID.value)
+    self.assertEqual(message, 'Invalid Value')
+
+  def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
+    status_code = 900
+    response = {'status': status_code}
+    content = {'notErrorContentThatIsExpected': 'foo'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    with self.assertRaises(SystemExit) as context:
+      errors.get_gapi_error_detail(err)
+    self.assertEqual(4, context.exception.code)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gapi/reports.py

@@ -0,0 +1,325 @@
+import datetime
+import sys
+
+import __main__
+from var import *
+import controlflow
+import display
+import gapi
+import utils
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('reports')
+
+
+REPORT_CHOICE_MAP = {
+    'access': 'access_transparency',
+    'accesstransparency': 'access_transparency',
+    'calendars': 'calendar',
+    'customers': 'customer',
+    'doc': 'drive',
+    'docs': 'drive',
+    'domain': 'customer',
+    'enterprisegroups': 'groups_enterprise',
+    'google+': 'gplus',
+    'group': 'groups',
+    'groupsenterprise': 'groups_enterprise',
+    'hangoutsmeet': 'meet',
+    'logins': 'login',
+    'oauthtoken': 'token',
+    'tokens': 'token',
+    'users': 'user',
+    'useraccounts': 'user_accounts',
+}
+
+
+def showReport():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    report = sys.argv[2].lower()
+    report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    valid_apps = gapi.get_enum_values_minus_unspecified(
+        rep._rootDesc['resources']['activities']['methods']['list'][
+            'parameters']['applicationName']['enum'])+['customer', 'user']
+    if report not in valid_apps:
+        controlflow.expected_argument_exit(
+            "report", ", ".join(sorted(valid_apps)), report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    filters = parameters = actorIpAddress = startTime = endTime = eventName = orgUnitId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    to_drive = False
+    userKey = 'all'
+    fullDataRequired = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'date':
+            tryDate = utils.get_yyyymmdd(sys.argv[i+1])
+            i += 2
+        elif myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            i += 2
+        elif myarg == 'fulldatarequired':
+            fullDataRequired = []
+            fdr = sys.argv[i+1].lower()
+            if fdr and fdr != 'all':
+                fullDataRequired = fdr.replace(',', ' ').split()
+            i += 2
+        elif myarg == 'start':
+            startTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'end':
+            endTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'event':
+            eventName = sys.argv[i+1]
+            i += 2
+        elif myarg == 'user':
+            userKey = __main__.normalizeEmailAddressOrUID(sys.argv[i+1])
+            i += 2
+        elif myarg in ['filter', 'filters']:
+            filters = sys.argv[i+1]
+            i += 2
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i+1]
+            i += 2
+        elif myarg == 'ip':
+            actorIpAddress = sys.argv[i+1]
+            i += 2
+        elif myarg == 'todrive':
+            to_drive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam report")
+    if report == 'user':
+        while True:
+            try:
+                if fullDataRequired is not None:
+                    warnings = gapi.get_items(rep.userUsageReport(), 'get',
+                                              'warnings',
+                                              throw_reasons=throw_reasons,
+                                              date=tryDate, userKey=userKey,
+                                              customerId=customerId,
+                                              orgUnitID=orgUnitId,
+                                              fields='warnings')
+                    fullData, tryDate = _check_full_data_available(
+                        warnings, tryDate, fullDataRequired)
+                    if fullData < 0:
+                        print('No user report available.')
+                        sys.exit(1)
+                    if fullData == 0:
+                        continue
+                page_message = gapi.got_total_items_msg('Users', '...\n')
+                usage = gapi.get_all_pages(rep.userUsageReport(), 'get',
+                                           'usageReports',
+                                           page_message=page_message,
+                                           throw_reasons=throw_reasons,
+                                           date=tryDate, userKey=userKey,
+                                           customerId=customerId,
+                                           orgUnitID=orgUnitId,
+                                           filters=filters,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No user report available.')
+            sys.exit(1)
+        titles = ['email', 'date']
+        csvRows = []
+        ptypes = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
+        for user_report in usage:
+            if 'entity' not in user_report:
+                continue
+            row = {'email': user_report['entity']
+                            ['userEmail'], 'date': tryDate}
+            for item in user_report.get('parameters', []):
+                if 'name' not in item:
+                    continue
+                name = item['name']
+                if not name in titles:
+                    titles.append(name)
+                for ptype in ptypes:
+                    if ptype in item:
+                        row[name] = item[ptype]
+                        break
+                else:
+                    row[name] = ''
+            csvRows.append(row)
+        display.write_csv_file(
+            csvRows, titles, f'User Reports - {tryDate}', to_drive)
+    elif report == 'customer':
+        while True:
+            try:
+                if fullDataRequired is not None:
+                    warnings = gapi.get_items(rep.customerUsageReports(),
+                                              'get', 'warnings',
+                                              throw_reasons=throw_reasons,
+                                              customerId=customerId,
+                                              date=tryDate,
+                                              fields='warnings')
+                    fullData, tryDate = _check_full_data_available(
+                        warnings, tryDate, fullDataRequired)
+                    if fullData < 0:
+                        print('No customer report available.')
+                        sys.exit(1)
+                    if fullData == 0:
+                        continue
+                usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                                           'usageReports',
+                                           throw_reasons=throw_reasons,
+                                           customerId=customerId,
+                                           date=tryDate,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No customer report available.')
+            sys.exit(1)
+        titles = ['name', 'value', 'client_id']
+        csvRows = []
+        auth_apps = list()
+        for item in usage[0]['parameters']:
+            if 'name' not in item:
+                continue
+            name = item['name']
+            if 'intValue' in item:
+                value = item['intValue']
+            elif 'msgValue' in item:
+                if name == 'accounts:authorized_apps':
+                    for subitem in item['msgValue']:
+                        app = {}
+                        for an_item in subitem:
+                            if an_item == 'client_name':
+                                app['name'] = 'App: ' + \
+                                    subitem[an_item].replace('\n', '\\n')
+                            elif an_item == 'num_users':
+                                app['value'] = f'{subitem[an_item]} users'
+                            elif an_item == 'client_id':
+                                app['client_id'] = subitem[an_item]
+                        auth_apps.append(app)
+                    continue
+                values = []
+                for subitem in item['msgValue']:
+                    if 'count' in subitem:
+                        mycount = myvalue = None
+                        for key, value in list(subitem.items()):
+                            if key == 'count':
+                                mycount = value
+                            else:
+                                myvalue = value
+                            if mycount and myvalue:
+                                values.append(f'{myvalue}:{mycount}')
+                        value = ' '.join(values)
+                    elif 'version_number' in subitem \
+                         and 'num_devices' in subitem:
+                        values.append(
+                            f'{subitem["version_number"]}:'
+                            f'{subitem["num_devices"]}')
+                    else:
+                        continue
+                    value = ' '.join(sorted(values, reverse=True))
+            csvRows.append({'name': name, 'value': value})
+        for app in auth_apps:  # put apps at bottom
+            csvRows.append(app)
+        display.write_csv_file(
+            csvRows, titles, f'Customer Report - {tryDate}', todrive=to_drive)
+    else:
+        page_message = gapi.got_total_items_msg('Activities', '...\n')
+        activities = gapi.get_all_pages(rep.activities(), 'list', 'items',
+                                        page_message=page_message,
+                                        applicationName=report,
+                                        userKey=userKey,
+                                        customerId=customerId,
+                                        actorIpAddress=actorIpAddress,
+                                        startTime=startTime, endTime=endTime,
+                                        eventName=eventName, filters=filters,
+                                        orgUnitID=orgUnitId)
+        if activities:
+            titles = ['name']
+            csvRows = []
+            for activity in activities:
+                events = activity['events']
+                del activity['events']
+                activity_row = utils.flatten_json(activity)
+                purge_parameters = True
+                for event in events:
+                    for item in event.get('parameters', []):
+                        if set(item) == set(['value', 'name']):
+                            event[item['name']] = item['value']
+                        elif set(item) == set(['intValue', 'name']):
+                            if item['name'] in ['start_time', 'end_time']:
+                                val = item.get('intValue')
+                                if val is not None:
+                                    val = int(val)
+                                    if val >= 62135683200:
+                                        event[item['name']] = \
+                                            datetime.datetime.fromtimestamp(
+                                                val-62135683200).isoformat()
+                            else:
+                                event[item['name']] = item['intValue']
+                        elif set(item) == set(['boolValue', 'name']):
+                            event[item['name']] = item['boolValue']
+                        elif set(item) == set(['multiValue', 'name']):
+                            event[item['name']] = ' '.join(item['multiValue'])
+                        elif item['name'] == 'scope_data':
+                            parts = {}
+                            for message in item['multiMessageValue']:
+                                for mess in message['parameter']:
+                                    value = mess.get('value', ' '.join(
+                                        mess.get('multiValue', [])))
+                                    parts[mess['name']] = parts.get(
+                                        mess['name'], [])+[value]
+                            for part, v in parts.items():
+                                if part == 'scope_name':
+                                    part = 'scope'
+                                event[part] = ' '.join(v)
+                        else:
+                            purge_parameters = False
+                    if purge_parameters:
+                        event.pop('parameters', None)
+                    row = utils.flatten_json(event)
+                    row.update(activity_row)
+                    for item in row:
+                        if item not in titles:
+                            titles.append(item)
+                    csvRows.append(row)
+            display.sort_csv_titles(['name', ], titles)
+            display.write_csv_file(
+                csvRows, titles, f'{report.capitalize()} Activity Report',
+                to_drive)
+
+
+def _adjust_date(errMsg):
+    match_date = re.match('Data for dates later than (.*) is not yet '
+                          'available. Please check back later', errMsg)
+    if not match_date:
+        match_date = re.match('Start date can not be later than (.*)', errMsg)
+    if not match_date:
+        controlflow.system_error_exit(4, errMsg)
+    return str(match_date.group(1))
+
+
+def _check_full_data_available(warnings, tryDate, fullDataRequired):
+    one_day = datetime.timedelta(days=1)
+    for warning in warnings:
+        if warning['code'] == 'PARTIAL_DATA_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    tryDateTime = datetime.datetime.strptime(
+                        tryDate, YYYYMMDD_FORMAT)
+                    tryDateTime -= one_day
+                    return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+        elif warning['code'] == 'DATA_NOT_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    return (-1, tryDate)
+    return (1, tryDate)

src/gapi/storage.py

@@ -0,0 +1,73 @@
+import base64
+import os
+import re
+import sys
+
+import googleapiclient
+
+import __main__
+from var import *
+import controlflow
+import fileutils
+import gapi
+import utils
+
+
+def build_gapi():
+    return __main__.buildGAPIObject('storage')
+
+
+def get_cloud_storage_object(s, bucket, object_, local_file=None,
+                           expectedMd5=None):
+    if not local_file:
+        local_file = object_
+    if os.path.exists(local_file):
+        sys.stdout.write(' File already exists. ')
+        sys.stdout.flush()
+        if expectedMd5:
+            sys.stdout.write(f'Verifying {expectedMd5} hash...')
+            sys.stdout.flush()
+            if utils.md5_matches_file(local_file, expectedMd5, False):
+                print('VERIFIED')
+                return
+            print('not verified. Downloading again and over-writing...')
+        else:
+            return  # nothing to verify, just assume we're good.
+    print(f'saving to {local_file}')
+    request = s.objects().get_media(bucket=bucket, object=object_)
+    file_path = os.path.dirname(local_file)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    f = fileutils.open_file(local_file, 'wb')
+    downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+    done = False
+    while not done:
+        status, done = downloader.next_chunk()
+        sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
+        sys.stdout.flush()
+    sys.stdout.write('\n Download complete. Flushing to disk...\n')
+    fileutils.close_file(f, True)
+    if expectedMd5:
+        f = fileutils.open_file(local_file, 'rb')
+        sys.stdout.write(f' Verifying file hash is {expectedMd5}...')
+        sys.stdout.flush()
+        utils.md5_matches_file(local_file, expectedMd5, True)
+        print('VERIFIED')
+        fileutils.close_file(f)
+
+
+def download_bucket():
+    bucket = sys.argv[3]
+    s = build_gapi()
+    page_message = gapi.got_total_items_msg('Files', '...')
+    fields = 'nextPageToken,items(name,id,md5Hash)'
+    objects = gapi.get_all_pages(s.objects(), 'list', 'items',
+                                 page_message=page_message, bucket=bucket,
+                                 projection='noAcl', fields=fields)
+    i = 1
+    for object_ in objects:
+        print(f'{i}/{len(objects)}')
+        expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
+        get_cloud_storage_object(
+            s, bucket, object_['name'], expectedMd5=expectedMd5)
+        i += 1

src/gapi/vault.py

@@ -0,0 +1,757 @@
+import datetime
+import json
+import sys
+
+import googleapiclient.http
+
+import __main__
+from var import *
+import controlflow
+import display
+import fileutils
+import gapi
+import gapi.storage
+import utils
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('vault')
+
+
+def validateCollaborators(collaboratorList, cd):
+    collaborators = []
+    for collaborator in collaboratorList.split(','):
+        collaborator_id = __main__.convertEmailAddressToUID(collaborator, cd)
+        if not collaborator_id:
+            controlflow.system_error_exit(4, f'failed to get a UID for '
+                                             f'{collaborator}. Please make '
+                                             f'sure this is a real user.')
+        collaborators.append({'email': collaborator, 'id': collaborator_id})
+    return collaborators
+
+
+def createMatter():
+    v = buildGAPIObject()
+    matter_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    body = {'name': f'New Matter - {matter_time}'}
+    collaborators = []
+    cd = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['collaborator', 'collaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create matter")
+    matterId = gapi.call(v.matters(), 'create', body=body,
+                         fields='matterId')['matterId']
+    print(f'Created matter {matterId}')
+    for collaborator in collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {'matterPermission': {
+            'role': 'COLLABORATOR',
+            'accountId': collaborator['id']}}
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+
+
+VAULT_SEARCH_METHODS_MAP = {
+    'account': 'ACCOUNT',
+    'accounts': 'ACCOUNT',
+    'entireorg': 'ENTIRE_ORG',
+    'everyone': 'ENTIRE_ORG',
+    'orgunit': 'ORG_UNIT',
+    'ou': 'ORG_UNIT',
+    'room': 'ROOM',
+    'rooms': 'ROOM',
+    'shareddrive': 'SHARED_DRIVE',
+    'shareddrives': 'SHARED_DRIVE',
+    'teamdrive': 'SHARED_DRIVE',
+    'teamdrives': 'SHARED_DRIVE',
+}
+VAULT_SEARCH_METHODS_LIST = ['accounts',
+                             'orgunit', 'shareddrives', 'rooms', 'everyone']
+
+
+def createExport():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['corpus']['enum'])
+    allowed_scopes = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['dataScope']['enum'])
+    allowed_formats = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['MailExportOptions']['properties']
+        ['exportFormat']['enum'])
+    export_format = 'MBOX'
+    showConfidentialModeContent = None  # default to not even set
+    matterId = None
+    body = {'query': {'dataScope': 'ALL_DATA'}, 'exportOptions': {}}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            body['matterId'] = matterId
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'corpus':
+            body['query']['corpus'] = sys.argv[i+1].upper()
+            if body['query']['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit(
+                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+            i += 2
+        elif myarg in VAULT_SEARCH_METHODS_MAP:
+            if body['query'].get('searchMethod'):
+                message = f'Multiple search methods ' \
+                          f'({", ".join(VAULT_SEARCH_METHODS_LIST)})' \
+                          f'specified, only one is allowed'
+                controlflow.system_error_exit(3, message)
+            searchMethod = VAULT_SEARCH_METHODS_MAP[myarg]
+            body['query']['searchMethod'] = searchMethod
+            if searchMethod == 'ACCOUNT':
+                body['query']['accountInfo'] = {
+                    'emails': sys.argv[i+1].split(',')}
+                i += 2
+            elif searchMethod == 'ORG_UNIT':
+                body['query']['orgUnitInfo'] = {
+                    'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+                i += 2
+            elif searchMethod == 'SHARED_DRIVE':
+                body['query']['sharedDriveInfo'] = {
+                    'sharedDriveIds': sys.argv[i+1].split(',')}
+                i += 2
+            elif searchMethod == 'ROOM':
+                body['query']['hangoutsChatInfo'] = {
+                    'roomId': sys.argv[i+1].split(',')}
+                i += 2
+            else:
+                i += 1
+        elif myarg == 'scope':
+            body['query']['dataScope'] = sys.argv[i+1].upper()
+            if body['query']['dataScope'] not in allowed_scopes:
+                controlflow.expected_argument_exit(
+                    "scope", ", ".join(allowed_scopes), sys.argv[i+1])
+            i += 2
+        elif myarg in ['terms']:
+            body['query']['terms'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            body['query']['startTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            body['query']['endTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i+1])
+            i += 2
+        elif myarg in ['timezone']:
+            body['query']['timeZone'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['excludedrafts']:
+            body['query']['mailOptions'] = {
+                'excludeDrafts': __main__.getBoolean(sys.argv[i+1], myarg)}
+            i += 2
+        elif myarg in ['driveversiondate']:
+            body['query'].setdefault('driveOptions', {})['versionDate'] = \
+                utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['includeshareddrives', 'includeteamdrives']:
+            body['query'].setdefault('driveOptions', {})[
+                'includeSharedDrives'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg in ['includerooms']:
+            body['query']['hangoutsChatOptions'] = {
+                'includeRooms': __main__.getBoolean(sys.argv[i+1], myarg)}
+            i += 2
+        elif myarg in ['format']:
+            export_format = sys.argv[i+1].upper()
+            if export_format not in allowed_formats:
+                controlflow.expected_argument_exit(
+                    "export format", ", ".join(allowed_formats), export_format)
+            i += 2
+        elif myarg in ['showconfidentialmodecontent']:
+            showConfidentialModeContent = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg in ['region']:
+            allowed_regions = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['ExportOptions']['properties'][
+                    'region']['enum'])
+            body['exportOptions']['region'] = sys.argv[i+1].upper()
+            if body['exportOptions']['region'] not in allowed_regions:
+                controlflow.expected_argument_exit("region", ", ".join(
+                    allowed_regions), body['exportOptions']['region'])
+            i += 2
+        elif myarg in ['includeaccessinfo']:
+            body['exportOptions'].setdefault('driveOptions', {})[
+                'includeAccessInfo'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create export")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new export.')
+    if 'corpus' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a corpus for the ' \
+          f'new export. Choose one of {", ".join(allowed_corpuses)}')
+    if 'searchMethod' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a search method ' \
+          'for the new export. Choose one of ' \
+          f'{", ".join(VAULT_SEARCH_METHODS_LIST)}')
+    if 'name' not in body:
+        corpus_name = body["query"]["corpus"]
+        corpus_date = datetime.datetime.now()
+        body['name'] = f'GAM {corpus_name} export - {corpus_date}'
+    options_field = None
+    if body['query']['corpus'] == 'MAIL':
+        options_field = 'mailOptions'
+    elif body['query']['corpus'] == 'GROUPS':
+        options_field = 'groupsOptions'
+    elif body['query']['corpus'] == 'HANGOUTS_CHAT':
+        options_field = 'hangoutsChatOptions'
+    if options_field:
+        body['exportOptions'].pop('driveOptions', None)
+        body['exportOptions'][options_field] = {'exportFormat': export_format}
+        if showConfidentialModeContent is not None:
+            body['exportOptions'][options_field][
+                'showConfidentialModeContent'] = showConfidentialModeContent
+    results = gapi.call(v.matters().exports(), 'create',
+                        matterId=matterId, body=body)
+    print(f'Created export {results["id"]}')
+    display.print_json(results)
+
+
+def deleteExport():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    print(f'Deleting export {sys.argv[4]} / {exportId}')
+    gapi.call(v.matters().exports(), 'delete',
+              matterId=matterId, exportId=exportId)
+
+
+def getExportInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    export = gapi.call(v.matters().exports(), 'get',
+                       matterId=matterId, exportId=exportId)
+    display.print_json(export)
+
+
+def createHold():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Hold']['properties']['corpus']['enum'])
+    body = {'query': {}}
+    i = 3
+    query = None
+    start_time = None
+    end_time = None
+    matterId = None
+    accounts = []
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg == 'corpus':
+            body['corpus'] = sys.argv[i+1].upper()
+            if body['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit(
+                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+            i += 2
+        elif myarg in ['accounts', 'users', 'groups']:
+            accounts = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new hold.')
+    if not body.get('name'):
+        controlflow.system_error_exit(
+            3, 'you must specify a name for the new hold.')
+    if not body.get('corpus'):
+        controlflow.system_error_exit(3, f'you must specify a corpus for ' \
+          f'the new hold. Choose one of {", ".join(allowed_corpuses)}')
+    if body['corpus'] == 'HANGOUTS_CHAT':
+        query_type = 'hangoutsChatQuery'
+    else:
+        query_type = f'{body["corpus"].lower()}Query'
+    body['query'][query_type] = {}
+    if body['corpus'] == 'DRIVE':
+        if query:
+            try:
+                body['query'][query_type] = json.loads(query)
+            except ValueError as e:
+                controlflow.system_error_exit(3, f'{str(e)}, query: {query}')
+    elif body['corpus'] in ['GROUPS', 'MAIL']:
+        if query:
+            body['query'][query_type] = {'terms': query}
+        if start_time:
+            body['query'][query_type]['startTime'] = start_time
+        if end_time:
+            body['query'][query_type]['endTime'] = end_time
+    if accounts:
+        body['accounts'] = []
+        cd = __main__.buildGAPIObject('directory')
+        account_type = 'group' if body['corpus'] == 'GROUPS' else 'user'
+        for account in accounts:
+            body['accounts'].append(
+                {'accountId': __main__.convertEmailAddressToUID(account,
+                                                                cd,
+                                                                account_type)}
+            )
+    holdId = gapi.call(v.matters().holds(), 'create',
+                       matterId=matterId, body=body, fields='holdId')
+    print(f'Created hold {holdId["holdId"]}')
+
+
+def deleteHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam delete hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    print(f'Deleting hold {hold} / {holdId}')
+    gapi.call(v.matters().holds(), 'delete', matterId=matterId, holdId=holdId)
+
+
+def getHoldInfo():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam info hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    results = gapi.call(v.matters().holds(), 'get',
+                        matterId=matterId, holdId=holdId)
+    cd = __main__.buildGAPIObject('directory')
+    if 'accounts' in results:
+        account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
+        for i in range(0, len(results['accounts'])):
+            uid = f'uid:{results["accounts"][i]["accountId"]}'
+            acct_email = __main__.convertUIDtoEmailAddress(
+                uid, cd, [account_type])
+            results['accounts'][i]['email'] = acct_email
+    if 'orgUnit' in results:
+        results['orgUnit']['orgUnitPath'] = __main__.doGetOrgInfo(
+            results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
+    display.print_json(results)
+
+
+def convertExportNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'exports(id,name),nextPageToken'
+    exports = gapi.get_all_pages(v.matters().exports(
+    ), 'list', 'exports', matterId=matterId, fields=fields)
+    for export in exports:
+        if export['name'].lower() == nameOrID:
+            return export['id']
+    controlflow.system_error_exit(4, f'could not find export name {nameOrID} '
+                                     f'in matter {matterId}')
+
+
+def convertHoldNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'holds(holdId,name),nextPageToken'
+    holds = gapi.get_all_pages(v.matters().holds(
+    ), 'list', 'holds', matterId=matterId, fields=fields)
+    for hold in holds:
+        if hold['name'].lower() == nameOrID:
+            return hold['holdId']
+    controlflow.system_error_exit(4, f'could not find hold name {nameOrID} '
+                                     f'in matter {matterId}')
+
+
+def convertMatterNameToID(v, nameOrID):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'matters(matterId,name),nextPageToken'
+    matters = gapi.get_all_pages(v.matters(
+    ), 'list', 'matters', view='BASIC', fields=fields)
+    for matter in matters:
+        if matter['name'].lower() == nameOrID:
+            return matter['matterId']
+    return None
+
+
+def getMatterItem(v, nameOrID):
+    matterId = convertMatterNameToID(v, nameOrID)
+    if not matterId:
+        controlflow.system_error_exit(4, f'could not find matter {nameOrID}')
+    return matterId
+
+
+def updateHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    body = {}
+    query = None
+    add_accounts = []
+    del_accounts = []
+    start_time = None
+    end_time = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['addusers', 'addaccounts', 'addgroups']:
+            add_accounts = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg in ['removeusers', 'removeaccounts', 'removegroups']:
+            del_accounts = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam update hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    if query or start_time or end_time or body.get('orgUnit'):
+        fields = 'corpus,query,orgUnit'
+        old_body = gapi.call(v.matters().holds(
+        ), 'get', matterId=matterId, holdId=holdId, fields=fields)
+        body['query'] = old_body['query']
+        body['corpus'] = old_body['corpus']
+        if 'orgUnit' in old_body and 'orgUnit' not in body:
+            # bah, API requires this to be sent
+            # on update even when it's not changing
+            body['orgUnit'] = old_body['orgUnit']
+        query_type = f'{body["corpus"].lower()}Query'
+        if body['corpus'] == 'DRIVE':
+            if query:
+                try:
+                    body['query'][query_type] = json.loads(query)
+                except ValueError as e:
+                    message = f'{str(e)}, query: {query}'
+                    controlflow.system_error_exit(3, message)
+        elif body['corpus'] in ['GROUPS', 'MAIL']:
+            if query:
+                body['query'][query_type]['terms'] = query
+            if start_time:
+                body['query'][query_type]['startTime'] = start_time
+            if end_time:
+                body['query'][query_type]['endTime'] = end_time
+    if body:
+        print(f'Updating hold {hold} / {holdId}')
+        gapi.call(v.matters().holds(), 'update',
+                  matterId=matterId, holdId=holdId, body=body)
+    if add_accounts or del_accounts:
+        cd = __main__.buildGAPIObject('directory')
+        for account in add_accounts:
+            print(f'adding {account} to hold.')
+            add_body = {'accountId': __main__.convertEmailAddressToUID(account, cd)}
+            gapi.call(v.matters().holds().accounts(), 'create',
+                      matterId=matterId, holdId=holdId, body=add_body)
+        for account in del_accounts:
+            print(f'removing {account} from hold.')
+            accountId = __main__.convertEmailAddressToUID(account, cd)
+            gapi.call(v.matters().holds().accounts(), 'delete',
+                      matterId=matterId, holdId=holdId, accountId=accountId)
+
+
+def updateMatter(action=None):
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    body = {}
+    action_kwargs = {'body': {}}
+    add_collaborators = []
+    remove_collaborators = []
+    cd = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            action = sys.argv[i+1].lower()
+            if action not in VAULT_MATTER_ACTIONS:
+                controlflow.system_error_exit(3, f'allowed actions are ' \
+                    f'{", ".join(VAULT_MATTER_ACTIONS)}, got {action}')
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['addcollaborator', 'addcollaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            add_collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        elif myarg in ['removecollaborator', 'removecollaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            remove_collaborators.extend(
+                validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam update matter")
+    if action == 'delete':
+        action_kwargs = {}
+    if body:
+        print(f'Updating matter {sys.argv[3]}...')
+        if 'name' not in body or 'description' not in body:
+            # bah, API requires name/description to be sent
+            # on update even when it's not changing
+            result = gapi.call(v.matters(), 'get',
+                               matterId=matterId, view='BASIC')
+            body.setdefault('name', result['name'])
+            body.setdefault('description', result.get('description'))
+        gapi.call(v.matters(), 'update', body=body, matterId=matterId)
+    if action:
+        print(f'Performing {action} on matter {sys.argv[3]}')
+        gapi.call(v.matters(), action, matterId=matterId, **action_kwargs)
+    for collaborator in add_collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {'matterPermission': {'role': 'COLLABORATOR',
+                                     'accountId': collaborator['id']}}
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+    for collaborator in remove_collaborators:
+        print(f' removing collaborator {collaborator["email"]}')
+        gapi.call(v.matters(), 'removePermissions', matterId=matterId,
+                  body={'accountId': collaborator['id']})
+
+
+def getMatterInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    result = gapi.call(v.matters(), 'get', matterId=matterId, view='FULL')
+    if 'matterPermissions' in result:
+        cd = __main__.buildGAPIObject('directory')
+        for i in range(0, len(result['matterPermissions'])):
+            uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
+            user_email = __main__.convertUIDtoEmailAddress(uid, cd)
+            result['matterPermissions'][i]['email'] = user_email
+    display.print_json(result)
+
+
+def downloadExport():
+    verifyFiles = True
+    extractFiles = True
+    v = buildGAPIObject()
+    s = gapi.storage.build_gapi()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i+1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        elif myarg == 'noverify':
+            verifyFiles = False
+            i += 1
+        elif myarg == 'noextract':
+            extractFiles = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam download export")
+    export = gapi.call(v.matters().exports(), 'get',
+                       matterId=matterId, exportId=exportId)
+    for s_file in export['cloudStorageSink']['files']:
+        bucket = s_file['bucketName']
+        s_object = s_file['objectName']
+        filename = os.path.join(targetFolder, s_object.replace('/', '-'))
+        print(f'saving to {filename}')
+        request = s.objects().get_media(bucket=bucket, object=s_object)
+        f = fileutils.open_file(filename, 'wb')
+        downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+        done = False
+        while not done:
+            status, done = downloader.next_chunk()
+            sys.stdout.write(
+                ' Downloaded: {0:>7.2%}\r'.format(status.progress()))
+            sys.stdout.flush()
+        sys.stdout.write('\n Download complete. Flushing to disk...\n')
+        fileutils.close_file(f, True)
+        if verifyFiles:
+            expected_hash = s_file['md5Hash']
+            sys.stdout.write(f' Verifying file hash is {expected_hash}...')
+            sys.stdout.flush()
+            utils.md5_matches_file(filename, expected_hash, True)
+            print('VERIFIED')
+        if extractFiles and re.search(r'\.zip$', filename):
+            __main__.extract_nested_zip(filename, targetFolder)
+
+
+def printMatters():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'name', 'description', 'state']
+    titles = initialTitles[:]
+    view = 'FULL'
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            view = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print matters")
+    __main__.printGettingAllItems('Vault Matters', None)
+    page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
+    matters = gapi.get_all_pages(
+        v.matters(), 'list', 'matters', page_message=page_message, view=view)
+    for matter in matters:
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(matter), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Matters', todrive)
+
+
+def printExports():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'id', 'name', 'createTime', 'status']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print exports")
+    if not matters:
+        fields = 'matters(matterId,state),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(
+        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        for matter in matters_results:
+            matterState = matter['state']
+            matterId = matter['matterId']
+            matterIds.append(matterId)
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving exports for matter {matterId}\n')
+        exports = gapi.get_all_pages(
+            v.matters().exports(), 'list', 'exports', matterId=matterId)
+        for export in exports:
+            display.add_row_titles_to_csv_file(utils.flatten_json(
+                export, flattened={'matterId': matterId}), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Exports', todrive)
+
+
+def printHolds():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'holdId', 'name', 'corpus', 'updateTime']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print holds")
+    if not matters:
+        fields = 'matters(matterId,state),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(
+        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        for matter in matters_results:
+            matterState = matter['state']
+            matterId = matter['matterId']
+            matterIds.append(matterId)
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving holds for matter {matterId}\n')
+        holds = gapi.get_all_pages(
+            v.matters().holds(), 'list', 'holds', matterId=matterId)
+        for hold in holds:
+            display.add_row_titles_to_csv_file(utils.flatten_json(
+                hold, flattened={'matterId': matterId}), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Holds', todrive)

src/linux-gam.spec

@@ -1,8 +1,13 @@
 # -*- mode: python -*-
+
+import sys
+
+sys.modules['FixTk'] = None
+
 a = Analysis(['gam.py'],
              hiddenimports=[],
              hookspath=None,
-             excludes=['_tkinter'],
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
              runtime_hooks=None)
 for d in a.datas:
     if 'pyconfig' in d[0]:

src/macos-gam.spec

@@ -1,8 +1,12 @@
 # -*- mode: python -*-
+import sys
+
+sys.modules['FixTk'] = None
+
 a = Analysis(['gam.py'],
              hiddenimports=[],
              hookspath=None,
-             excludes=['_tkinter'],
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
              runtime_hooks=None)
 for d in a.datas:
     if 'pyconfig' in d[0]:

src/project-apis.txt

@@ -2,11 +2,15 @@ admin.googleapis.com
 alertcenter.googleapis.com
 appsactivity.googleapis.com
 calendar-json.googleapis.com
+chat.googleapis.com
 classroom.googleapis.com
+cloudidentity.googleapis.com
 contacts.googleapis.com
 drive.googleapis.com
+iap.googleapis.com
 gmail.googleapis.com
 groupssettings.googleapis.com
+iam.googleapis.com
 licensing.googleapis.com
 plus.googleapis.com
 reseller.googleapis.com

src/requirements.txt

@@ -1,9 +1,10 @@
+cryptography
 python-dateutil
 distro; sys_platform == 'linux'
 filelock
 google-api-python-client>=1.7.10
-google-auth
+google-auth>=1.11.2
 google-auth-httplib2
-google-auth-oauthlib==0.4.0
-httplib2>=0.13.0
-passlib
+google-auth-oauthlib>=0.4.1
+httplib2>=0.17.0
+passlib>=1.7.2; sys_platform == 'win32'

src/tools/parse-aue.py

@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-from xml.etree import ElementTree as ET
-import requests
-from html.parser import HTMLParser
-import string
-import sys
-import json
-import dateutil.parser
-
-class MyHTMLParser(HTMLParser):
-    def handle_starttag(self, tag, attrs):
-        global next_data_is_oem, next_data_is_td
-        if tag == 'h2' and attrs == [('class', 'zippy')]:
-            next_data_is_oem = True
-        elif tag == 'td':
-            next_data_is_td = True
-
-    def handle_data(self, data):
-        global oem, next_data_is_oem, next_data_is_td, data_is_date, model, printable, output_rows
-        if next_data_is_oem:
-            oem = ''.join(filter(lambda x: x in printable, data))
-            next_data_is_oem = False
-        elif next_data_is_td:
-            if data_is_date:
-                if model.lower().startswith(oem.lower()):
-                    fullname = model.lower()
-                else:
-                    fullname = '%s %s' % (oem, model)
-                    fullname = fullname.lower()
-                date = dateutil.parser.parse(data).replace(day=1).strftime('%Y-%m-%dT00:00:00.000Z')
-                output_rows[fullname] = date
-                if fullname in exceptions:
-                    for value in exceptions[fullname]:
-                        output_rows[value] = date
-                data_is_date = False
-            else:
-                model = ''.join(filter(lambda x: x in printable, data))
-                data_is_date = True
-            next_data_is_td = False
-
-global oem, next_data_is_oem, next_data_is_td, data_is_date, model, printable, exceptions, output_rows
-output_rows = {}
-printable = set(string.printable)
-exceptions = {
-        # 'AUE OEM MODEL': ['API MODEL 1', ...]
-        'acer c7 chromebook': ['acer c7 chromebook (c710)'],
-        'acer chromebook 11 (c720, c720p)': ['acer c720 chromebook', 'acer c740 chromebook'],
-        'acer chromebook 11 (cb3-111, c730, c730e)': ['chromebook 11 (c730 / cb3-111)'],
-        'acer chromebook 11 (cb3-131, c735)': ['chromebook 11 (c735)'],
-        'acer chromebook 15 (cb515-1h,cb515-1ht)': ['chromebook 15 (cb515 - 1ht / 1h)'],
-        'acer chromebook 13(cb5-311, c810)': ['acer chromebook 13 (cb5-311)'],
-        'acer chromebook 15 (cb5-571, c910)': ['acer chromebook 15 (c910 / cb5-571)'],
-        'acer chromebook 311 (c721, c733, c733u, c733t)': ['acer chromebook 311', 'chromebook 311 (c721)'],
-        'acer chromebook 315 (cb315-2h)': ['acer chromebook 315'],
-        'acer chromebook spin 311 (r721t)': ['acer chromebook 311'],
-        'acer chromebook spin 511 (r752t, r752tn)': ['acer chromebook spin 511'],
-        'acer chromebox cxi2 / cxv2': ['acer chromebox cxi2'],
-        'asus chromebook c200': ['asus chromebook c200ma'],
-        'asus chromebook c201pa': ['asus chromebook c201pa'],
-        'asus chromebook c204': ['asus chromebook c204'],
-        'asus chromebook c300': ['asus chromebook c300ma'],
-        'asus chromebook flip c213': ['asus chromebook c213na'],
-        'asus chromebox 2 (cn62)': ['asus chromebox cn62'],
-        'asus chromebox 3 (cn65)': ['asus chromebox 3'],
-        'asus chromebox (cn60)': ['asus chromebox cn60'],
-        'ctl chromebook nl7 / nl7t-360 / nl7tw-360': ['ctl chromebook nl7'],
-        'ctl chromebook tablet tx1 for education': ['ctl chromebook tab tx1'],
-        'ctl nl61 chromebook': ['mecer v2 chromebook'],
-        'google cr-48': ['cr-48'],
-        'haier chromebook 11e': ['chromebook pcm-116e', 'lumos education chromebook'],
-        'haier chromebook 11': ['true idc chromebook 11', 'xolo chromebook'],
-        'hisense chromebook 11': ['epik 11.6" chromebook  elb1101', 'mecer chromebook', 'videonet chromebook bl10'],
-        'hp chromebook 11 g1': ['hp chromebook 11 1100-1199 / hp chromebook 11 g1'],
-        'hp chromebook 11 g2': ['hp chromebook 11 2000-2099 / hp chromebook 11 g2'],
-        'hp chromebook 11 g3': ['hp chromebook 11 2100-2199 / hp chromebook 11 g3'],
-        'hp chromebook 11 g4/g4 ee': ['hp chromebook 11 2200-2299 / hp chromebook 11 g4/g4 ee'],
-        'hp chromebook 11 g5': ['hp chromebook 11 g5 / hp chromebook 11-vxxx'],
-        'hp chromebook 14a g5': ['hp chromebook 14 db0000-db0999'],
-        'hp chromebook 14 g3': ['hp chromebook 14 x000-x999 / hp chromebook 14 g3'],
-        'hp chromebook 14 g4': ['hp chromebook 14 ak000-099 / hp chromebook 14 g4'],
-        'hp chromebook 14 g5': ['hp chromebook 14 / hp chromebook 14 g5'],
-        'hp chromebox g1': ['hp chromebox cb1-(000-099) / hp chromebox g1/ hp chromebox for meetings'],
-        'lenovo ideapad c330 chromebook': ['lenovo chromebook c330'],
-        'lenovo ideapad s330 chromebook': ['lenovo chromebook s330'],
-        'lenovo n21 chromebook': ['asi chromebook', 'crambo chromebook', 'jp sa couto chromebook', 'rgs education chromebook', 'true idc chromebook', 'videonet chromebook', 'consumer chromebook'],
-        'lenovo thinkpad 11e 3rd gen chromebook': ['thinkpad 11e chromebook 3rd gen (yoga/clamshell)'],
-        'lenovo thinkpad 11e 4th gen chromebook': ['lenovo thinkpad 11e chromebook (4th gen)/lenovo thinkpad yoga 11e chromebook (4th gen)'],
-        'lenovo thinkpad 13': ['thinkpad 13 chromebook'],
-        'poin2 chromebook 14': ['poin2 chromebook 11c'],
-        'prowise chromebook eduline': ['viglen chromebook 11c'],
-        'prowise chromebook entryline': ['prowise 11.6\" entry line chromebook'],
-        'prowise chromebook proline': ['prowise proline chromebook'],
-        'samsung chromebook - xe303': ['samsung chromebook'],
-        }
-next_data_is_oem = False
-next_data_is_td = False
-data_is_date = False
-auepage = requests.get('https://support.google.com/chrome/a/answer/6220366?hl=en')
-parser = MyHTMLParser()
-parser.feed(auepage.content.decode('utf-8'))
-print(json.dumps(output_rows, indent=2, sort_keys=True))

src/transport.py

@@ -0,0 +1,98 @@
+"""Methods related to network transport."""
+
+import google_auth_httplib2
+import httplib2
+
+from var import GAM_INFO
+from var import GC_CA_FILE
+from var import GC_TLS_MAX_VERSION
+from var import GC_TLS_MIN_VERSION
+from var import GC_Values
+
+
+def create_http(cache=None,
+                timeout=None,
+                override_min_tls=None,
+                override_max_tls=None):
+  """Creates a uniform HTTP transport object.
+
+  Args:
+    cache: The HTTP cache to use.
+    timeout: The cache timeout, in seconds.
+    override_min_tls: The minimum TLS version to require. If not provided, the
+      default is used.
+    override_max_tls: The maximum TLS version to require. If not provided, the
+      default is used.
+
+  Returns:
+    httplib2.Http with the specified options.
+  """
+  tls_minimum_version = override_min_tls if override_min_tls else GC_Values[GC_TLS_MIN_VERSION]
+  tls_maximum_version = override_max_tls if override_max_tls else GC_Values[GC_TLS_MAX_VERSION]
+  httpObj = httplib2.Http(ca_certs=GC_Values[GC_CA_FILE],
+                          tls_maximum_version=tls_maximum_version,
+                          tls_minimum_version=tls_minimum_version,
+                          cache=cache,
+                          timeout=timeout)
+  httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
+  return httpObj
+
+
+def create_request(http=None):
+  """Creates a uniform Request object with a default http, if not provided.
+
+  Args:
+    http: Optional httplib2.Http compatible object to be used with the request.
+      If not provided, a default HTTP will be used.
+
+  Returns:
+    Request: A google_auth_httplib2.Request compatible Request.
+  """
+  if not http:
+    http = create_http()
+  return Request(http)
+
+
+GAM_USER_AGENT = GAM_INFO
+
+
+def _force_user_agent(user_agent):
+  """Creates a decorator which can force a user agent in HTTP headers."""
+
+  def decorator(request_method):
+    """Wraps a request method to insert a user-agent in HTTP headers."""
+
+    def wrapped_request_method(*args, **kwargs):
+      """Modifies HTTP headers to include a specified user-agent."""
+      if kwargs.get('headers') is not None:
+        if kwargs['headers'].get('user-agent'):
+          if user_agent not in kwargs['headers']['user-agent']:
+            # Save the existing user-agent header and tack on our own.
+            kwargs['headers']['user-agent'] = f'{user_agent} {kwargs["headers"]["user-agent"]}'
+        else:
+          kwargs['headers']['user-agent'] = user_agent
+      else:
+        kwargs['headers'] = {'user-agent': user_agent}
+      return request_method(*args, **kwargs)
+
+    return wrapped_request_method
+
+  return decorator
+
+
+class Request(google_auth_httplib2.Request):
+  """A Request which forces a user agent."""
+
+  @_force_user_agent(GAM_USER_AGENT)
+  def __call__(self, *args, **kwargs):
+    """Inserts the GAM user-agent header in requests."""
+    return super(Request, self).__call__(*args, **kwargs)
+
+
+class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
+  """An AuthorizedHttp which forces a user agent during requests."""
+
+  @_force_user_agent(GAM_USER_AGENT)
+  def request(self, *args, **kwargs):
+    """Inserts the GAM user-agent header in requests."""
+    return super(AuthorizedHttp, self).request(*args, **kwargs)

src/transport_test.py

@@ -0,0 +1,179 @@
+"""Tests for transport."""
+
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import google_auth_httplib2
+import httplib2
+
+import transport
+
+
+class CreateHttpTest(unittest.TestCase):
+
+  def setUp(self):
+    SetGlobalVariables()
+    super(CreateHttpTest, self).setUp()
+
+  def test_create_http_sets_default_values_on_http(self):
+    http = transport.create_http()
+    self.assertIsNone(http.cache)
+    self.assertIsNone(http.timeout)
+    self.assertEqual(http.tls_minimum_version,
+                     transport.GC_Values[transport.GC_TLS_MIN_VERSION])
+    self.assertEqual(http.tls_maximum_version,
+                     transport.GC_Values[transport.GC_TLS_MAX_VERSION])
+    self.assertEqual(http.ca_certs, transport.GC_Values[transport.GC_CA_FILE])
+
+  def test_create_http_sets_tls_min_version(self):
+    http = transport.create_http(override_min_tls='TLSv1_1')
+    self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
+
+  def test_create_http_sets_tls_max_version(self):
+    http = transport.create_http(override_max_tls='TLSv1_3')
+    self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
+
+  def test_create_http_sets_cache(self):
+    fake_cache = {}
+    http = transport.create_http(cache=fake_cache)
+    self.assertEqual(http.cache, fake_cache)
+
+  def test_create_http_sets_cache_timeout(self):
+    http = transport.create_http(timeout=1234)
+    self.assertEqual(http.timeout, 1234)
+
+
+class TransportTest(unittest.TestCase):
+
+  def setUp(self):
+    self.mock_http = MagicMock(spec=httplib2.Http)
+    self.mock_response = MagicMock(spec=httplib2.Response)
+    self.mock_content = MagicMock()
+    self.mock_http.request.return_value = (self.mock_response,
+                                           self.mock_content)
+    self.mock_credentials = MagicMock()
+    self.test_uri = 'http://example.com'
+    super(TransportTest, self).setUp()
+
+  @patch.object(transport, 'create_http')
+  def test_create_request_uses_default_http(self, mock_create_http):
+    request = transport.create_request()
+    self.assertEqual(request.http, mock_create_http.return_value)
+
+  def test_create_request_uses_provided_http(self):
+    request = transport.create_request(http=self.mock_http)
+    self.assertEqual(request.http, self.mock_http)
+
+  def test_create_request_returns_request_with_forced_user_agent(self):
+    request = transport.create_request()
+    self.assertIsInstance(request, transport.Request)
+
+  def test_request_is_google_auth_httplib2_compatible(self):
+    request = transport.create_request()
+    self.assertIsInstance(request, google_auth_httplib2.Request)
+
+  def test_request_call_returns_response_content(self):
+    request = transport.Request(self.mock_http)
+    response = request(self.test_uri)
+    self.assertEqual(self.mock_response.status, response.status)
+    self.assertEqual(self.mock_content, response.data)
+
+  def test_request_call_forces_user_agent_no_provided_headers(self):
+    request = transport.Request(self.mock_http)
+
+    request(self.test_uri)
+    headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', headers)
+    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+  def test_request_call_forces_user_agent_no_agent_in_headers(self):
+    request = transport.Request(self.mock_http)
+    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
+
+    request(self.test_uri, headers=fake_request_headers)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+    self.assertEqual('someData',
+                     final_headers['some-header-thats-not-a-user-agent'])
+
+  def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
+    request = transport.Request(self.mock_http)
+    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+    request(self.test_uri, headers=headers_with_user_agent)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn('existing-user-agent', final_headers['user-agent'])
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+  def test_request_call_same_user_agent_already_in_headers(self):
+    request = transport.Request(self.mock_http)
+    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+    request(self.test_uri, headers=same_user_agent_header)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    # Make sure the header wasn't duplicated
+    self.assertEqual(
+        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))
+
+  def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
+    http = transport.AuthorizedHttp(self.mock_credentials)
+    self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
+
+  def test_authorizedhttp_request_returns_response_content(self):
+    http = transport.AuthorizedHttp(self.mock_credentials, http=self.mock_http)
+    response, content = http.request(self.test_uri)
+    self.assertEqual(self.mock_response, response)
+    self.assertEqual(self.mock_content, content)
+
+  def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    authorized_http.request(self.test_uri)
+    headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', headers)
+    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+  def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
+
+    authorized_http.request(self.test_uri, headers=fake_request_headers)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+    self.assertEqual('someData',
+                     final_headers['some-header-thats-not-a-user-agent'])
+
+  def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
+      self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+    authorized_http.request(self.test_uri, headers=headers_with_user_agent)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn('existing-user-agent', final_headers['user-agent'])
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+  def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+    authorized_http.request(self.test_uri, headers=same_user_agent_header)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    # Make sure the header wasn't duplicated
+    self.assertEqual(
+        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))

src/travis/linux-arm64-before-install.sh

@@ -0,0 +1,75 @@
+export whereibelong=$(pwd)
+export dist=$(lsb_release --codename --short)
+echo "We are running on Ubuntu $dist"
+export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+cpucount=$(nproc --all)
+echo "This device has $cpucount CPUs for compiling..."
+echo -e "nameserver 8.8.8.8\nnameserver 8.8.4.4" > /tmp/resolv.conf
+sudo cp /tmp/resolv.conf /etc
+sudo apt-get -qq --yes update > /dev/null
+sudo apt-get -qq --yes install xz-utils > /dev/null
+SSLVER=$(~/ssl/bin/openssl version)
+SSLRESULT=$?
+PYVER=$(~/python/bin/python3 -V)
+PYRESULT=$?
+if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+  echo "RUNNING: apt dist-upgrade..."
+  sudo apt-get -qq --yes dist-upgrade > /dev/null
+  echo "Installing build tools..."
+  sudo apt-get -qq --yes install build-essential
+
+  echo "Installing deps for python3"
+  sudo cp -v /etc/apt/sources.list /tmp
+  sudo chmod a+rwx /tmp/sources.list
+  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+  sudo cp -v /tmp/sources.list /etc/apt
+  sudo apt-get -qq --yes update > /dev/null
+  sudo apt-get -qq --yes build-dep python3 > /dev/null
+  sudo apt-get -qq --yes install zlib1g-dev > /dev/null
+  sudo apt-get -qq --yes install libffi-dev > /dev/null
+
+  # Compile latest OpenSSL
+  echo "Downloading OpenSSL..."
+  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  echo "Extracting OpenSSL..."
+  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  cd openssl-$BUILD_OPENSSL_VERSION
+  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+  ./config shared --prefix=$HOME/ssl
+  echo "Running make for OpenSSL..."
+  make -j$cpucount -s
+  echo "Running make install for OpenSSL..."
+  make install > /dev/null
+  cd ~
+
+  # Compile latest Python
+  echo "Downloading Python $BUILD_PYTHON_VERSION..."
+  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+  echo "Extracting Python..."
+  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+  cd Python-$BUILD_PYTHON_VERSION
+  echo "Compiling Python $BUILD_PYTHON_VERSION..."
+  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+  unsafe_flags="--enable-optimizations --with-lto"  
+  echo "running configure with safe and unsafe"
+  ./configure $safe_flags $unsafe_flags > /dev/null
+  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+  echo "Installing Python..."
+  make install > /dev/null
+fi
+
+python=~/python/bin/python3
+pip=~/python/bin/pip3
+
+$python -V
+
+cd $whereibelong
+
+echo "Upgrading pip packages..."
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+
+mkdir ~/.ruby
+export GEM_HOME=~/.ruby
+export PATH=$PATH:~/.ruby/bin

src/travis/linux-arm64-install.sh

@@ -0,0 +1,33 @@
+cd src
+if [ "$VMTYPE" == "test" ]; then
+  export gam="$python gam.py"
+  export gampath=$(readlink -e .)
+else
+  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+  export gam="gam/gam"
+  export gampath=$(readlink -e gam)
+  export GAMVERSION=`$gam version simple`
+  cp LICENSE $gampath
+  cp whatsnew.txt $gampath
+  cp GamCommands.txt $gampath
+  this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+  GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
+  rm $gampath/lastupdatecheck.txt
+  tar cfJ $GAM_ARCHIVE gam/
+  echo "PyInstaller GAM info:"
+  du -h gam/gam
+  time $gam version extended
+
+  if [[ "$dist" == "precise" ]]; then
+    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
+    $python -OO -m staticx gam/gam gam/gam-staticx
+    strip gam/gam-staticx
+    rm gam/gam
+    mv gam/gam-staticx gam/gam
+    tar cfJ $GAM_LEGACY_ARCHIVE gam/
+    echo "Legacy StaticX GAM info:"
+    du -h gam/gam
+    time $gam version extended
+  fi
+
+fi

src/travis/linux-x86_64-before-install.sh

@@ -7,79 +7,90 @@ else
   export whereibelong=$(pwd)
   export dist=$(lsb_release --codename --short)
   echo "We are running on Ubuntu $dist"
-  echo "RUNNING: apt update..."
-  sudo apt-get -qq --yes update > /dev/null
-  echo "RUNNING: apt dist-upgrade..."
-#  sudo apt-get -qq --yes dist-upgrade > /dev/null
-  echo "Installing build tools..."
-  sudo apt-get -qq --yes install build-essential
-
-  echo "Installing deps for python3"
-  sudo cp -v /etc/apt/sources.list /tmp
-  sudo chmod a+rwx /tmp/sources.list
-  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
-  sudo cp -v /tmp/sources.list /etc/apt
-  sudo apt-get -qq --yes update > /dev/null
-  sudo apt-get -qq --yes build-dep python3 > /dev/null
-
-  mypath=$HOME
-  echo "My Path is $mypath"
+  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   cpucount=$(nproc --all)
   echo "This device has $cpucount CPUs for compiling..."
+  SSLVER=$(~/ssl/bin/openssl version)
+  SSLRESULT=$?
+  PYVER=$(~/python/bin/python3 -V)
+  PYRESULT=$?
+  if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+    echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+    if [ $SSLRESULT -ne 0 ]; then
+      echo "sslresult -ne 0"
+    fi
+    if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+      echo "sslver not equal to..."
+    fi
+    if [ $PYRESULT -ne 0 ]; then
+      echo "pyresult -ne 0"
+    fi
+    if [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION" ]]; then
+      echo "pyver not equal to..."
+    fi
+    cd ~
+    rm -rf ssl
+    rm -rf python
+    mkdir ssl
+    mkdir python
+    echo "RUNNING: apt update..."
+    sudo apt-get -qq --yes update > /dev/null
+    echo "RUNNING: apt dist-upgrade..."
+    sudo apt-get -qq --yes dist-upgrade > /dev/null
+    echo "Installing build tools..."
+    sudo apt-get -qq --yes install build-essential
+    echo "Installing deps for python3"
+    sudo cp -v /etc/apt/sources.list /tmp
+    sudo chmod a+rwx /tmp/sources.list
+    echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+    sudo cp -v /tmp/sources.list /etc/apt
+    sudo apt-get -qq --yes update > /dev/null
+    sudo apt-get -qq --yes build-dep python3 > /dev/null
 
-  cd ~/pybuild
-  # Compile latest OpenSSL
-  if [ ! -d openssl-$BUILD_OPENSSL_VERSION ]; then
+    # Compile latest OpenSSL
     wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
     echo "Extracting OpenSSL..."
     tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  fi
-  cd openssl-$BUILD_OPENSSL_VERSION
-  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-  ./config shared --prefix=$mypath/ssl
-  echo "Running make for OpenSSL..."
-  make -j$cpucount -s
-  echo "Running make install for OpenSSL..."
-  make install > /dev/null
-  export LD_LIBRARY_PATH=~/ssl/lib
-  cd ~
+    cd openssl-$BUILD_OPENSSL_VERSION
+    echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+    ./config shared --prefix=$HOME/ssl
+    echo "Running make for OpenSSL..."
+    make -j$cpucount -s
+    echo "Running make install for OpenSSL..."
+    make install > /dev/null
+    cd ~
 
-  cd ~/pybuild
-  # Compile latest Python
-  if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
+    # Compile latest Python
     echo "Downloading Python $BUILD_PYTHON_VERSION..."
     curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
     echo "Extracting Python..."
     tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+    cd Python-$BUILD_PYTHON_VERSION
+    echo "Compiling Python $BUILD_PYTHON_VERSION..."
+    safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+    unsafe_flags="--enable-optimizations --with-lto"
+    if [ ! -e Makefile ]; then
+      echo "running configure with safe and unsafe"
+      ./configure $safe_flags $unsafe_flags > /dev/null
+    fi
+    make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+    RESULT=$?
+    echo "First make exited with $RESULT"
+    if [ $RESULT != 0 ]; then
+      echo "Trying Python compile again without unsafe flags..."
+      make clean
+      ./configure $safe_flags > /dev/null
+      make -j$cpucount -s
+      echo "Sticking with safe Python for now..."
+    fi
+    echo "Installing Python..."
+    make install > /dev/null
+    cd ~
   fi
-  cd Python-$BUILD_PYTHON_VERSION
-  echo "Compiling Python $BUILD_PYTHON_VERSION..."
-  safe_flags="--with-openssl=$mypath/ssl --enable-shared --prefix=$mypath/python --with-ensurepip=upgrade"
-  unsafe_flags="--enable-optimizations --with-lto"  
-  if [ ! -e Makefile ]; then
-    echo "running configure with safe and unsafe"
-    ./configure $safe_flags $unsafe_flags > /dev/null
-  fi
-  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
-  RESULT=$?
-  echo "First make exited with $RESULT"
-  if [ $RESULT != 0 ]; then
-    echo "Trying Python compile again without unsafe flags..."
-    make clean
-    ./configure $safe_flags > /dev/null
-    make -j$cpucount -s
-    echo "Sticking with safe Python for now..."
-  fi
-  echo "Installing Python..."
-  make install > /dev/null
-  cd ~
 
-  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   python=~/python/bin/python3
   pip=~/python/bin/pip3
 
-  $python -V
-
   if [[ "$dist" == "precise" ]]; then
     echo "Installing deps for StaticX..."
     sudo apt-get install --yes scons
@@ -94,10 +105,11 @@ else
     fi
     $pip install git+https://github.com/JonathonReinhart/staticx.git@master
   fi
+
   cd $whereibelong
 fi
 
 echo "Upgrading pip packages..."
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade pyinstaller
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz

src/travis/linux-x86_64-install.sh

@@ -24,10 +24,10 @@ else
     strip gam/gam-staticx
     rm gam/gam
     mv gam/gam-staticx gam/gam
+    chmod 755 gam/gam
     tar cfJ $GAM_LEGACY_ARCHIVE gam/
     echo "Legacy StaticX GAM info:"
     du -h gam/gam
     time $gam version extended
   fi
-
 fi

src/travis/osx-x86_64-before-install.sh

@@ -1,78 +1,110 @@
 mypath=$HOME
 whereibelong=$(pwd)
+cpucount=$(sysctl -n hw.ncpu)
+echo "This device has $cpucount CPUs for compiling..."
+
 #echo "Brew installing xz..."
 #brew install xz > /dev/null
 
 #brew upgrade
 
-cd ~/pybuild
+# prefer standard GNU tools like date over MacOS defaults
+export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
 
-#if [ ! -f python-$BUILD_PYTHON_VERSION-macosx10.9.pkg ]; then
-#  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
+cd ~
+
+#if [ ! -f python-$MIN_PYTHON_VERSION-macosx10.9.pkg ]; then
+#  wget --quiet https://www.python.org/ftp/python/$MIN_PYTHON_VERSION/python-$MIN_PYTHON_VERSION-macosx10.9.pkg
 #fi
-#sudo installer -pkg python-$BUILD_PYTHON_VERSION-macosx10.9.pkg -target /
+#sudo installer -pkg python-$MIN_PYTHON_VERSION-macosx10.9.pkg -target /
 
-brew install openssl@1.1
-brew upgrade python
+#brew install openssl@1.1
+#brew upgrade python
 
-export python=python3
-export pip=pip3
+#export python=python3
+#export pip=pip3
 
-echo "Python location:"
-which $python
+#echo "Python location:"
+#which $python
 
-# Compile latest OpenSSL
-#if [ ! -d openssl-$BUILD_OPENSSL_VERSION ]; then
-#  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
-#  echo "Extracting OpenSSL..."
-#  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-#fi
-#cd openssl-$BUILD_OPENSSL_VERSION
-#echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-#./config shared --prefix=$mypath/ssl
-#echo "Running make for OpenSSL..."
-#make -j$cpucount -s
-#echo "Running make install for OpenSSL..."
-#make install > /dev/null
-#export LD_LIBRARY_PATH=~/ssl/lib
-#cd ~/pybuild
+cd ~
 
-# Compile latest Python
-#if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
-#  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
-#  echo "Extracting Python..."
-#  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
-#fi
-#cd Python-$BUILD_PYTHON_VERSION
-#echo "Compiling Python $BUILD_PYTHON_VERSION..."
-#safe_flags="--with-openssl=$mypath/ssl --enable-shared --prefix=$mypath/python --with-ensurepip=upgrade"
-#unsafe_flags="--enable-optimizations --with-lto"
-#if [ ! -e Makefile ]; then
-#  ./configure $safe_flags $unsafe_flags > /dev/null
-#fi
-#make -j$cpucount -s
-#RESULT=$?
-#echo "Make Python exited with $RESULT"
-#if [ $RESULT != 0 ]; then
-#  echo "Trying Python make again without unsafe flags..."
-#  make clean
-#  ./configure $safe_flags > /dev/null
-#  make -j$cpucount -s
-#fi
-#echo "Installing Python..."
-#make install > /dev/null
-#cd ~
+export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+export openssl=~/ssl/bin/openssl
+export python=~/python/bin/python3
+export pip=~/python/bin/pip3
+SSLVER=$($openssl version)
+SSLRESULT=$?
+PYVER=$($python -V)
+PYRESULT=$?
+if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+  echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+  if [ $SSLRESULT -ne 0 ]; then
+    echo "sslresult -ne 0"
+  fi
+  if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+    echo "sslver not equal to..."
+  fi
+  if [ $PYRESULT -ne 0 ]; then
+    echo "pyresult -ne 0"
+  fi
+  if [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION" ]]; then
+    echo "pyver not equal to..."
+  fi
 
-#export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
-#python=~/python/bin/python3
-#pip=~/python/bin/pip3
+  # Start clean
+  rm -rf python
+  rm -rf ssl
+  mkdir python
+  mkdir ssl
+  
+  # Compile latest OpenSSL
+  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  echo "Extracting OpenSSL..."
+  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  cd openssl-$BUILD_OPENSSL_VERSION
+  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+  ./config shared --prefix=$HOME/ssl
+  echo "Running make for OpenSSL..."
+  make -j$cpucount -s
+  echo "Running make install for OpenSSL..."
+  make install > /dev/null
+  cd ~
+
+  # Compile latest Python
+  echo "Downloading Python $BUILD_PYTHON_VERSION..."
+  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+  echo "Extracting Python..."
+  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+  cd Python-$BUILD_PYTHON_VERSION
+  echo "Compiling Python $BUILD_PYTHON_VERSION..."
+  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+  unsafe_flags="--enable-optimizations --with-lto"
+  if [ ! -e Makefile ]; then
+    echo "running configure with safe and unsafe"
+    ./configure $safe_flags $unsafe_flags > /dev/null
+  fi
+  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+  RESULT=$?
+  echo "First make exited with $RESULT"
+  if [ $RESULT != 0 ]; then
+    echo "Trying Python compile again without unsafe flags..."
+    make clean
+    ./configure $safe_flags > /dev/null
+    make -j$cpucount -s
+    echo "Sticking with safe Python for now..."
+  fi
+  echo "Installing Python..."
+  make install > /dev/null
+  cd ~
+fi
 
 $python -V
 
 cd $whereibelong
 
-export PATH=/usr/local/opt/python/libexec/bin:$PATH
+#export PATH=/usr/local/opt/python/libexec/bin:$PATH
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade pyinstaller
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 

src/travis/set_printer_csv_filter.sh

@@ -1 +0,0 @@
-export GAM_CSV_ROW_FILTER='{"type": "regex:^GOOGLE"}'

src/travis/windows-x86-before-install.sh

@@ -1,7 +1,7 @@
 echo "Installing Net-Framework-Core..."
 export mypath=$(pwd)
 until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~/pybuild
+cd ~
 export exefile=Win32OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
 if [ ! -e $exefile ]; then
   echo "Downloading $exefile..."
@@ -9,33 +9,31 @@ if [ ! -e $exefile ]; then
 fi
 echo "Installing $exefile..."
 powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
-cinst -y --forcex86 python3
+export python_file=python-$BUILD_PYTHON_VERSION.exe
+wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
+powershell ".\\${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python"
 until cinst -y wixtoolset; do echo "trying again..."; done
-echo "OpenSSL dlls..."
-ls -alRF /c/ssl
-echo "c drive"
-ls -al /c/
-echo "Python dlls..."
-ls -al /c/Python37/DLLs
-until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
-export PATH=$PATH:/c/Python37/scripts
+until cp -v /c/ssl/libcrypto-1_1.dll /c/python/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
+until cp -v /c/ssl/libssl-1_1.dll /c/python/DLLs/libssl-1_1.dll; do echo "trying again..."; done
+export PATH=$PATH:/c/python/scripts
 cd $mypath
-export python=/c/Python37/python.exe
-export pip=pip
+export python=/c/python/python.exe
+export pip=/c/python/scripts/pip.exe
+until [ -f $python ]; do :; done
+until [ -f $pip ]; do :; done
 
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-
 #$pip install --upgrade pyinstaller
 # Install PyInstaller from source and build bootloader
 # to try and avoid getting flagged as malware since
 # lots of malware uses PyInstaller default bootloader
 # https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
 echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
-tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
-cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+tar xf develop.tar.gz
+cd pyinstaller-develop/bootloader
 echo "bootloader before:"
 md5sum ../PyInstaller/bootloader/Windows-32bit/*
 $python ./waf all --target-arch=32bit

src/travis/windows-x86_64-before-install.sh

@@ -1,7 +1,7 @@
 echo "Installing Net-Framework-Core..."
 export mypath=$(pwd)
 until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~/pybuild
+cd ~
 export exefile=Win64OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
 if [ ! -e $exefile ]; then
   echo "Downloading $exefile..."
@@ -9,14 +9,20 @@ if [ ! -e $exefile ]; then
 fi
 echo "Installing $exefile..."
 powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
-cinst -y python3
+export python_file=python-$BUILD_PYTHON_VERSION-amd64.exe
+wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
+powershell ".\\${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python"
 until cinst -y wixtoolset; do echo "trying again..."; done
-until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/Python37/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
-until cp -v /c/ssl/libssl-1_1-x64.dll /c/Python37/DLLs/libssl-1_1.dll; do echo "trying again..."; done
-export PATH=$PATH:/c/Python37/scripts
+until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/python/DLLs/; do echo "trying libcrypto copy again..."; done
+cp -v /c/python/DLLs/libcrypto-1_1-x64.dll /c/python/DLLs/libcrypto-1_1.dll
+until cp -v /c/ssl/libssl-1_1-x64.dll /c/python/DLLs/; do echo "trying libssl copy again..."; done
+cp -v /c/python/DLLs/libssl-1_1-x64.dll /c/python/DLLs/libssl-1_1.dll
+export PATH=$PATH:/c/python/scripts
 cd $mypath
-export python=/c/Python37/python.exe
-export pip=pip
+export python=/c/python/python.exe
+export pip=/c/python/scripts/pip.exe
+until [ -f $python ]; do :; done
+until [ -f $pip ]; do :; done
 
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
@@ -27,9 +33,9 @@ $pip install --upgrade -r src/requirements.txt
 # lots of malware uses PyInstaller default bootloader
 # https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
 echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
-tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
-cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+tar xf develop.tar.gz
+cd pyinstaller-develop/bootloader
 echo "bootloader before:"
 md5sum ../PyInstaller/bootloader/Windows-64bit/*
 $python ./waf all --target-arch=64bit
@@ -38,6 +44,5 @@ md5sum ../PyInstaller/bootloader/Windows-64bit/*
 echo "PATH: $PATH"
 cd ..
 $python setup.py install
-echo "cd to $mypath..."
-until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
+echo "cd to $mypath"
 cd $mypath

src/travis/windows-x86_64-install.sh

@@ -1,7 +1,9 @@
 cd src
+echo "compiling GAM with pyinstaller..."
 pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
 export gam="gam/gam"
 export gampath=$(readlink -e gam)
+echo "running compiled GAM..."
 $gam version
 export GAMVERSION=`$gam version simple`
 rm gam/lastupdatecheck.txt

src/utils.py

@@ -1,16 +1,12 @@
 import datetime
 import re
 import sys
+from hashlib import md5
 from html.entities import name2codepoint
 from html.parser import HTMLParser
 
-ONE_KILO_BYTES = 1000
-ONE_MEGA_BYTES = 1000000
-ONE_GIGA_BYTES = 1000000000
-
-
-def convertUTF8(data):
-  return data
+from var import *
+import fileutils
 
 
 class _DeHTMLParser(HTMLParser):
@@ -30,7 +26,7 @@ class _DeHTMLParser(HTMLParser):
     if cp:
       self.__text.append(chr(cp))
     else:
-      self.__text.append('&' + name)
+      self.__text.append('&'+name)
 
   def handle_starttag(self, tag, attrs):
     if tag == 'p':
@@ -40,22 +36,20 @@ class _DeHTMLParser(HTMLParser):
     elif tag == 'a':
       for attr in attrs:
         if attr[0] == 'href':
-          self.__text.append('({0}) '.format(attr[1]))
+          self.__text.append(f'({attr[1]}) ')
           break
     elif tag == 'div':
       if not attrs:
         self.__text.append('\n')
-    elif tag in ['http:', 'https']:
-      self.__text.append(' ({0}//{1}) '.format(tag, attrs[0][0]))
+    elif tag in {'http:', 'https'}:
+      self.__text.append(f' ({tag}//{attrs[0][0]}) ')
 
   def handle_startendtag(self, tag, attrs):
     if tag == 'br':
       self.__text.append('\n\n')
 
   def text(self):
-    return re.sub(r'\n{2}\n+', '\n\n',
-                  re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
-
+    return re.sub(r'\n{2}\n+', '\n\n', re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
 
 def dehtml(text):
   try:
@@ -68,10 +62,27 @@ def dehtml(text):
     print_exc(file=sys.stderr)
     return text
 
-
 def indentMultiLineText(message, n=0):
   return message.replace('\n', '\n{0}'.format(' ' * n)).rstrip()
 
+def flatten_json(structure, key='', path='', flattened=None, listLimit=None):
+  if flattened is None:
+    flattened = {}
+  if not isinstance(structure, (dict, list)):
+    flattened[((path + '.') if path else '') + key] = structure
+  elif isinstance(structure, list):
+    for i, item in enumerate(structure):
+      if listLimit and (i >= listLimit):
+        break
+      flatten_json(item, f'{i}', '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit)
+  else:
+    for new_key, value in list(structure.items()):
+      if new_key in ['kind', 'etag', '@type']:
+        continue
+      if value == NEVER_TIME:
+        value = 'Never'
+      flatten_json(value, new_key, '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit)
+  return flattened
 
 def formatTimestampYMD(timestamp):
   return datetime.datetime.fromtimestamp(int(timestamp)/1000).strftime('%Y-%m-%d')
@@ -82,21 +93,162 @@ def formatTimestampYMDHMS(timestamp):
 def formatTimestampYMDHMSF(timestamp):
   return str(datetime.datetime.fromtimestamp(int(timestamp)/1000))
 
-
 def formatFileSize(fileSize):
   if fileSize == 0:
     return '0kb'
   if fileSize < ONE_KILO_BYTES:
     return '1kb'
   if fileSize < ONE_MEGA_BYTES:
-    return '{0}kb'.format(fileSize // ONE_KILO_BYTES)
+    return f'{fileSize // ONE_KILO_BYTES}kb'
   if fileSize < ONE_GIGA_BYTES:
-    return '{0}mb'.format(fileSize // ONE_MEGA_BYTES)
-  return '{0}gb'.format(fileSize // ONE_GIGA_BYTES)
-
+    return f'{fileSize // ONE_MEGA_BYTES}mb'
+  return f'{fileSize // ONE_GIGA_BYTES}gb'
 
 def formatMilliSeconds(millis):
   seconds, millis = divmod(millis, 1000)
   minutes, seconds = divmod(seconds, 60)
   hours, minutes = divmod(minutes, 60)
-  return '%02d:%02d:%02d' % (hours, minutes, seconds)
+  return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
+
+def get_string(i, item, optional=False, minLen=1, maxLen=None):
+  if i < len(sys.argv):
+    argstr = sys.argv[i]
+    if argstr:
+      if (len(argstr) >= minLen) and ((maxLen is None) or (len(argstr) <= maxLen)):
+        return argstr
+      controlflow.system_error_exit(2, f'expected <{integerLimits(minLen, maxLen, "string length")} for {item}>')
+    if optional or (minLen == 0):
+      return ''
+    controlflow.system_error_exit(2, f'expected a Non-empty <{item}>')
+  elif optional:
+    return ''
+  controlflow.system_error_exit(2, f'expected a <{item}>')
+
+def get_delta(argstr, pattern):
+  tg = pattern.match(argstr.lower())
+  if tg is None:
+    return None
+  sign = tg.group(1)
+  delta = int(tg.group(2))
+  unit = tg.group(3)
+  if unit == 'y':
+    deltaTime = datetime.timedelta(days=delta*365)
+  elif unit == 'w':
+    deltaTime = datetime.timedelta(weeks=delta)
+  elif unit == 'd':
+    deltaTime = datetime.timedelta(days=delta)
+  elif unit == 'h':
+    deltaTime = datetime.timedelta(hours=delta)
+  elif unit == 'm':
+    deltaTime = datetime.timedelta(minutes=delta)
+  if sign == '-':
+    return -deltaTime
+  return deltaTime
+
+def get_delta_date(argstr):
+  deltaDate = get_delta(argstr, DELTA_DATE_PATTERN)
+  if deltaDate is None:
+    controlflow.system_error_exit(2, f'expected a <{DELTA_DATE_FORMAT_REQUIRED}>; got {argstr}')
+  return deltaDate
+
+def get_delta_time(argstr):
+  deltaTime = get_delta(argstr, DELTA_TIME_PATTERN)
+  if deltaTime is None:
+    controlflow.system_error_exit(2, f'expected a <{DELTA_TIME_FORMAT_REQUIRED}>; got {argstr}')
+  return deltaTime
+
+def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
+  argstr = argstr.strip()
+  if argstr:
+    if argstr[0] in ['+', '-']:
+      today = datetime.date.today()
+      argstr = (datetime.datetime(today.year, today.month, today.day)+getDeltaDate(argstr)).strftime(YYYYMMDD_FORMAT)
+    try:
+      dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
+      if returnTimeStamp:
+        return time.mktime(dateTime.timetuple())*1000
+      if returnDateTime:
+        return dateTime
+      return argstr
+    except ValueError:
+      controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>; got {argstr}')
+  elif minLen == 0:
+    return ''
+  controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>')
+
+def get_time_or_delta_from_now(time_string):
+  """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+  time_string = time_string.strip().upper()
+  if time_string:
+    if time_string[0] not in ['+', '-']:
+      return time_string
+    return (datetime.datetime.utcnow() + get_delta_time(time_string)).isoformat() + 'Z'
+  controlflow.system_error_exit(2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+def get_row_filter_date_or_delta_from_now(date_string):
+  """Get an ISO 8601 date or a positive/negative delta applied to now.
+  Args:
+    date_string (string): The time or delta (e.g. '2017-09-01' or '-4y')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+  date_string = date_string.strip().upper()
+  if date_string:
+    if date_string[0] in ['+', '-']:
+      deltaDate = get_delta(date_string, DELTA_DATE_PATTERN)
+      if deltaDate is None:
+        return (False, DELTA_DATE_FORMAT_REQUIRED)
+      today = datetime.date.today()
+      return (True, (datetime.datetime(today.year, today.month, today.day)+deltaDate).isoformat()+'Z')
+    try:
+      deltaDate = dateutil.parser.parse(date_string, ignoretz=True)
+      return (True, datetime.datetime(deltaDate.year, deltaDate.month, deltaDate.day).isoformat()+'Z')
+    except ValueError:
+      pass
+  return (False, YYYYMMDD_FORMAT_REQUIRED)
+
+def get_row_filter_time_or_delta_from_now(time_string):
+  """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  Exits:
+    2: Not a valid delta.
+  """
+  time_string = time_string.strip().upper()
+  if time_string:
+    if time_string[0] in ['+', '-']:
+      deltaTime = get_delta(time_string, DELTA_TIME_PATTERN)
+      if deltaTime is None:
+        return (False, DELTA_TIME_FORMAT_REQUIRED)
+      return (True, (datetime.datetime.utcnow()+deltaTime).isoformat()+'Z')
+    try:
+      deltaTime = dateutil.parser.parse(time_string, ignoretz=True)
+      return (True, deltaTime.isoformat()+'Z')
+    except ValueError:
+      pass
+  return (False, YYYYMMDDTHHMMSS_FORMAT_REQUIRED)
+
+def get_date_zero_time_or_full_time(time_string):
+  time_string = time_string.strip()
+  if time_string:
+    if YYYYMMDD_PATTERN.match(time_string):
+      return get_yyyymmdd(time_string)+'T00:00:00.000Z'
+    return get_time_or_delta_from_now(time_string)
+  controlflow.system_error_exit(2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+def md5_matches_file(local_file, expected_md5, exitOnError):
+  f = fileutils.open_file(local_file, 'rb')
+  hash_md5 = md5()
+  for chunk in iter(lambda: f.read(4096), b""):
+    hash_md5.update(chunk)
+  actual_hash = hash_md5.hexdigest()
+  if exitOnError and actual_hash != expected_md5:
+    controlflow.system_error_exit(6, f'actual hash was {actual_hash}. Exiting on corrupt file.')
+  return actual_hash == expected_md5

src/var.py

@@ -6,21 +6,19 @@ import platform
 import re
 
 gam_author = 'Jay Lee <jay0lee@gmail.com>'
-gam_version = '4.95'
+gam_version = '5.01'
 gam_license = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
-GAM_INFO = 'GAM {0} - {1} / {2} / Python {3}.{4}.{5} {6} / {7} {8} /'.format(gam_version, GAM_URL,
-                                                                             gam_author,
-                                                                             sys.version_info[0], sys.version_info[1],
-                                                                             sys.version_info[2], sys.version_info[3],
-                                                                             platform.platform(), platform.machine())
+GAM_INFO = (f'GAM {gam_version} - {GAM_URL} / {gam_author} / '
+           f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
+           f'{platform.platform()} {platform.machine()}')
 
 GAM_RELEASES = 'https://github.com/jay0lee/GAM/releases'
 GAM_WIKI = 'https://github.com/jay0lee/GAM/wiki'
 GAM_ALL_RELEASES = 'https://api.github.com/repos/jay0lee/GAM/releases'
 GAM_LATEST_RELEASE = GAM_ALL_RELEASES+'/latest'
-GAM_PROJECT_FILEPATH = 'https://raw.githubusercontent.com/jay0lee/GAM/master/'
+GAM_PROJECT_FILEPATH = 'https://raw.githubusercontent.com/jay0lee/GAM/master/src/'
 
 true_values = ['on', 'yes', 'enabled', 'true', '1']
 false_values = ['off', 'no', 'disabled', 'false', '0']
@@ -49,7 +47,7 @@ SKUS = {
   '1010310002': {
     'product': '101031', 'aliases': ['gsefe', 'e4e', 'gsuiteenterpriseeducation'], 'displayName': 'G Suite Enterprise for Education'},
   '1010310003': {
-    'product': '101031', 'aliases': ['gsefes', 'e4es', 'gsuiteenterpriseeducationstudent'], 'displayName': 'G Suite Enterprise for Education Student'},
+    'product': '101031', 'aliases': ['gsefes', 'e4es', 'gsuiteenterpriseeducationstudent'], 'displayName': 'G Suite Enterprise for Education (Student)'},
   '1010330003': {
     'product': '101033', 'aliases': ['gvstarter', 'voicestarter', 'googlevoicestarter'], 'displayName': 'Google Voice Starter'},
   '1010330004': {
@@ -107,7 +105,6 @@ SKUS = {
 PRODUCTID_NAME_MAPPINGS = {
   '101001': 'Cloud Identity Free',
   '101005': 'Cloud Identity Premium',
-  '101006': 'Drive Enterprise',
   '101031': 'G Suite Enterprise for Education',
   '101033': 'Google Voice',
   '101034': 'G Suite Archived',
@@ -139,17 +136,22 @@ API_VER_MAPPING = {
   'calendar': 'v3',
   'classroom': 'v1',
   'cloudprint': 'v2',
+  'cloudresourcemanager': 'v2',
+  'cloudresourcemanagerv1': 'v1',
   'datatransfer': 'datatransfer_v1',
   'directory': 'directory_v1',
   'drive': 'v2',
   'drive3': 'v3',
   'gmail': 'v1',
   'groupssettings': 'v1',
+  'iam': 'v1',
+  'iap': 'v1',
   'licensing': 'v1',
   'oauth2': 'v2',
   'pubsub': 'v1',
   'reports': 'reports_v1',
   'reseller': 'v1',
+  'servicemanagement': 'v1',
   'sheets': 'v4',
   'siteVerification': 'v1',
   'storage': 'v1',
@@ -450,6 +452,11 @@ DOCUMENT_FORMATS_MAP = {
                  {'mime': 'application/vnd.oasis.opendocument.text', 'ext': '.odt'}],
   }
 
+REFRESH_PERM_ERRORS = [
+  'invalid_grant: reauth related error (rapt_required)', # no way to reauth today
+  'invalid_grant: Token has been expired or revoked.',
+  ]
+
 DNS_ERROR_CODES_MAP = {
   1: 'DNS Query Format Error',
   2: 'Server failed to complete the DNS request',
@@ -620,8 +627,6 @@ CROS_SCALAR_PROPERTY_PRINT_ORDER = [
   'manufactureDate',
   'supportEndDate',
   'autoUpdateExpiration',
-  'guessedAUEDate',
-  'guessedAUEModel',
   'tpmVersionInfo',
   'willAutoRenew',
   ]
@@ -935,73 +940,18 @@ CLEAR_NONE_ARGUMENT = ['clear', 'none',]
 #
 MESSAGE_API_ACCESS_CONFIG = 'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access'
 MESSAGE_API_ACCESS_DENIED = 'API access Denied.\n\nPlease make sure the Client ID: {0} is authorized for the API Scope(s): {1}'
-MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = '\nGo to the following link in your browser:\n\n\t{url}\n'
-MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
 MESSAGE_GAM_EXITING_FOR_UPDATE = 'GAM is now exiting so that you can overwrite this old version with the latest release'
 MESSAGE_GAM_OUT_OF_MEMORY = 'GAM has run out of memory. If this is a large G Suite instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.'
 MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = 'Header "{0}" not found in CSV headers of "{1}".'
 MESSAGE_HIT_CONTROL_C_TO_UPDATE = '\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.'
 MESSAGE_INVALID_JSON = 'The file {0} has an invalid format.'
-MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = '\nYour browser has been opened to visit:\n\n\t{url}\n\nIf your browser is on a different machine then press CTRL+C and create a file called nobrowser.txt in the same folder as GAM.\n'
-MESSAGE_LOCAL_SERVER_SUCCESS = 'The authentication flow has completed. You may close this browser window and return to GAM.'
 MESSAGE_NO_DISCOVERY_INFORMATION = 'No online discovery doc and {0} does not exist locally'
 MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = 'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb'
 MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = 'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.'
 MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}. Please make sure service is enabled for user and run\n\ngam user <user> check serviceaccount\n\nfor further instructions'
 MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON = 'Please run\n\ngam create project\ngam user <user> check serviceaccount\n\nto create and configure a service account.'
-MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from Google by %s'
-# oauth errors
-OAUTH2_TOKEN_ERRORS = [
-  'access_denied',
-  'access_denied: Requested client not authorized',
-  'internal_failure: Backend Error',
-  'internal_failure: None',
-  'invalid_grant',
-  'invalid_grant: Bad Request',
-  'invalid_grant: Invalid email or User ID',
-  'invalid_grant: Not a valid email',
-  'invalid_grant: Invalid JWT: No valid verifier found for issuer',
-  'invalid_request: Invalid impersonation prn email address',
-  'unauthorized_client: Client is unauthorized to retrieve access tokens using this method',
-  'unauthorized_client: Client is unauthorized to retrieve access tokens using this method, or client not authorized for any of the scopes requested',
-  'unauthorized_client: Unauthorized client or scope in request',
-  ]
-#
-# callGAPI throw reasons
-GAPI_ABORTED = 'aborted'
-GAPI_AUTH_ERROR = 'authError'
-GAPI_BACKEND_ERROR = 'backendError'
-GAPI_BAD_REQUEST = 'badRequest'
-GAPI_CONDITION_NOT_MET = 'conditionNotMet'
-GAPI_CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
-GAPI_DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
-GAPI_DOMAIN_NOT_FOUND = 'domainNotFound'
-GAPI_DUPLICATE = 'duplicate'
-GAPI_FAILED_PRECONDITION = 'failedPrecondition'
-GAPI_FORBIDDEN = 'forbidden'
-GAPI_GROUP_NOT_FOUND = 'groupNotFound'
-GAPI_INTERNAL_ERROR = 'internalError'
-GAPI_INVALID = 'invalid'
-GAPI_INVALID_ARGUMENT = 'invalidArgument'
-GAPI_INVALID_MEMBER = 'invalidMember'
-GAPI_MEMBER_NOT_FOUND = 'memberNotFound'
-GAPI_NOT_FOUND = 'notFound'
-GAPI_NOT_IMPLEMENTED = 'notImplemented'
-GAPI_PERMISSION_DENIED = 'permissionDenied'
-GAPI_QUOTA_EXCEEDED = 'quotaExceeded'
-GAPI_RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
-GAPI_RESOURCE_NOT_FOUND = 'resourceNotFound'
-GAPI_SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
-GAPI_SYSTEM_ERROR = 'systemError'
-GAPI_USER_NOT_FOUND = 'userNotFound'
-GAPI_USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
-#
-GAPI_DEFAULT_RETRY_REASONS = [GAPI_QUOTA_EXCEEDED, GAPI_RATE_LIMIT_EXCEEDED, GAPI_USER_RATE_LIMIT_EXCEEDED, GAPI_BACKEND_ERROR, GAPI_INTERNAL_ERROR]
-GAPI_GMAIL_THROW_REASONS = [GAPI_SERVICE_NOT_AVAILABLE]
-GAPI_GROUP_GET_THROW_REASONS = [GAPI_GROUP_NOT_FOUND, GAPI_DOMAIN_NOT_FOUND, GAPI_DOMAIN_CANNOT_USE_APIS, GAPI_FORBIDDEN, GAPI_BAD_REQUEST]
-GAPI_GROUP_GET_RETRY_REASONS = [GAPI_INVALID, GAPI_SYSTEM_ERROR]
-GAPI_MEMBERS_THROW_REASONS = [GAPI_GROUP_NOT_FOUND, GAPI_DOMAIN_NOT_FOUND, GAPI_DOMAIN_CANNOT_USE_APIS, GAPI_INVALID, GAPI_FORBIDDEN]
-GAPI_MEMBERS_RETRY_REASONS = [GAPI_SYSTEM_ERROR]
+MESSAGE_UPDATE_GAM_TO_64BIT = "You're running a 32-bit version of GAM on a 64-bit version of Windows, upgrade to a windows-x86_64 version of GAM"
+MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from %s by %s'
 
 USER_ADDRESS_TYPES = ['home', 'work', 'other']
 USER_EMAIL_TYPES = ['home', 'work', 'other']
@@ -1228,3 +1178,22 @@ MAX_RESULTS_API_EXCEPTIONS = {
   'directory.chromeosdevices.list': 200,
   'drive.files.list': 1000,
   }
+
+ONE_KILO_BYTES = 1000
+ONE_MEGA_BYTES = 1000000
+ONE_GIGA_BYTES = 1000000000
+
+DELTA_DATE_PATTERN = re.compile(r'^([+-])(\d+)([dwy])$')
+DELTA_DATE_FORMAT_REQUIRED = '(+|-)<Number>(d|w|y)'
+
+DELTA_TIME_PATTERN = re.compile(r'^([+-])(\d+)([mhdwy])$')
+DELTA_TIME_FORMAT_REQUIRED = '(+|-)<Number>(m|h|d|w|y)'
+
+YYYYMMDD_FORMAT = '%Y-%m-%d'
+YYYYMMDD_FORMAT_REQUIRED = 'yyyy-mm-dd'
+
+YYYYMMDDTHHMMSS_FORMAT_REQUIRED = 'yyyy-mm-ddThh:mm:ss[.fff](Z|(+|-(hh:mm)))'
+
+YYYYMMDD_PATTERN = re.compile(r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$')
+
+UID_PATTERN = re.compile(r'u?id: ?(.+)', re.IGNORECASE)

src/windows-gam.spec

@@ -1,9 +1,13 @@
 # -*- mode: python -*-
+import sys
+
+sys.modules['FixTk'] = None
+
 a = Analysis(['gam.py'],
              pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
              hiddenimports=[],
              hookspath=None,
-             excludes=['_tkinter'],
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
              runtime_hooks=None)
 for d in a.datas:
     if 'pyconfig' in d[0]: