re-add filter_secrets for arm64

- Programmatically set Consent (Branding page) saving a user step
- record created_by in client_secrets.json for easier backtracking
- use local project-apis.txt if it exists

.travis.yml

@@ -1,11 +1,12 @@
 if: tag IS blank
+os: linux
+language: python
 
 env:
   global:
-    - BUILD_PYTHON_VERSION=3.7.4
-    - BUILD_OPENSSL_VERSION=1.1.1c
+    - BUILD_PYTHON_VERSION=3.8.1
+    - BUILD_OPENSSL_VERSION=1.1.1d
     - PATCHELF_VERSION=0.9
-    - MUSL_VERSION=1.1.22
     - PYINSTALLER_VERSION=3.5
     - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
     - secure: "J9380tGLOZWa7dSH1y5Il8T5JQpN6ad81gI6VR1HIU0svpRdjgikyDA7ca2MKYDUYYY9yVSkTV6gCl6iIU/9+SKaYugpP+tkvdGYkC2moJdcTgYM/WOnIK9ExQ3BPhN1neGxJjPTwKo1ft27mtZ2I5vuCiBwIcnKWLnKPyW3PD+mWpfqiLuEzkHoAh6G3jC4qbcCrZDeX/knE+PzqESUEi+8k1G8gYcSDWujba9ypSsqZ8T/MXagGla6l7y2Rz+/KZTJmFHwKAA10V+xPLVqxoiqi4ar66yUqy0BamwRXPcseI+ns3Q+4lUpMqVQ5GlRy7LF1xC8myjmcAexXk0F9hg+CMzewKI8UgmQH/ZJvQZEh8s6mW26+CqA4d3zMQkWaR0WtEtpiuH7AGHCflIqvEQ6UiG7ia3B8iZfW2wl0j/kqx4OuHkS3r0pWKVVIIvCj9Ow2BHP7SpiV1AcUGsVxzwbgTh67fitna3Z3c6Uj8ccQlNr7ZIt1az6Wf3w5njijkLOiBpQSLKunTTCTSge/JzBTKUcie3RE9vzirl58gUxAt36nDtPWnory+RttMZrOkBVbTeSxp+IUe8pNwLFPHABsafXsjkfzBOtFmm+0ZXWt2Rlog5NvlemJfQUWDlsL4g+BSakzN+4sIPKzSauWDHyaEeULY7Uprkil6c5zwo="
@@ -23,7 +24,7 @@ env:
     - secure: "Is2Lv/rxKKrXnxFns9KQYseD02tjY9qbgSteVtJavG1cLJDvkTwb6X+Thvgo4cxk5fQPiXScrQaYzHjVVuNleD+dyD1HC/8CU2Xq+tjBhPjdcccHFSbk06DpcETmLGRyMORXG9JkYlgXHLLXtu/9icppWEHgra+zvchVL2YDhofYne8FMNBb/lq0AAC2wgzAS33tW1+57HaYnzl0hf6+Q6lwqoH2/aTfGMRFDgyJ0HK+5IVfLnQJ+OuGFSrj8/0FWSggR5+EXDIddDgovFgaCghMjHYp21bzn0eIAJtuNFFultwk/UC1lT9joXTKEzgLTAh+w13yz1T3x9rNuv6FDKCotBIS/ZDtPmgvyZ8xvB4SzyULnTRSVn7YvspKR6PAO/qxGNudUD5H8tRKer4qKnKjHzSUcVBlRHb4yE6FqqY1Z9RLEcomWO43nJwb6saNHR9BYedyi0gA+EbA+P259QFClW1dWEQ2LQhDa+0VRssOqZ0BQblPFyz+e5Vc9kfAMbOuoss8fjkiYv+twXv7nT27xrVT5okfKDSiy5opZD6d36N5FibZPYiMrVx00YZdkFB+5EqQuJ7lqKUMkZJTeApLzj+h+/4aAOWd3paj5ghv7m+9ReohsNKFHyjaSy97RhMAZjzqgMMdD8rjUSKDhvNKvvQECWHlaXwL129GB0s="
     - secure: "l7vWxfcu1RgXbStq76Mzz2I5Iu4e31729OyLYqulXZzft3wO+idvgQKy/JSwajiKgOxlpBuI0wrncgIUYshcRvE4yB0y9+QIMDTegJzTADtRSUyVNCIZfTgvtOvzrlW0iCdVsxLBtYcJWJVPdjF2q59ED1jahd1AuJJqX5e61gfr0eZ9+cNiHbX1u3VpmGchFNWQF4KvebE4WKs5xWEds+AtbTODdQq3H6kKQK3fTVJnbz6WMO+bEWgWI7orfSE20lku/3Q3eMLhNOcPwH8WnUoTTvDWol5Cq5NfPhkKF5aV7kIbNXkxswM7yBPAPumBiXdM1BHpfd4+0YQ/fqRtnqxw85HEYpT8dRewHimCl4IccgGCR+G0tK8RNleKL28GWrz86gRVpXMEhOU3ILwb8as3SdQWLwhXXy5uKGJmsdrCNAH4/8eu3XczO5VN2wOo7narYuBgGcl2eLW9TOLyNVKFKxbQnDLBiOybLNoOV42DCRYt7v3Eknkv1Z0dmX6n23q1z9if9kkfAFgRQWsTbNZyWeIwWuX8b2a0Zq0znS3JflSKxzqS7RHADfBOJEVM86AiGkw0XkR4o31yDrY+zOrkJ3GxfV/HpqG7LzCd9tFeexanneDCE1FJhCkDjpnc0Mdyx+1gVf+u2MkgrU0BbCf2EESBAlC/FTRvxTmk/6s="
     - secure: "bvtQ348bxKwTtB8X0zMxeTsM0jEJozbS6/rzH/88Fk90a+KO8SdXen0Kj9/LahV4duMn2xTTRmxMCVj424FbcVTgBkJpIO7btqTcNASORkmK+9wGTK6Bgb9R5sHLbVrbrMYJzsMQR0MWxE8ibPLlyom+ssUIqr7HAjnRyYSDiKChGhgBfdE/0G9OE/DuQaU/ZkTuEYfc4527QNLJ8Bt1aLDdCHJLxzCojNaTHB215Tz7dmpnJjWa45BXxkMwLTpxIJuY7wA7K+2UBJfLvZv5QiPYyeUlosV7FwhCN9e90+dc2x8HPJbwe/Ysv5dm8/FfNTgTe6IkKw1z9kev9/W1tYCtqMzn9GnlIH2000ZUomhHKbc4UUl8sskF2jGr14rSz7pfaTyvXNvK7nKLKM7mZAO/cAaQvZoGEx0s6B7a7YX38NmJWcM0UUD603n4G9uVZNoxjyr8HWC6pho8MY8/u5aFKMPd+C8DaDWeSzNJDZRyi62v+t2iyyZuQUtVXtnDpv8GQW4tYFIJCkkrTm5VNNgD2x3WRsEr3LsBa3BonOmi3a1VrdIpyidDzBEWFUR59a/C0lU5J2UX5W1vu/Pnr9/8p898sDvvDbluz47OR9xoVOjtmGdt/ENJa4EOY1q/eRIN6zrt/cwwjIKgTcYOVk7DFDMXQuYlDeEFuHSJ5Xs="
-    - secure: "I05ZVphCT2Alu4najchCjDLIgflsdSCR/Z20OkMbcESprwJB5h7nb64kuVkqPQT7GeMcXMnK1xahx0KvE1Gn8OqOlDPt5WgnvcYERnVOKEJabmRXws8nM20AehwAzRoc6uDhaTnmwHRuliPSeuxiwY2Rkyj5hdyvEMzaOSY+qoIOYtDYQu0MM1XIC3cAjSbEfEGuqnKcN0p9VG0l7WscEUtcKPSFBShOlJZWDiHy7lisgO6OEzvAeDpZU/1ZdljXBl0RoDX8l4LqH4SzQqE68kK7qqIhZNmx4hkrh7qtx8tVqVKy/R3IUYhdxrJ84eJxrex6RQb9SgJQRGwi/3+rkKxVS41KiQ4quJHJhnqd2A1+5rDMN/fsjbjtM1I3VIl6w4oaT1C1QtwSjQOvuZCKDLIszVutW0PuGHveNyjwqQSm2PboOkLTg0dNrFIflT2hGMynv1s19WzGY8qQDP2UgODNR4EQVBy0PRk43ne5W/AFtzY1wUUUCBfKi7WYi4uURG/1Xu456GEElbIOwx9F1nC0s3z6ZFVJ3bVbmFxUlHtCyL5wJ3fiVHkAoPSygVARvv2UhHmXvJXXjxBMOvsnZZbpjvcbTeZoguffxMZEMNhuI4W22WjLdVTLxi9uh1VTdm8vkfAeNjMPw53gwPaSz7Jv6A82LFqpmbwAHxNAEy8="
+    - secure: "VWY3aJGMFB+E5ObJpUUXNQrnkEpA1pbfey8Z0pe9X/n5ubmDBIAfCPP7hqT/lAF/1hj5+HA6oWEa2FzeJZOL9q8cuymHzkehiX9n3f6GktirMtEoG3Hk/CiW/ZxKf9KyBNRVsqsOiXvfzMcMSU6ZrSV+4ZHIVl0zO5cehSaaiTTqVU5K474N5TCc8SAvpSRpxjZVRXy8rWF7WMaG4Xm6aa/IELccjJL8s6sahk3wT++5pzewxv4XBWf4bNA7+MNLpnzKmm2S5T/uohsw00F/j82g9xFT3qYm6vgaR4ql0aNvxd9gZ/6vdVpqBIUZmcFi9yTSFSsSSH2cNGFk3IU9Ecdh1Psi3B5pTpYzy7BbLWyKo70wBkOoY8qCwJ/wtj+2TcfR72qR8ryXgkbIoXxe04VkPqRNEezk3U+UxUvLLDEWthxjuCRCydT8FaCfYm9N+l5Un8TNmY4W9mOSu5IXKJ7oSQDrJG65Z2uZKQrWm0ToOo1nNLFclFuM+K8JWNOKysfh451LHdbmA0rkoaLUat2ttDBtQpbw3h+Hwn3SW6BfELg6WDJXHyAPNRMPSc2Grk2o7mFfTMJhHQG4U+k2bL+AHJU0nPWzuEJyMraquqKOCcHz56A+6j1PN3wpLbIC18nNPN9Q3TDPiBmwzwEg+zLu0DKTu3Oe4IRKRNTuocI="
     - secure: "BSZK3v2RYsNKv/8lBZ/dKagYF+znGXIeDY271A2nCS8DXx+y0octwI5RT08+UqdD1AvhPOPjdnSXG+M2NOSLlfBoXF/0t9S39+qy3EhGgxYAcWcVDXnrOrHEOALg1cQkCA2D/CG+eeG/36SmAn6FINTLaAOilxWGGCeL0KOB5mqPPnPogaYhowIwDkj0Hfsfrw8011XzgbufNr15GIYJ+nP7f41NcgIbnNkqXY6Q1jiWs1DuKXNDr5WGPqztHtVf7RtcpVmShzptAcAMdRqCGKF7atCZhXPbd69j3qf7F533/tCSCIrszR/Wp4BKlJfVCRCk1oiEcfRvprJqeFJ+0WJCVzyD7hXfnkBg2Tb5PmzvxOFM+hsqRu4mY3UdjqXukzAe5O0O6Uo8sqzqQUIjUooRnNQ4GdPd+7wMbyBZn4PJaW6YTi/7zg7mAegqot6uyEGGpWb6iFYrf6DX75GUfTciNktv+ez0AeqYFwNBLgcOsmaq+3V+aR+dIxsxIzC/i5GCTHvMYOIp6Q2tPCuIug0tX6uxm++vMOoMLK+vG1fVQ0Cd8m6yQIWEXUu0VBs1MTlJD+vX6LsK8CGo0Dt3ZS3JmC4TGGo9CBSEkeDvnKvQeX6x3NBIFWvgt+Hjxh4S9U+vW+AaUXpV9k+NEhesC/8Ys0UGbGYsTDzHdx9TMBY="
     - secure: "sskZYJ/+xmHh5GNvw3QRf10+sBGOjlub29jOTxjXIRYUyZfkhjsF8CR9NP59CBFuqgN716Mj1uEVEcx0aaepr/zWweQSnqa9lZgFD9nDYALVNh1b4oyqfhYG1sw57ZHdBsJOBF4mKBnahly/QQj11Ya25GyIS2IewwCWVNtCOJYietSssG9l0+qc+RPG5Ub/lEKA3VRrtbUuApcIYszt33DumgZ4wzkzICl1SuOy8V15vHVlkBqASJheEDa4Hia+eAMo6e7OCE5KWjl7K2MxTvtszwFi7lgMCyPCOy6DkaULACfBnQeloh42B2Qhd/ta04vuRKg6y4fKSrnegpeOAa9aGxF1ufvG9IvPsRTsu4+w63b9xsQUN9MyDgxcJe0YeUHPgPYL6mqAjCIKvL62LlIyrQ8IAteoh3MC+4Xb8crXaLGINTcLwYvLwxsHMuC/58hdQ23I4DqnyZGAS3L1IhpX4QvYN6xc7H+ptaiQttweoH5VpMAduSrmSnNmvLOc4g2PvbRES6D95moQ7qk7iX6vpu+PevL5HtQCbB8SFyFLTYWsqZaG+4NrjpIi2hLzUT35meHwrvxG/MsfeqOVlnBfa93VnX75vxOkRFNY56sOtrTJaiVZ+rQUdszAZz3KGPbyrSlz3KxV+ZKxZH6/0oFteAJttXFjWQdICnKGDSo="
     - secure: "sdzU5bPs1w7Nzf3F5Gtk/iq2Kfq3zfLQNGcehLZp1gJXzNf7F6HZLOn6GaXQ/5RVlhqR5nOmzMpVQs88rZv+6PE6YqGMugTxHIQeNmXtGnuEDJSBvGT9Ok8ENxcwKwL93g9SCd9P8mTspxklOsW2Bk3No2+Zlc/aQguBcX44TwYF42KuBU4O7oS6pUg8NjnuQp2zTyhp0ouzyAudatPyu1BLci/3lbx+MehutQw1y4Om6g8tfwf950UONZQdqq9MBluu7yYb1oHkdC1J4qgwdCZkJslWIwQHCH5UE4AW9iVG0qVrLpzBGtV27Kfy/Vf8r2gMYzbiur2h2+zzWSDm8/bk8YLn7u1FBpjGJdJ0pX0ZrZr+hMV2vH3e54NvA5WyRU7tw8mZ1PoDYd/FM5KXIYbscSbSRCqTsbPgyVIHuoOWXeeSe+/Ef1ifZv3HHf6ARfUIWupKfAipxChc7QUMI/HEQ9QPsqgBaooZD9chGsWAgv+8tFxdteqkx4Yh+AuZp1rVykB/9vAamUBebQxy0oeGm65j6X1rksfjAPkfeDYB7L4Ruy7tUwPtvYrAHPoWrf9O0g/qDRsw0vdqp42CjszpIxhuPhVRDx0i0wquqw2LnIU3ejRv2R8d1SecVKBBcVsWLFvc9iR4rNIi5JnxITRtiwL1Xv3Vcgx7WwxExtE="
@@ -33,30 +34,49 @@ cache:
     - $HOME/.cache/pip
     - $HOME/python
     - $HOME/ssl
-    - $HOME/pybuild
 
-matrix:
+jobs:
   include:
-#    - os: linux
-#      name: "Linux 64-bit Bionic"
-#      dist: bionic
-#      language: bash
-#      env:
-#         - GAMOS=linux
-#         - PLATFORM=x86_64
-#         - VMTYPE=cache
+    - os: linux
+      name: "Linux 64-bit Bionic"
+      dist: bionic
+      language: shell
+      env:
+         - GAMOS=linux
+         - PLATFORM=x86_64
+         - VMTYPE=build
     - os: linux
       name: "Linux 64-bit Xenial"
       dist: xenial
-      language: bash
+      language: shell
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=build
+    - os: linux
+      dist: bionic
+      arch: arm64
+      name: "Linux ARM64 Bionic"
+      language: shell
+      filter_secrets: false
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
+    - os: linux
+      dist: xenial
+      arch: arm64
+      name: "Linux ARM64 Xenial"
+      language: shell
+      filter_secrets: false
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
     - os: linux
       name: "Linux 64-bit Trusty"
       dist: trusty
-      language: bash
+      language: shell
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
@@ -64,63 +84,74 @@ matrix:
     - os: linux
       name: "Linux 64-bit Precise"
       dist: precise
-      language: bash
+      language: shell
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=build
     - os: linux
-      name: "Linux 64-bit Xenial - Python 3.5 Source Testing"
-      dist: xenial
+      name: "Python 3.6 Source Testing"
+      dist: bionic
       language: python
-      python:
-        - "3.5"
+      python: 3.6
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=test
     - os: linux
-      name: "Linux 64-bit Xenial - Python 3.6 Source Testing"
-      dist: xenial
+      name: "Python 3.7 Source Testing"
+      dist: bionic
       language: python
-      python:
-        - "3.6"
+      python: 3.7
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=test
     - os: linux
-      name: "Linux 64-bit Xenial - Python 3.8-dev Source Testing"
-      dist: xenial
+      name: "Python nightly Source Testing"
+      dist: bionic
       language: python
-      python:
-        - "3.8-dev"
+      python: nightly
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=test
     - os: linux
-      name: "Linux 64-bit Xenial - Python nightly Source Testing"
+      name: "Python PyPi Source Testing"
       dist: xenial
       language: python
-      python:
-        - "nightly"
+      python: pypy3
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=test
     - os: osx
-      name: "MacOS 64-bit"
+      name: "MacOS 10.12"
       language: generic
       osx_image: xcode9.2
       env:
         - GAMOS=macos
         - PLATFORM=x86_64
         - VMTYPE=build
+    - os: osx
+      name: "MacOS 10.13"
+      language: generic
+      osx_image: xcode10.1
+      env:
+        - GAMOS=macos
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: osx
+      name: "MacOS 10.14"
+      language: generic
+      osx_image: xcode11.3
+      env:
+        - GAMOS=macos
+        - PLATFORM=x86_64
+        - VMTYPE=build
     - os: windows
       name: "Windows 64-bit"
       language: shell
-      filter_secrets: false
       env:
         - GAMOS=windows
         - PLATFORM=x86_64
@@ -128,7 +159,6 @@ matrix:
     - os: windows
       name: "Windows 32-bit"
       language: shell
-      filter_secrets: false
       env:
         - GAMOS=windows
         - PLATFORM=x86
@@ -141,12 +171,14 @@ install:
 - source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
 
 script:
+# Discover and run all Python unit tests. Buffer output so that it's not sent to the build log.
+- $python -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
 - $gam version extended
 - $gam version | grep travis # travis should be part of the path (not /tmp or such)
 # determine which Python version GAM is built with and ensure it's at least build version from above.
-- if [ "VMTYPE" == "build" ]; then vline=$(gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; tools/a_atleast_b.py $this_python $BUILD_PYTHON_VERSION; fi
+- if [ "$VMTYPE" == "build" ]; then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $BUILD_PYTHON_VERSION; fi
 # determine which OpenSSL version GAM is built with and ensure it's at least build version from above.
-- if [ "VMTYPE" == "build" ]; then vline=$(gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; tools/a_atleast_b.py $this_openssl $BUILD_OPENSSL_VERSION; fi
+- if [ "$VMTYPE" == "build" ]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $BUILD_OPENSSL_VERSION; fi
 - if [ "$VMTYPE" == "build" ]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
 - if [ "$VMTYPE" == "build" ]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
 - export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
@@ -170,13 +202,15 @@ script:
   for i in {01..20};
     do echo $newbase-bulkuser-$i >> sample.csv;
   done; fi
-- if [ "$e2e" = true ]; then $gam create user $newuser firstname Travis lastname $jid password random travis.jid $jid; fi
+- if [ "$e2e" = true ]; then $gam create user $newuser firstname Travis lastname $jid password random recoveryphone 12125121110 recoveryemail jay0lee@gmail.com travis.jid $jid; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "Travis test message"; fi
 - if [ "$e2e" = true ]; then $gam create group $newgroup name "Travis $jid group" description "This is a description" isarchived true; fi
 - if [ "$e2e" = true ]; then $gam user $newuser add license gsuitebusiness; fi
 - if [ "$e2e" = true ]; then $gam update group $newgroup add owner $gam_user; fi
 - if [ "$e2e" = true ]; then $gam update group $newgroup add member $newuser; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam create user ~~email~~ firstname "Travis Bulk" lastname ~~email~~ travis.jid $jid; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add license gsuitebusiness; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user $gam_user sendemail recipient ~~email~~@pdl.jaylee.us subject "test message $newbase" message "Travis test message"; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam update group $newgroup add member ~email; fi
@@ -186,6 +220,9 @@ script:
 - if [ "$e2e" = true ]; then $gam user $newuser show imap; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user $newuser delegate to ~email; fi
 - if [ "$e2e" = true ]; then $gam user $newuser show delegates; fi
+- if [ "$e2e" = true ]; then $gam user $newuser label "✔ unicode checkmark ✔"; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels > labels.txt; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user importemail subject "Travis import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user insertemail subject "Travis insert $newbase" file gam.py labels INBOX,UNREAD; fi # yep body is gam code
 - if [ "$e2e" = true ]; then $gam user $gam_user sendemail subject "Travis send $gam_user $newbase" file gam.py recipient admin@pdl.jaylee.us; fi
@@ -193,6 +230,7 @@ script:
 - if [ "$e2e" = true ]; then $gam users "$gam_user $newbase-bulkuser-01 $newbase-bulkuser-02 $newbase-bulkuser-03" delete messages query in:anywhere maxtodelete 99999 doit; fi
 - if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-04 $newbase-bulkuser-05 $newbase-bulkuser-06" trash messages query in:anywhere maxtotrash 99999 doit; fi
 - if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-07 $newbase-bulkuser-08 $newbase-bulkuser-09" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit; fi
+- if [ "$e2e" = true ]; then $gam user $newuser delete label --ALL_LABELS--; fi
 - if [ "$e2e" = true ]; then $gam create feature name Whiteboard-$newbase; fi
 - if [ "$e2e" = true ]; then $gam create feature name VC-$newbase; fi
 - if [ "$e2e" = true ]; then $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."; fi
@@ -227,20 +265,23 @@ script:
 - if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
 - if [ "$e2e" = true ]; then $gam delete user $newuser; fi
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
+- if [ "$e2e" = true ]; then $gam print mobile; fi
+- if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$TRAVIS_OS_NAME" != "windows" ]; then bash <(curl -s -S -L https://git.io/install-gam) -l; fi
 
 before_deploy:
 - export TRAVIS_TAG="preview"
+- unset LD_LIBRARY_PATH
 
 deploy:
   provider: releases
-  api_key:
+  token:
     secure: bzambMcQwyv/o5c5GrKGCsZHgE5R85tg8sNFvPfpISz3+uosCjnBXas7wvCKzT75XUFi2ztfbYak6HdKf4sGnNHk0saEicB3slH+ghPyZbYzp76yvvduhFO2nWW3/F01tL+Yfqqt4/q8wFaWGjrC5km+6GLVyB4lWA/Uyu49qKnz02uSwyhBD/VFbO7DOQ65a1iWk9HngyMsu0Oi7HIbSjSLtxTHedNfOf3waW0NivTTxYXiYGX/MCu3GWhgIGj47a+H3A6FcQ/9QWvnKgnoixdgPBUz7kDb7ktsWwQsILPGStgH7iMuG49ZlXdEFmqwifBri2wvzmFEevBGZjHcupy1IGrNFRG+IUGKMotio+OkLHlLjuv7ZJtqCz/Vf5SNFgNyMSanx6jKEUJuYvndVg99IRXmYVwHFwPu5BAcJACpU6C0AfyGmmSqqwxCd46uXL62ynxNFpHuRfOqlDnmCTfZgjOciJSlDDpf+Xz9fF7+oCoeCi3mrcZVFjhd3tT6Oxw5HrsDtm0ZNld1cdLidaq8H6vOFgHMd0A9yNYZzTzXTvpmxzkXT4Zc7s+PYKN6z5fRZ+pJeckUjRXblvVEfs5HFSymavcOc5AkRwxpvOsTQMNmlnaJCBo5UNs0K/rVmRi5cFmaiwTcBCY0kTllOBJ4zWsfq8seiokWwNUNK2g=
   file_glob: true
   overwrite: true
   file: gam-$GAMVERSION-*
   skip_cleanup: true
   draft: true
-  all_branches: true
   on:
     repo: jay0lee/GAM
     condition: $VMTYPE = build

src/GamCommands.txt

@@ -140,6 +140,7 @@ If an item contains spaces, it should be surrounded by ".
 
 <AccessToken> ::= <String>
 <ACLScope> ::= [user:]<EmailAddress>|group:<EmailAddress>|domain[:<DomainName>]|default
+<APIScopeURL> ::= <String>
 <ASPID> ::= <String>
 <BuildingID> ::= <String>|id:<String>
 <CalendarACLRole> ::= editor|freebusy|freebusyreader|owner|reader|writer
@@ -221,6 +222,7 @@ If an item contains spaces, it should be surrounded by ".
 <SchemaName> ::= <String>
 <Section> ::= <String>
 <SerialNumber> ::= <String>
+<ServiceAccountKey> ::= <String>
 <S/MIMEID> ::= <String>	
 <SMTPHostName> ::= <String>
 <StudentItem> ::= <EmailAddress>|<UniqueID>|<String>
@@ -257,16 +259,20 @@ If an item contains spaces, it should be surrounded by ".
 	annotatedassetid|assedid|asset|
 	annotatedlocation|location|
 	annotateduser|user|
+	autoupdateexpiration|
 	bootmode|
 	cpustatusreports|
 	devicefiles|
 	deviceid|
 	diskvolumereports|
+	dockmacaddress|
 	ethernetmacaddress|
+	ethernetmacaddress0|
 	firmwareversion|
 	lastenrollmenttime|
 	lastsync|
 	macaddress|
+	manufacturedate|
 	meid|
 	model|
 	notes|
@@ -299,6 +305,7 @@ If an item contains spaces, it should be surrounded by ".
 	cancomment|
 	canreadrevisions|
 	copyable|
+	copyrequireswriterpermission|
 	createddate|createdtime|
 	description|
 	editable|
@@ -521,6 +528,8 @@ If an item contains spaces, it should be surrounded by ".
 	phones|phone|
 	posixaccounts|posix|
 	primaryemail|username|
+	recoveryemail|
+	recoveryphone|
 	relations|relation|
 	ssh|sshkeys|sshpublickeys|
 	suspended|
@@ -545,6 +554,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
     "'it em' 'it,em' \"it'em\""
 
 <ACLList> ::= "<ACLScope>(,<ACLScope>)*"
+<APIScopeURLList> ::= "<APIScopeURL>(,<APIScopeURL>)*"
 <ASPIDList> ::= "<ASPID>(,<ASPID>)*"
 <CalendarList> ::= "<CalendarItem>(,<CalendarItem>)*"
 <ChatRoomList> ::= "<ChatRoom>(,<ChatRoom>)*"
@@ -583,6 +593,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
 <SKUIDList> ="<SKUID>(,<SKUID>)*"
 <SchemaNameList> ::= "<SchemaName>(,<SchemaName>)*"
 <SerialNumberList> ::= "<SerialNumber>(,<SerialNumber>)*"
+<ServiceAccountKeyList> ::= "<ServiceAccountKey>(,<ServiceAccountKey>)*"
 <TeamDriveIDList> ::= "<TeamDriveID>(,<TeamDriveID>)*"
 <UserFieldNameList> ::= "<UserFieldName>(,<UserFieldName>)*"
 <UserList> ::= "<UserItem>(,<UserItem>)*"
@@ -664,12 +675,16 @@ Specify a collection of Users by directly specifying them or by specifiying item
 
 <DriveFileAddAttributes> ::=
 	(localfile <FileName>)|
-	(convert)|(ocr)|(ocrlanguage <Language>)|(restricted|restrict)|(starred|star)|(trashed|trash)|(viewed|view)|
+	(convert)|(ocr)|(ocrlanguage <Language>)|
+	(restricted|restrict)|(starred|star)|(trashed|trash)|(viewed|view)|
+	copyrequireswriterpermission|
 	(lastviewedbyme <Time>)|(modifieddate|modifiedtime <Time>)|(description <String>)|(mimetype <MimeType>)|
 	(parentid <DriveFolderID>)|(parentname <DriveFolderName>)|(anyownerparentname <DriveFolderName>)|writerscantshare
 <DriveFileUpdateAttributes> ::=
 	(localfile <FileName>)|
-	(convert)|(ocr)|(ocrlanguage <Language>)|(restricted|restrict <Boolean>)|(starred|star <Boolean>)|(trashed|trash <Boolean>)|(viewed|view <Boolean>)|
+	(convert)|(ocr)|(ocrlanguage <Language>)|
+	(restricted|restrict <Boolean>)|(starred|star <Boolean>)|(trashed|trash <Boolean>)|(viewed|view <Boolean>)|
+	(copyrequireswriterpermission <Boolean>)|
 	(lastviewedbyme <Time>)|(modifieddate <Time>)|(description <String>)|(mimetype <MimeType>)|
 	(parentid <DriveFolderID>)|(parentname <DriveFolderName>)|(anyownerparentname <DriveFolderName>)|writerscantshare
 <GroupSettingsAttribute> ::=
@@ -776,6 +791,8 @@ Specify a collection of Users by directly specifying them or by specifiying item
 	(note clear|([text_html|text_plain] <String>|(file <FileName> [charset <Charset>])))|
 	(org|ou|orgunitpath <OrgUnitPath>)
 	(password random|<Password>)|
+	(recoveryemail <EmailAddress>)|
+	(recoveryphone <string>)|
 	(suspended <Boolean>)|
 	(<SchemaName>.<FieldName> [multivalued|multivalue|value|multinonempty [type home|other|work|(custom <String>)]] <String>)
 <UserMultiAttributes> ::=
@@ -799,7 +816,7 @@ Specify a collection of Users by directly specifying them or by specifiying item
 	<UserBasicAttribute>|
 	<UserMultiAttribute>
 
-gam version [check|checkrc|simple|extended] [location <HostName>]
+gam version [check|checkrc|simple|extended] [timeoffset] [location <HostName>]
 gam help
 
 gam batch <FileName>|- [charset <Charset>]
@@ -821,12 +838,17 @@ gam delete project [<EmailAddress>] [gam|<ProjectID>|(filter <String>)]
 gam show projects [<EmailAddress>] [all|gam|<ProjectID>|(filter <String>)]
 gam print projects [<EmailAddress>] [all|gam|<ProjectID>|(filter <String>)] [todrive]
 
+gam rotate sakey|sakeys [retain_none|retain_existing|replace_current]
+         [(algorithm KEY_ALG_RSA_1024|KEY_ALG_RSA_2048)|(localkeysize 1024|2048|4096)]
+gam delete sakey|sakeys <ServiceAccountKeyList>+ [doit]
+gam show sakey|sakeys [all|system|user]
+
 gam oauth|oauth2 create|request [<EmailAddress>]
 gam oauth|oauth2 delete|revoke
 gam oauth|oauth2 info|verify [accesstoken <AccessToken>] [idtoken <IDToken>] [showsecret]
 gam oauth|oauth2 refresh
 
-gam <UserTypeEntity> check serviceaccount
+gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
 gam whatis <EmailItem>
 
@@ -858,6 +880,25 @@ gam update resoldsubscription <CustomerID> <SKUID>
 gam delete resoldsubscription <CustomerID> <SKUID> cancel|downgrade|transfer_to_direct
 gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 
+<ActivityApplicationName> ::=
+	access|accesstransparency|
+	admin|
+	calendar|calendars|
+	chat|
+	drive|doc|docs|
+	enterprisegroups|groupsenterprise|
+	gcp|
+	google+|gplus|
+	group|groups|
+	hangoutsmeet|meet|
+	jamboard|
+	login|logins|
+	mobile|
+	oauthtoken|token|tokens|
+	rules|
+	saml|
+	useraccounts
+
 <ReportsApp> ::=
 	accounts|
 	app_maker|
@@ -877,7 +918,7 @@ gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppL
 	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)] [filter|filters <String>] [fields|parameters <String>]
 gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
 	[fields|parameters <String>]
-gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token [todrive]
+gam report <ActivityApplicationName> [todrive]
 	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
@@ -1057,7 +1098,7 @@ gam print mobile [todrive] [(query <QueryMobile>)|(queries <QueryMobileList>)] [
 	fields <MobileFieldNameList>] [delimiter <Character>] [appslimit <Number>] [listlimit <Number>]
 
 gam create group <EmailAddress> <GroupAttributes>*
-gam update group <GroupItem> [admincreated <Boolean>] [email <EmailAddress>] <GroupAttributes>*
+gam update group <GroupItem> [email <EmailAddress>] <GroupAttributes>*
 gam update group <GroupItem> add [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
 gam update group <GroupItem> delete|remove [owner|manager|member] <UserTypeEntity>
 gam update group <GroupItem> sync [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
@@ -1075,6 +1116,7 @@ gam info member <UserItem> <GroupItem>
 gam print group-members|groups-members [todrive]
 	([domain <DomainName>] ([member <UserItem>]|[query <QueryGroup>]))|[group|group_ns|group_susp <GroupItem>] [notsuspended|suspended]
 	[roles <GroupRoleList>] [membernames] [fields <MembersFieldNameList>]
+	[includederivedmembership]
 
 gam print licenses [todrive] [(products|product <ProductIDList>)|(skus|sku <SKUIDList>)|allskus|gsuite] [countsonly]
 gam show license|licenses|licence|licences [(products|product <ProductIDList>)|(skus|sku <SKUIDList>)|allskus|gsuite]
@@ -1295,7 +1337,7 @@ gam <UserTypeEntity> draftemail [recipient|to <EmailAddress>] [from <EmailAddres
 gam <UserTypeEntity> importemail [recipient|to <EmailAddress>] [from <EmailAddress>]
 	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]
 	[labels <LabelNameList>] (header <String> <String>)*
-        [deleted] [date <Time>]
+	[deleted] [date <Time>]
 	[nevercheckspam] [processforcalendar]
 gam <UserTypeEntity> insertemail [recipient|to <EmailAddress>] [from <EmailAddress>]
 	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]

src/controlflow.py

@@ -0,0 +1,99 @@
+"""Methods related to the central control flow of an application."""
+import random
+import sys
+import time
+
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from var import MESSAGE_INVALID_JSON
+
+
+def system_error_exit(return_code, message):
+  """Raises a system exit with the given return code and message.
+
+  Args:
+    return_code: Int, the return code to yield when the system exits.
+    message: An error message to print before the system exits.
+  """
+  if message:
+    display.print_error(message)
+  sys.exit(return_code)
+
+
+def invalid_argument_exit(argument, command):
+  '''Indicate that the argument is not valid for the command.
+
+  Args:
+    argument: the invalid agrument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'{argument} is not a valid argument for "{command}"')
+
+def missing_argument_exit(argument, command):
+  '''Indicate that the argument is missing for the command.
+
+  Args:
+    argument: the missingagrument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'missing argument {argument} for "{command}"')
+
+def expected_argument_exit(name, expected, argument):
+  '''Indicate that the argument does not have an expected value for the command.
+
+  Args:
+    name: the field name
+    expected: the expected values
+    argument: the invalid argument
+  '''
+  system_error_exit(
+      2,
+      f'{name} must be one of {expected}; got {argument}')
+
+def csv_field_error_exit(field_name, field_names):
+  """Raises a system exit when a CSV field is malformed.
+
+  Args:
+    field_name: The CSV field name for which a header does not exist in the
+      existing CSV headers.
+    field_names: The known list of CSV headers.
+  """
+  system_error_exit(
+      2,
+      MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                     ','.join(field_names)))
+
+
+def invalid_json_exit(file_name):
+  """Raises a sysyem exit when invalid JSON content is encountered."""
+  system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+
+
+def wait_on_failure(current_attempt_num,
+                    total_num_retries,
+                    error_message,
+                    error_print_threshold=3):
+  """Executes an exponential backoff-style system sleep.
+
+  Args:
+    current_attempt_num: Int, the current number of retries.
+    total_num_retries: Int, the total number of times the current action will be
+      retried.
+    error_message: String, a message to be displayed that will give more context
+      around why the action is being retried.
+    error_print_threshold: Int, the number of attempts which will have their
+      error messages suppressed. Any current_attempt_num greater than
+      error_print_threshold will print the prescribed error.
+  """
+  wait_on_fail = min(2**current_attempt_num,
+                     60) + float(random.randint(1, 1000)) / 1000
+  if current_attempt_num > error_print_threshold:
+    sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+        f'{int(wait_on_fail)} seconds, Retry: '
+        f'{current_attempt_num}/{total_num_retries}\n'))
+    sys.stderr.flush()
+  time.sleep(wait_on_fail)

src/controlflow_test.py

@@ -0,0 +1,106 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+  def test_system_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(1, 'exit message')
+
+  def test_system_error_exit_raises_systemexit_with_return_code(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertEqual(context_manager.exception.code, 100)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+  def test_csv_field_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+
+  def test_csv_field_error_exit_exits_code_2(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    self.assertEqual(context_manager.exception.code, 2)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('aField', printed_message)
+    self.assertIn('unusedField1', printed_message)
+    self.assertIn('unusedField2', printed_message)
+
+  def test_invalid_json_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+
+  def test_invalid_json_exit_exit_exits_code_17(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.invalid_json_exit('filename')
+    self.assertEqual(context_manager.exception.code, 17)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('filename', printed_message)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+    sleep_calls = mock_sleep.call_args_list
+    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+    total_attempts = 20
+    for attempt in range(1, total_attempts + 1):
+      controlflow.wait_on_failure(
+          attempt,
+          total_attempts,
+          'Attempt #%s' % attempt,
+          # Suppress messages while we make a lot of attempts.
+          error_print_threshold=total_attempts + 1)
+      # Wait time may be between 60 and 61 secs, due to rand addition.
+      self.assertLessEqual(mock_sleep.call_args[0][0], 61)
+
+  # Prevent the system from actually sleeping and thus slowing down the test.
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
+    message = 'An error message to display'
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+    self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_only_prints_after_threshold(self, unused_mock_sleep):
+    total_attempts = 5
+    threshold = 3
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      for attempt in range(1, total_attempts + 1):
+        controlflow.wait_on_failure(
+            attempt,
+            total_attempts,
+            'Attempt #%s' % attempt,
+            error_print_threshold=threshold)
+    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

src/display.py

@@ -0,0 +1,15 @@
+"""Methods related to display of information to the user."""
+
+import sys
+from var import ERROR_PREFIX
+from var import WARNING_PREFIX
+
+
+def print_error(message):
+  """Prints a one-line error message to stderr in a standard format."""
+  sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
+
+
+def print_warning(message):
+  """Prints a one-line warning message to stderr in a standard format."""
+  sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))

src/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+import display
+from var import ERROR_PREFIX
+from var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+  def test_print_error_prints_to_stderr(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  def test_print_error_prints_error_prefix(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(ERROR_PREFIX), printed_message.find(message),
+        'The error prefix does not appear before the error message')
+
+  def test_print_error_ends_message_with_newline(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The error message does not end in a newline.')
+
+  def test_print_warning_prints_to_stderr(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  def test_print_warning_prints_error_prefix(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(WARNING_PREFIX), printed_message.find(message),
+        'The warning prefix does not appear before the error message')
+
+  def test_print_warning_ends_message_with_newline(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The warning message does not end in a newline.')

src/fileutils.py

@@ -0,0 +1,174 @@
+"""Common file operations."""
+
+import io
+import os
+import sys
+
+import controlflow
+import display
+from var import GM_Globals
+from var import GM_SYS_ENCODING
+from var import UTF8_SIG
+
+
+def _open_file(filename, mode, encoding=None, newline=None):
+  """Opens a file with no error handling."""
+  # Determine which encoding to use
+  if 'b' in mode:
+    encoding = None
+  elif not encoding:
+    encoding = GM_Globals[GM_SYS_ENCODING]
+  elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
+    encoding = UTF8_SIG
+
+  return open(
+      os.path.expanduser(filename), mode, newline=newline, encoding=encoding)
+
+
+def open_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              strip_utf_bom=False):
+  """Opens a file.
+
+  Args:
+    filename: String, the name of the file to open, or '-' to use stdin/stdout,
+      to read/write, depending on the mode param, respectively.
+    mode: String, the common file mode to open the file with. Default is read.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    strip_utf_bom: Boolean, True if the file being opened should seek past the
+      UTF Byte Order Mark before being returned.
+        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+
+  Returns:
+    The opened file.
+  """
+  try:
+    if filename == '-':
+      # Read from stdin, rather than a file
+      if 'r' in mode:
+        return io.StringIO(str(sys.stdin.read()))
+      return sys.stdout
+
+    # Open a file on disk
+    f = _open_file(filename, mode, newline=newline, encoding=encoding)
+    if strip_utf_bom:
+      utf_bom = u'\ufeff'
+      has_bom = False
+
+      if 'b' in mode:
+        has_bom = f.read(3).decode('UTF-8') == utf_bom
+      elif f.encoding and not f.encoding.lower().startswith('utf'):
+        # Convert UTF BOM into ISO-8859-1 via Bytes
+        utf8_bom_bytes = utf_bom.encode('UTF-8')
+        iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
+            'iso-8859-1')
+        has_bom = f.read(3).encode('iso-8859-1', 'replace') == iso_8859_1_bom
+      else:
+        has_bom = f.read(1) == utf_bom
+
+      if not has_bom:
+        f.seek(0)
+
+    return f
+
+  except IOError as e:
+    controlflow.system_error_exit(6, e)
+
+
+def close_file(f):
+  """Closes a file.
+
+  Args:
+    f: The file to close
+
+  Returns:
+    Boolean, True if the file was successfully closed. False if an error
+        was encountered while closing.
+  """
+  try:
+    f.close()
+    return True
+  except IOError as e:
+    display.print_error(e)
+    return False
+
+
+def read_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              continue_on_error=False,
+              display_errors=True):
+  """Reads a file from disk.
+
+  Args:
+    filename: String, the path of the file to open from disk, or "-" to read
+      from stdin.
+    mode: String, the mode in which to open the file.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    The contents of the file, or stdin if filename == "-". Returns None if
+    an error is encountered and continue_on_errors is True.
+  """
+  try:
+    if filename == '-':
+      # Read from stdin, rather than a file.
+      return str(sys.stdin.read())
+
+    with _open_file(filename, mode, newline=newline, encoding=encoding) as f:
+      return f.read()
+
+  except IOError as e:
+    if continue_on_error:
+      if display_errors:
+        display.print_warning(e)
+      return None
+    controlflow.system_error_exit(6, e)
+  except (LookupError, UnicodeDecodeError, UnicodeError) as e:
+    controlflow.system_error_exit(2, str(e))
+
+
+def write_file(filename,
+               data,
+               mode='w',
+               continue_on_error=False,
+               display_errors=True):
+  """Writes data to a file.
+
+  Args:
+    filename: String, the path of the file to write to disk.
+    data: Serializable data to write to the file.
+    mode: String, the mode in which to open the file and write to it.
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    Boolean, True if the write operation succeeded, or False if not.
+  """
+  try:
+    with _open_file(filename, mode) as f:
+      f.write(data)
+    return True
+
+  except IOError as e:
+    if continue_on_error:
+      if display_errors:
+        display.print_error(e)
+      return False
+    else:
+      controlflow.system_error_exit(6, e)

src/fileutils_test.py

@@ -0,0 +1,234 @@
+"""Tests for fileutils."""
+
+import io
+import os
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import fileutils
+
+
+class FileutilsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_path = '/some/path/to/file'
+    super(FileutilsTest, self).setUp()
+
+  @patch.object(fileutils.sys, 'stdin')
+  def test_open_file_stdin(self, mock_stdin):
+    mock_stdin.read.return_value = 'some stdin content'
+    f = fileutils.open_file('-', mode='r')
+    self.assertIsInstance(f, fileutils.io.StringIO)
+    self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
+
+  def test_open_file_stdout(self):
+    f = fileutils.open_file('-', mode='w')
+    self.assertEqual(fileutils.sys.stdout, f)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_opens_correct_path(self, mock_open):
+    f = fileutils.open_file(self.fake_path)
+    self.assertEqual(self.fake_path, mock_open.call_args[0][0])
+    self.assertEqual(mock_open.return_value, f)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_expands_user_file_path(self, mock_open):
+    file_path = '~/some/path/containing/tilde/shortcut/to/home'
+    fileutils.open_file(file_path)
+    opened_path = mock_open.call_args[0][0]
+    home_path = os.environ.get('HOME')
+    self.assertIsNotNone(home_path)
+    self.assertIn(home_path, opened_path)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_opens_correct_mode(self, mock_open):
+    fileutils.open_file(self.fake_path)
+    self.assertEqual('r', mock_open.call_args[0][1])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_encoding_for_binary(self, mock_open):
+    fileutils.open_file(self.fake_path, mode='b')
+    self.assertIsNone(mock_open.call_args[1]['encoding'])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_default_system_encoding(self, mock_open):
+    fileutils.open_file(self.fake_path)
+    self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
+                     mock_open.call_args[1]['encoding'])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_utf8_encoding_specified(self, mock_open):
+    fileutils.open_file(self.fake_path, encoding='UTF-8')
+    self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
+
+  def test_open_file_strips_utf_bom_in_utf(self):
+    bom_prefixed_data = u'\ufefffoobar'
+    fake_file = io.StringIO(bom_prefixed_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      self.assertEqual('foobar', f.read())
+
+  def test_open_file_strips_utf_bom_in_non_utf(self):
+    bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
+
+    # We need to trick the method under test into believing that a StringIO
+    # instance is a file with an encoding. Since StringIO does not usually have,
+    # an encoding, we'll mock it and add our own encoding, but send the other
+    # methods in use (read and seek) back to the real StringIO object.
+    real_stringio = io.StringIO(bom_prefixed_data)
+    mock_file = MagicMock(spec=io.StringIO)
+    mock_file.read.side_effect = real_stringio.read
+    mock_file.seek.side_effect = real_stringio.seek
+    mock_file.encoding = 'iso-8859-1'
+
+    mock_open = MagicMock(spec=open, return_value=mock_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      self.assertEqual('foobar', f.read())
+
+  def test_open_file_strips_utf_bom_in_binary(self):
+    bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
+    fake_file = io.BytesIO(bom_prefixed_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, mode='rb', strip_utf_bom=True)
+      self.assertEqual(b'foobar', f.read())
+
+  def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
+    no_bom_data = 'This data has no BOM'
+    fake_file = io.StringIO(no_bom_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      # Since there was no opening BOM, we should be back at the beginning of
+      # the file.
+      self.assertEqual(fake_file.tell(), 0)
+      self.assertEqual(f.read(), no_bom_data)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_exits_on_io_error(self, mock_open):
+    mock_open.side_effect = IOError('Fake IOError')
+    with self.assertRaises(SystemExit) as context:
+      fileutils.open_file(self.fake_path)
+    self.assertEqual(context.exception.code, 6)
+
+  def test_close_file_closes_file_successfully(self):
+    mock_file = MagicMock()
+    self.assertTrue(fileutils.close_file(mock_file))
+    self.assertEqual(mock_file.close.call_count, 1)
+
+  def test_close_file_with_error(self):
+    mock_file = MagicMock()
+    mock_file.close.side_effect = IOError()
+    self.assertFalse(fileutils.close_file(mock_file))
+    self.assertEqual(mock_file.close.call_count, 1)
+
+  @patch.object(fileutils.sys, 'stdin')
+  def test_read_file_from_stdin(self, mock_stdin):
+    mock_stdin.read.return_value = 'some stdin content'
+    self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_default_params(self, mock_open_file):
+    fake_content = 'some fake content'
+    mock_open_file.return_value.__enter__().read.return_value = fake_content
+    self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
+    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+    self.assertEqual(mock_open_file.call_args[0][1], 'r')
+    self.assertIsNone(mock_open_file.call_args[1]['newline'])
+
+  @patch.object(fileutils.display, 'print_warning')
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_continues_on_errors_without_displaying(
+      self, mock_open_file, mock_print_warning):
+    mock_open_file.side_effect = IOError()
+    contents = fileutils.read_file(
+        self.fake_path, continue_on_error=True, display_errors=False)
+    self.assertIsNone(contents)
+    self.assertFalse(mock_print_warning.called)
+
+  @patch.object(fileutils.display, 'print_warning')
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_displays_errors(self, mock_open_file, mock_print_warning):
+    mock_open_file.side_effect = IOError()
+    fileutils.read_file(
+        self.fake_path, continue_on_error=True, display_errors=True)
+    self.assertTrue(mock_print_warning.called)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_6_when_continue_on_error_is_false(
+      self, mock_open_file):
+    mock_open_file.side_effect = IOError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path, continue_on_error=False)
+    self.assertEqual(context.exception.code, 6)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
+    mock_open_file.return_value.__enter__().read.side_effect = LookupError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
+    mock_open_file.return_value.__enter__().read.side_effect = UnicodeError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
+    fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0, 1,
+                                           'testing only')
+    mock_open_file.return_value.__enter__().read.side_effect = fake_decode_error
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_writes_data_to_file(self, mock_open_file):
+    fake_data = 'some fake data'
+    fileutils.write_file(self.fake_path, fake_data)
+    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+    self.assertEqual(mock_open_file.call_args[0][1], 'w')
+
+    opened_file = mock_open_file.return_value.__enter__()
+    self.assertTrue(opened_file.write.called)
+    self.assertEqual(opened_file.write.call_args[0][0], fake_data)
+
+  @patch.object(fileutils.display, 'print_error')
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_continues_on_errors_without_displaying(
+      self, mock_open_file, mock_print_error):
+    mock_open_file.side_effect = IOError()
+    status = fileutils.write_file(
+        self.fake_path,
+        'foo data',
+        continue_on_error=True,
+        display_errors=False)
+    self.assertFalse(status)
+    self.assertFalse(mock_print_error.called)
+
+  @patch.object(fileutils.display, 'print_error')
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
+    mock_open_file.side_effect = IOError()
+    fileutils.write_file(
+        self.fake_path, 'foo data', continue_on_error=True, display_errors=True)
+    self.assertTrue(mock_print_error.called)
+
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_exits_code_6_when_continue_on_error_is_false(
+      self, mock_open_file):
+    mock_open_file.side_effect = IOError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.write_file(self.fake_path, 'foo data', continue_on_error=False)
+    self.assertEqual(context.exception.code, 6)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gam-install.sh

@@ -10,6 +10,7 @@ OPTIONS:
    -d      Directory where gam folder will be installed. Default is \$HOME/bin/
    -a      Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m".
    -o      OS we are running (linux, macos). Default is to detect your OS with "uname -s".
+   -b      OS version. Default is to detect on MacOS and Linux.
    -l      Just upgrade GAM to latest version. Skips project creation and auth.
    -p      Profile update (true, false). Should script add gam command to environment. Default is true.
    -u      Admin user email address to use with GAM. Default is to prompt.
@@ -21,20 +22,23 @@ EOF
 target_dir="$HOME/bin"
 gamarch=$(uname -m)
 gamos=$(uname -s)
+osversion=""
 update_profile=true
 upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.23 2.19 2.15"
+gam_glibc_vers="2.27 2.23 2.19 2.15"
+gam_macos_vers="10.14.4 10.13.6 10.12.6"
 
-while getopts "hd:a:o:lp:u:r:v:" OPTION
+while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
      case $OPTION in
          h) usage; exit;;
          d) target_dir="$OPTARG";;
          a) gamarch="$OPTARG";;
          o) gamos="$OPTARG";;
+         b) osversion="$OPTARG";;
          l) upgrade_only=true;;
          p) update_profile="$OPTARG";;
          u) adminuser="$OPTARG";;
@@ -48,7 +52,7 @@ done
 target_dir=${target_dir%/}
 
 update_profile() {
-	[ -f "$1" ] || return 1
+	[ $2 -eq 1 ] || [ -f "$1" ] || return 1
 
 	grep -F "$alias_line" "$1" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
@@ -79,13 +83,26 @@ echo -e '\x1B[0m'
 
 version_gt()
 {
-test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+# MacOS < 10.13 doesn't support sort -V
+echo "" | sort -V > /dev/null 2>&1
+vsort_failed=$?
+if [ "${1}" = "${2}" ]; then
+  true
+elif (( $vsort_failed != 0 )); then
+  false
+else
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+fi
 }
 
 case $gamos in
   [lL]inux)
     gamos="linux"
-    this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    if [ "$osversion" == "" ]; then
+      this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    else
+      this_glibc_ver=$osversion
+    fi
     echo "This Linux distribution uses glibc $this_glibc_ver"
     useglibc="legacy"
     for gam_glibc_ver in $gam_glibc_vers; do
@@ -97,24 +114,33 @@ case $gamos in
     done
     case $gamarch in
       x86_64) gamfile="linux-x86_64-$useglibc.tar.xz";;
-      i?86) gamfile="linux-i686.tar.xz";;
-      arm|armv7l) gamfile="linux-armv7l.tar.xz";;
-      arm64|aarch64) gamfile="linux-aarch64.tar.xz";;
+      arm64|aarch64) gamfile="linux-arm64-$useglibc.tar.xz";;
       *)
-        echo_red "ERROR: this installer currently only supports i386, x86_64, arm and arm64 Linux. Looks like you're running on $gamarch. Exiting."
+        echo_red "ERROR: this installer currently only supports x86_64 and arm64 Linux. Looks like you're running on $gamarch. Exiting."
         exit
     esac
     ;;
   [Mm]ac[Oo][sS]|[Dd]arwin)
-    osver=$(sw_vers -productVersion | awk -F'.' '{print $2}')
-    if (( $osver < 13 )); then
-      echo_red "ERROR: GAM currently requires MacOS 10.13 or newer. You are running MacOS 10.$osver. Please upgrade." 
-      exit
-    else
-      echo_green "Good, you're running MacOS 10.$osver..."
-    fi
     gamos="macos"
-    gamfile="macos-x86_64.tar.xz"
+    if [ "$osversion" == "" ]; then
+      this_macos_ver=$(sw_vers -productVersion)
+    else
+      this_macos_ver=$osversion
+    fi
+    echo "You are running MacOS $this_macos_ver"
+    use_macos_ver=""
+    for gam_macos_ver in $gam_macos_vers; do
+      if version_gt $this_macos_ver $gam_macos_ver; then
+        use_macos_ver="MacOS$gam_macos_ver"
+        echo_green "Using GAM compiled on $use_macos_ver"
+        break
+      fi
+    done
+    if [ "$use_macos_ver" == "" ]; then
+      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
+      exit
+    fi
+    gamfile="macos-x86_64-$use_macos_ver.tar.xz"
     ;;
   *)
     echo_red "Sorry, this installer currently only supports Linux and MacOS. Looks like you're runnning on $gamos. Exiting."
@@ -161,11 +187,16 @@ try:
 except KeyError:
   print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
-pycmd="python"
+pycmd="python3"
 $pycmd -V >/dev/null 2>&1
 rc=$?
 if (( $rc != 0 )); then
-  pycmd="python3"
+  pycmd="python"
+fi
+$pycmd -V >/dev/null 2>&1
+rc=$?
+if (( $rc != 0 )); then
+  pycmd="python2"
 fi
 $pycmd -V >/dev/null 2>&1
 rc=$?
@@ -203,9 +234,21 @@ else
   echo_green "Finished extracting GAM archive."
 fi
 
+# Update profile to add gam command
+if [ "$update_profile" = true ]; then
+  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
+  if [ "$gamos" == "linux" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0
+  elif [ "$gamos" == "macos" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0 || update_profile "$HOME/.profile" 1
+  fi
+else
+  echo_yellow "skipping profile update."
+fi
+
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
-  "$target_dir/gam/gam" version
+  "$target_dir/gam/gam" version extended
   rc=$?
   if (( $rc != 0 )); then
     echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -216,18 +259,6 @@ if [ "$upgrade_only" = true ]; then
   exit
 fi
 
-# Update profile to add gam command
-if [ "$update_profile" = true ]; then
-  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
-  if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile"
-  elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile"
-  fi
-else
-  echo_yellow "skipping profile update."
-fi
-
 while true; do
   read -p "Can you run a full browser on this machine? (usually Y for MacOS, N for Linux if you SSH into this machine) " yn
   case $yn in
@@ -328,7 +359,7 @@ while $project_created; do
 done
 
 echo_green "Here's information about your new GAM installation:"
-"$target_dir/gam/gam" version
+"$target_dir/gam/gam" version extended
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."

src/gapi/__init__.py

@@ -0,0 +1,325 @@
+"""Methods related to execution of GAPI requests."""
+
+import sys
+
+import googleapiclient.errors
+import google.auth.exceptions
+import httplib2
+
+import controlflow
+import display
+from gapi import errors
+import transport
+from var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+                 GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
+                 MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
+                 MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
+
+
+def call(service,
+         function,
+         silent_errors=False,
+         soft_errors=False,
+         throw_reasons=None,
+         retry_reasons=None,
+         **kwargs):
+  """Executes a single request on a Google service function.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    silent_errors: Bool, If True, error messages are suppressed when
+      encountered.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A response object for the corresponding Google API call.
+  """
+  if throw_reasons is None:
+    throw_reasons = []
+  if retry_reasons is None:
+    retry_reasons = []
+
+  method = getattr(service, function)
+  retries = 10
+  parameters = dict(
+      list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+  for n in range(1, retries + 1):
+    try:
+      return method(**parameters).execute()
+    except googleapiclient.errors.HttpError as e:
+      http_status, reason, message = errors.get_gapi_error_detail(
+          e,
+          soft_errors=soft_errors,
+          silent_errors=silent_errors,
+          retry_on_http_error=n < 3)
+      if http_status == -1:
+        # The error detail indicated that we should retry this request
+        # We'll refresh credentials and make another pass
+        service._http.request.credentials.refresh(transport.create_http())
+        continue
+      if http_status == 0:
+        return None
+
+      is_known_error_reason = reason in [r.value for r in errors.ErrorReason]
+      if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
+        if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
+          raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
+              message)
+        raise e
+      if (n != retries) and (is_known_error_reason and errors.ErrorReason(
+          reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
+        controlflow.wait_on_failure(n, retries, reason)
+        continue
+      if soft_errors:
+        display.print_error(f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}')
+        return None
+      controlflow.system_error_exit(
+          int(http_status), f'{http_status}: {message} - {reason}')
+    except google.auth.exceptions.RefreshError as e:
+      handle_oauth_token_error(
+          e, soft_errors or
+          errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+      if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
+        raise errors.GapiServiceNotAvailableError(str(e))
+      display.print_error(f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
+      return None
+    except ValueError as e:
+      if hasattr(service._http, 'cache') and service._http.cache is not None:
+        service._http.cache = None
+        continue
+      controlflow.system_error_exit(4, str(e))
+    except (httplib2.ServerNotFoundError, RuntimeError) as e:
+      if n != retries:
+        service._http.connections = {}
+        controlflow.wait_on_failure(n, retries, str(e))
+        continue
+      controlflow.system_error_exit(4, str(e))
+    except TypeError as e:
+      controlflow.system_error_exit(4, str(e))
+
+
+def get_items(service,
+              function,
+              items='items',
+              throw_reasons=None,
+              retry_reasons=None,
+              **kwargs):
+  """Gets a single page of items from a Google service function that is paged.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the service
+      method's response object.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    The list of items in the first page of a response.
+  """
+  results = call(
+      service,
+      function,
+      throw_reasons=throw_reasons,
+      retry_reasons=retry_reasons,
+      **kwargs)
+  if results:
+    return results.get(items, [])
+  return []
+
+
+def _get_max_page_size_for_api_call(service, function, **kwargs):
+  """Gets the maximum number of results supported for a single API call.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of the service method to check for max page size.
+    **kwargs: Additional params that will be passed to the request method.
+
+  Returns:
+    Int, A value from discovery if it exists, otherwise value from
+        MAX_RESULTS_API_EXCEPTIONS, otherwise None
+  """
+  method = getattr(service, function)
+  api_id = method(**kwargs).methodId
+  for resource in service._rootDesc.get('resources', {}).values():
+    for a_method in resource.get('methods', {}).values():
+      if a_method.get('id') == api_id:
+        if not a_method.get('parameters') or a_method['parameters'].get(
+            'pageSize') or not a_method['parameters'].get('maxResults'):
+          # Make sure API call supports maxResults. For now we don't care to
+          # set pageSize since all known pageSize API calls have
+          # default pageSize == max pageSize.
+          return None
+        known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
+        max_results = a_method['parameters']['maxResults'].get(
+            'maximum', known_api_max)
+        return {'maxResults': max_results}
+
+  return None
+
+
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+def got_total_items_msg(items, eol):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+def got_total_items_first_last_msg(items):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}'+'\n'
+
+def get_all_pages(service,
+                  function,
+                  items='items',
+                  page_message=None,
+                  message_attribute=None,
+                  soft_errors=False,
+                  throw_reasons=None,
+                  retry_reasons=None,
+                  **kwargs):
+  """Aggregates and returns all pages of a Google service function response.
+
+  All pages of items are aggregated and returned as a single list.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the method's
+      response object. The items in this field will be aggregated across all
+      pages and returned.
+    page_message: String, a message to be displayed to the user during paging.
+      Template strings allow for dynamic content to be inserted during paging.
+        Supported template strings:
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
+            pages.
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
+            display a unique property of the first item in the current page.
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
+            display a unique property of the last item in the current page.
+    message_attribute: String, the name of a signature field within a single
+      returned item which identifies that unique item. This field is used with
+      `page_message` to templatize a paging status message.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A list of all items received from all paged responses.
+  """
+  if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
+    page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
+    if page_key:
+      kwargs.update(page_key)
+  all_items = []
+  page_token = None
+  total_items = 0
+  while True:
+    page = call(
+        service,
+        function,
+        soft_errors=soft_errors,
+        throw_reasons=throw_reasons,
+        retry_reasons=retry_reasons,
+        pageToken=page_token,
+        **kwargs)
+    if page:
+      page_token = page.get('nextPageToken')
+      page_items = page.get(items, [])
+      num_page_items = len(page_items)
+      total_items += num_page_items
+      all_items.extend(page_items)
+    else:
+      page_token = None
+      num_page_items = 0
+
+    # Show a paging message to the user that indicates paging progress
+    if page_message:
+      show_message = page_message.replace(TOTAL_ITEMS_MARKER, str(total_items))
+      if message_attribute:
+        first_item = page_items[0] if num_page_items > 0 else {}
+        last_item = page_items[-1] if num_page_items > 1 else first_item
+        show_message = show_message.replace(FIRST_ITEM_MARKER, str(first_item.get(message_attribute, '')))
+        show_message = show_message.replace(LAST_ITEM_MARKER, str(last_item.get(message_attribute, '')))
+      sys.stderr.write('\r')
+      sys.stderr.flush()
+      sys.stderr.write(show_message)
+
+    if not page_token:
+      # End the paging status message and return all items.
+      if page_message and (page_message[-1] != '\n'):
+        sys.stderr.write('\r\n')
+        sys.stderr.flush()
+      return all_items
+
+
+# TODO: Make this private once all execution related items that use this method
+# have been brought into this file
+def handle_oauth_token_error(e, soft_errors):
+  """On a token error, exits the application and writes a message to stderr.
+
+  Args:
+    e: google.auth.exceptions.RefreshError, The error to handle.
+    soft_errors: Boolean, if True, suppresses any applicable errors and instead
+      returns to the caller.
+  """
+  token_error = str(e).replace('.', '')
+  if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
+      'Invalid response'):
+    if soft_errors:
+      return
+    if not GM_Globals[GM_CURRENT_API_USER]:
+      display.print_error(
+          MESSAGE_API_ACCESS_DENIED.format(
+              GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+              ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+      controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
+    else:
+      controlflow.system_error_exit(
+          19,
+          MESSAGE_SERVICE_NOT_APPLICABLE.format(
+              GM_Globals[GM_CURRENT_API_USER]))
+  controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')

src/gapi/__init___test.py

@@ -0,0 +1,502 @@
+"""Tests for gapi."""
+
+import json
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import gapi
+from gapi import errors
+
+
+def create_http_error(status, reason, message):
+  """Creates a HttpError object similar to most Google API Errors.
+
+  Args:
+    status: Int, the error's HTTP response status number.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    googleapiclient.errors.HttpError
+  """
+  response = {
+      'status': status,
+      'content-type': 'application/json',
+  }
+  content = {
+      'error': {
+          'code': status,
+          'errors': [{
+              'reason': str(reason),
+              'message': message,
+          }]
+      }
+  }
+  content_bytes = json.dumps(content).encode('UTF-8')
+  return gapi.googleapiclient.errors.HttpError(response, content_bytes)
+
+
+class GapiTest(unittest.TestCase):
+
+  def setUp(self):
+    SetGlobalVariables()
+    self.mock_service = MagicMock()
+    self.mock_method_name = 'mock_method'
+    self.mock_method = getattr(self.mock_service, self.mock_method_name)
+
+    self.simple_3_page_response = [
+        {
+            'items': [{
+                'position': 'page1,item1'
+            }, {
+                'position': 'page1,item2'
+            }, {
+                'position': 'page1,item3'
+            }],
+            'nextPageToken': 'page2'
+        },
+        {
+            'items': [{
+                'position': 'page2,item1'
+            }, {
+                'position': 'page2,item2'
+            }, {
+                'position': 'page2,item3'
+            }],
+            'nextPageToken': 'page3'
+        },
+        {
+            'items': [{
+                'position': 'page3,item1'
+            }, {
+                'position': 'page3,item2'
+            }, {
+                'position': 'page3,item3'
+            }],
+        },
+    ]
+    self.empty_items_response = {'items': []}
+
+    super(GapiTest, self).setUp()
+
+  def test_call_returns_basic_200_response(self):
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, self.mock_method().execute.return_value)
+
+  def test_call_passes_target_method_params(self):
+    gapi.call(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    self.assertEqual(self.mock_method.call_count, 1)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(method_kwargs.get('my_param_1'), 1)
+    self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+  @patch.object(gapi.errors, 'get_gapi_error_detail')
+  def test_call_retries_with_soft_errors(self, mock_error_detail):
+    mock_error_detail.return_value = (-1, 'aReason', 'some message')
+
+    # Make the request fail first, then return the proper response on the retry.
+    fake_http_error = create_http_error(403, 'aReason', 'unused message')
+    fake_200_response = MagicMock()
+    self.mock_method.return_value.execute.side_effect = [
+        fake_http_error, fake_200_response
+    ]
+
+    response = gapi.call(
+        self.mock_service, self.mock_method_name, soft_errors=True)
+    self.assertEqual(response, fake_200_response)
+    self.assertEqual(
+        self.mock_service._http.request.credentials.refresh.call_count, 1)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+
+  def test_call_throws_for_provided_reason(self):
+    throw_reason = errors.ErrorReason.USER_NOT_FOUND
+    fake_http_error = create_http_error(404, throw_reason, 'forced throw')
+    self.mock_method.return_value.execute.side_effect = fake_http_error
+
+    gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
+    with self.assertRaises(gam_exception):
+      gapi.call(
+          self.mock_service,
+          self.mock_method_name,
+          throw_reasons=[throw_reason])
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_request_for_default_retry_reasons(
+      self, mock_wait_on_failure):
+
+    # Test using one of the default retry reasons
+    default_throw_reason = errors.ErrorReason.BACKEND_ERROR
+    self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
+
+    fake_http_error = create_http_error(404, default_throw_reason, 'message')
+    fake_200_response = MagicMock()
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_http_error, fake_200_response
+    ]
+
+    response = gapi.call(
+        self.mock_service, self.mock_method_name, retry_reasons=[])
+    self.assertEqual(response, fake_200_response)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+    # Make sure a backoff technique was used for retry.
+    self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_requests_for_provided_retry_reasons(
+      self, unused_mock_wait_on_failure):
+
+    retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
+    fake_retrieable_error1 = create_http_error(400, retry_reason1,
+                                               'Forced Error 1')
+    retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
+    fake_retrieable_error2 = create_http_error(400, retry_reason2,
+                                               'Forced Error 2')
+    non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
+    fake_non_retriable_error = create_http_error(
+        400, non_retriable_reason,
+        'This error should not cause the request to be retried')
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_retrieable_error1, fake_retrieable_error2, fake_non_retriable_error
+    ]
+
+    with self.assertRaises(SystemExit):
+      # The third call should raise the SystemExit when non_retriable_error is
+      # raised.
+      gapi.call(
+          self.mock_service,
+          self.mock_method_name,
+          retry_reasons=[retry_reason1, retry_reason2])
+
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
+
+  def test_call_exits_on_oauth_token_error(self):
+    # An error with any OAUTH2_TOKEN_ERROR
+    fake_token_error = gapi.google.auth.exceptions.RefreshError(
+        errors.OAUTH2_TOKEN_ERRORS[0])
+    self.mock_method.return_value.execute.side_effect = fake_token_error
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_exits_on_nonretriable_error(self):
+    error_reason = 'unknownReason'
+    fake_http_error = create_http_error(500, error_reason,
+                                        'Testing unretriable errors')
+    self.mock_method.return_value.execute.side_effect = fake_http_error
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_exits_on_request_valueerror(self):
+    self.mock_method.return_value.execute.side_effect = ValueError()
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_clears_bad_http_cache_on_request_failure(self):
+    self.mock_service._http.cache = 'something that is not None'
+    fake_200_response = MagicMock()
+    self.mock_method.return_value.execute.side_effect = [
+        ValueError(), fake_200_response
+    ]
+
+    self.assertIsNotNone(self.mock_service._http.cache)
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, fake_200_response)
+    # Assert the cache was cleared
+    self.assertIsNone(self.mock_service._http.cache)
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_requests_with_backoff_on_servernotfounderror(
+      self, mock_wait_on_failure):
+    fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
+    fake_200_response = MagicMock()
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_servernotfounderror, fake_200_response
+    ]
+
+    http_connections = self.mock_service._http.connections
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, fake_200_response)
+    # HTTP cached connections should be cleared on receiving this error
+    self.assertNotEqual(http_connections, self.mock_service._http.connections)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+    # Make sure a backoff technique was used for retry.
+    self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+  def test_get_items_calls_correct_service_function(self):
+    gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertTrue(self.mock_method.called)
+
+  def test_get_items_returns_one_page(self):
+    fake_response = {'items': [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertEqual(page, fake_response['items'])
+
+  def test_get_items_non_default_page_field_name(self):
+    field_name = 'things'
+    fake_response = {field_name: [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_items(
+        self.mock_service, self.mock_method_name, items=field_name)
+    self.assertEqual(page, fake_response[field_name])
+
+  def test_get_items_passes_additional_kwargs_to_service(self):
+    gapi.get_items(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    self.assertEqual(self.mock_method.call_count, 1)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(1, method_kwargs.get('my_param_1'))
+    self.assertEqual(2, method_kwargs.get('my_param_2'))
+
+  def test_get_items_returns_empty_list_when_no_items_returned(self):
+    non_items_response = {'noItemsInThisResponse': {}}
+    self.mock_method.return_value.execute.return_value = non_items_response
+    page = gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertIsInstance(page, list)
+    self.assertEqual(0, len(page))
+
+  def test_get_all_pages_returns_all_items(self):
+    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
+    page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
+    page_3 = {'items': ['3-1', '3-2', '3-3']}
+    self.mock_method.return_value.execute.side_effect = [page_1, page_2, page_3]
+    response_items = gapi.get_all_pages(self.mock_service,
+                                        self.mock_method_name)
+    self.assertListEqual(response_items,
+                         page_1['items'] + page_2['items'] + page_3['items'])
+
+  def test_get_all_pages_includes_next_pagetoken_in_request(self):
+    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
+    page_2 = {'items': ['2-1', '2-2', '2-3']}
+    self.mock_method.return_value.execute.side_effect = [page_1, page_2]
+
+    gapi.get_all_pages(self.mock_service, self.mock_method_name, pageSize=100)
+    self.assertEqual(self.mock_method.call_count, 2)
+    call_2_kwargs = self.mock_method.call_args_list[1][1]
+    self.assertIn('pageToken', call_2_kwargs)
+    self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
+
+  def test_get_all_pages_uses_default_max_page_size(self):
+    sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
+    sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
+    self.mock_method.return_value.methodId = sample_api_id
+    self.mock_service._rootDesc = {
+        'resources': {
+            'someResource': {
+                'methods': {
+                    'someMethod': {
+                        'id': sample_api_id,
+                        'parameters': {
+                            'maxResults': {
+                                'maximum': sample_api_max_results
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+    gapi.get_all_pages(self.mock_service, self.mock_method_name)
+    request_method_kwargs = self.mock_method.call_args[1]
+    self.assertIn('maxResults', request_method_kwargs)
+    self.assertEqual(request_method_kwargs['maxResults'],
+                     gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
+
+  def test_get_all_pages_max_page_size_overrided(self):
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+    gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, pageSize=123456)
+    request_method_kwargs = self.mock_method.call_args[1]
+    self.assertIn('pageSize', request_method_kwargs)
+    self.assertEqual(123456, request_method_kwargs['pageSize'])
+
+  def test_get_all_pages_prints_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    self.assertIn(paging_message, messages_written)
+
+  def test_get_all_pages_prints_paging_message_inline(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+
+    # Make sure a return carriage was written between two pages
+    paging_message_call_positions = [
+        i for i, message in enumerate(messages_written)
+        if message == paging_message
+    ]
+    self.assertGreater(len(paging_message_call_positions), 1)
+    printed_between_page_messages = messages_written[
+        paging_message_call_positions[0]:paging_message_call_positions[1]]
+    self.assertIn('\r', printed_between_page_messages)
+
+  def test_get_all_pages_ends_paging_message_with_newline(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    last_page_message_index = len(
+        messages_written) - messages_written[::-1].index(paging_message)
+    last_carriage_return_index = len(
+        messages_written) - messages_written[::-1].index('\r\n')
+    self.assertGreater(last_carriage_return_index, last_page_message_index)
+
+  def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'Total number of items discovered: %%total_items%%'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_item_count = len(self.simple_3_page_response[0]['items'])
+    page_1_message = paging_message.replace('%%total_items%%',
+                                            str(page_1_item_count))
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_item_count = len(self.simple_3_page_response[1]['items'])
+    page_2_message = paging_message.replace(
+        '%%total_items%%', str(page_1_item_count + page_2_item_count))
+    self.assertIn(page_2_message, messages_written)
+
+    page_3_item_count = len(self.simple_3_page_response[2]['items'])
+    page_3_message = paging_message.replace(
+        '%%total_items%%',
+        str(page_1_item_count + page_2_item_count + page_3_item_count))
+    self.assertIn(page_3_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%total_items', message)
+
+  def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'First item in page: %%first_item%%'
+
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_message = paging_message.replace(
+        '%%first_item%%',
+        self.simple_3_page_response[0]['items'][0]['position'])
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_message = paging_message.replace(
+        '%%first_item%%',
+        self.simple_3_page_response[1]['items'][0]['position'])
+    self.assertIn(page_2_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%first_item', message)
+
+  def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'Last item in page: %%last_item%%'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_message = paging_message.replace(
+        '%%last_item%%',
+        self.simple_3_page_response[0]['items'][-1]['position'])
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_message = paging_message.replace(
+        '%%last_item%%',
+        self.simple_3_page_response[1]['items'][-1]['position'])
+    self.assertIn(page_2_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%last_item', message)
+
+  def test_get_all_pages_prints_all_attributes_in_paging_message(self):
+    pass
+
+  def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+    gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(method_kwargs.get('my_param_1'), 1)
+    self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+  @patch.object(gapi, 'call')
+  def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
+    throw_for = MagicMock()
+    retry_for = MagicMock()
+    mock_call.return_value = self.empty_items_response
+    gapi.get_all_pages(
+        self.mock_service,
+        self.mock_method_name,
+        throw_reasons=throw_for,
+        retry_reasons=retry_for)
+    method_kwargs = mock_call.call_args[1]
+    self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
+    self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
+
+  def test_get_all_pages_non_default_items_field_name(self):
+    field_name = 'things'
+    fake_response = {field_name: [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, items=field_name)
+    self.assertEqual(page, fake_response[field_name])
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gapi/errors.py

@@ -0,0 +1,358 @@
+"""GAPI and OAuth Token related errors methods."""
+
+from enum import Enum
+import json
+
+import controlflow
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import UTF8
+
+
+class GapiAbortedError(Exception):
+  pass
+
+
+class GapiAuthErrorError(Exception):
+  pass
+
+
+class GapiBadGatewayError(Exception):
+  pass
+
+
+class GapiBadRequestError(Exception):
+  pass
+
+
+class GapiConditionNotMetError(Exception):
+  pass
+
+
+class GapiCyclicMembershipsNotAllowedError(Exception):
+  pass
+
+
+class GapiDomainCannotUseApisError(Exception):
+  pass
+
+
+class GapiDomainNotFoundError(Exception):
+  pass
+
+
+class GapiDuplicateError(Exception):
+  pass
+
+
+class GapiFailedPreconditionError(Exception):
+  pass
+
+
+class GapiForbiddenError(Exception):
+  pass
+
+
+class GapiGatewayTimeoutError(Exception):
+  pass
+
+
+class GapiGroupNotFoundError(Exception):
+  pass
+
+
+class GapiInvalidError(Exception):
+  pass
+
+
+class GapiInvalidArgumentError(Exception):
+  pass
+
+
+class GapiInvalidMemberError(Exception):
+  pass
+
+
+class GapiMemberNotFoundError(Exception):
+  pass
+
+
+class GapiNotFoundError(Exception):
+  pass
+
+
+class GapiNotImplementedError(Exception):
+  pass
+
+
+class GapiPermissionDeniedError(Exception):
+  pass
+
+
+class GapiResourceNotFoundError(Exception):
+  pass
+
+
+class GapiServiceNotAvailableError(Exception):
+  pass
+
+
+class GapiUserNotFoundError(Exception):
+  pass
+
+
+# GAPI Error Reasons
+class ErrorReason(Enum):
+  """The reason why a non-200 HTTP response was returned from a GAPI."""
+  ABORTED = 'aborted'
+  AUTH_ERROR = 'authError'
+  BACKEND_ERROR = 'backendError'
+  BAD_GATEWAY = 'badGateway'
+  BAD_REQUEST = 'badRequest'
+  CONDITION_NOT_MET = 'conditionNotMet'
+  CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
+  DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
+  DOMAIN_NOT_FOUND = 'domainNotFound'
+  DUPLICATE = 'duplicate'
+  FAILED_PRECONDITION = 'failedPrecondition'
+  FORBIDDEN = 'forbidden'
+  GATEWAY_TIMEOUT = 'gatewayTimeout'
+  GROUP_NOT_FOUND = 'groupNotFound'
+  INTERNAL_ERROR = 'internalError'
+  INVALID = 'invalid'
+  INVALID_ARGUMENT = 'invalidArgument'
+  INVALID_MEMBER = 'invalidMember'
+  MEMBER_NOT_FOUND = 'memberNotFound'
+  NOT_FOUND = 'notFound'
+  NOT_IMPLEMENTED = 'notImplemented'
+  PERMISSION_DENIED = 'permissionDenied'
+  QUOTA_EXCEEDED = 'quotaExceeded'
+  RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
+  RESOURCE_NOT_FOUND = 'resourceNotFound'
+  SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+  SYSTEM_ERROR = 'systemError'
+  USER_NOT_FOUND = 'userNotFound'
+  USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+
+  def __str__(self):
+    return str(self.value)
+
+
+# Common sets of GAPI error reasons
+DEFAULT_RETRY_REASONS = [
+  ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
+  ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
+  ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT,
+  ErrorReason.INTERNAL_ERROR
+  ]
+GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
+GROUP_GET_THROW_REASONS = [
+  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
+  ErrorReason.BAD_REQUEST
+  ]
+GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
+MEMBERS_THROW_REASONS = [
+  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
+  ErrorReason.FORBIDDEN
+  ]
+MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
+
+# A map of GAPI error reasons to the corresponding GAM Python Exception
+ERROR_REASON_TO_EXCEPTION = {
+  ErrorReason.ABORTED:
+    GapiAbortedError,
+  ErrorReason.AUTH_ERROR:
+    GapiAuthErrorError,
+  ErrorReason.BAD_GATEWAY:
+    GapiBadGatewayError,
+  ErrorReason.BAD_REQUEST:
+    GapiBadRequestError,
+  ErrorReason.CONDITION_NOT_MET:
+    GapiConditionNotMetError,
+  ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
+    GapiCyclicMembershipsNotAllowedError,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS:
+    GapiDomainCannotUseApisError,
+  ErrorReason.DOMAIN_NOT_FOUND:
+    GapiDomainNotFoundError,
+  ErrorReason.DUPLICATE:
+    GapiDuplicateError,
+  ErrorReason.FAILED_PRECONDITION:
+    GapiFailedPreconditionError,
+  ErrorReason.FORBIDDEN:
+    GapiForbiddenError,
+  ErrorReason.GATEWAY_TIMEOUT:
+    GapiGatewayTimeoutError,
+  ErrorReason.GROUP_NOT_FOUND:
+    GapiGroupNotFoundError,
+  ErrorReason.INVALID:
+    GapiInvalidError,
+  ErrorReason.INVALID_ARGUMENT:
+    GapiInvalidArgumentError,
+  ErrorReason.INVALID_MEMBER:
+    GapiInvalidMemberError,
+  ErrorReason.MEMBER_NOT_FOUND:
+    GapiMemberNotFoundError,
+  ErrorReason.NOT_FOUND:
+    GapiNotFoundError,
+  ErrorReason.NOT_IMPLEMENTED:
+    GapiNotImplementedError,
+  ErrorReason.PERMISSION_DENIED:
+    GapiPermissionDeniedError,
+  ErrorReason.RESOURCE_NOT_FOUND:
+    GapiResourceNotFoundError,
+  ErrorReason.SERVICE_NOT_AVAILABLE:
+    GapiServiceNotAvailableError,
+  ErrorReason.USER_NOT_FOUND:
+    GapiUserNotFoundError,
+  }
+
+# OAuth Token Errors
+OAUTH2_TOKEN_ERRORS = [
+  'access_denied',
+  'access_denied: Requested client not authorized',
+  'internal_failure: Backend Error',
+  'internal_failure: None',
+  'invalid_grant',
+  'invalid_grant: Bad Request',
+  'invalid_grant: Invalid email or User ID',
+  'invalid_grant: Not a valid email',
+  'invalid_grant: Invalid JWT: No valid verifier found for issuer',
+  'invalid_request: Invalid impersonation prn email address',
+  'unauthorized_client: Client is unauthorized to retrieve access tokens '
+  'using this method',
+  'unauthorized_client: Client is unauthorized to retrieve access tokens '
+  'using this method, or client not authorized for any of the scopes '
+  'requested',
+  'unauthorized_client: Unauthorized client or scope in request',
+  ]
+
+
+def _create_http_error_dict(status_code, reason, message):
+  """Creates a basic error dict similar to most Google API Errors.
+
+  Args:
+    status_code: Int, the error's HTTP response status code.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    dict
+  """
+  return {
+    'error': {
+      'code': status_code,
+      'errors': [{
+        'reason': str(reason),
+        'message': message,
+        }]
+      }
+  }
+
+
+def get_gapi_error_detail(e,
+                          soft_errors=False,
+                          silent_errors=False,
+                          retry_on_http_error=False):
+  """Extracts error detail from a non-200 GAPI Response.
+
+  Args:
+    e: googleapiclient.HttpError, The HTTP Error received.
+    soft_errors: Boolean, If true, causes error messages to be surpressed,
+      rather than sending them to stderr.
+    silent_errors: Boolean, If true, suppresses and ignores any errors from
+      being displayed
+    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
+      code, indicating that the request can be retried. TODO: Remove this param,
+        as it seems to be outside the scope of this method.
+
+  Returns:
+    A tuple containing the HTTP Response code, GAPI error reason, and error
+        message.
+  """
+  try:
+    error = json.loads(e.content.decode(UTF8))
+  except ValueError:
+    error_content = e.content.decode(UTF8) if isinstance(e.content,
+                                                         bytes) else e.content
+    if (e.resp['status'] == '503') and (
+        error_content == 'Quota exceeded for the current request'):
+      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
+    if (e.resp['status'] == '403') and (
+        error_content.startswith('Request rate higher than configured')):
+      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
+    if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
+      return (e.resp['status'], ErrorReason.BAD_GATEWAY.value, error_content)
+    if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
+      return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value, error_content)
+    if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
+      error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
+                                      'Domain not found')
+    elif (e.resp['status'] == '400') and (
+        'InvalidSsoSigningKey' in error_content):
+      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                      'InvalidSsoSigningKey')
+    elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
+      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                      'UnknownError')
+    elif retry_on_http_error:
+      return (-1, None, None)
+    elif soft_errors:
+      if not silent_errors:
+        display.print_error(error_content)
+      return (0, None, None)
+    else:
+      controlflow.system_error_exit(5, error_content)
+    # END: ValueError catch
+
+  if 'error' in error:
+    http_status = error['error']['code']
+    try:
+      message = error['error']['errors'][0]['message']
+    except KeyError:
+      message = error['error']['message']
+  else:
+    if 'error_description' in error:
+      if error['error_description'] == 'Invalid Value':
+        message = error['error_description']
+        http_status = 400
+        error = _create_http_error_dict(400, ErrorReason.INVALID.value, message)
+      else:
+        controlflow.system_error_exit(4, str(error))
+    else:
+      controlflow.system_error_exit(4, str(error))
+
+  # Extract the error reason
+  try:
+    reason = error['error']['errors'][0]['reason']
+    if reason == 'notFound':
+      if 'userKey' in message:
+        reason = ErrorReason.USER_NOT_FOUND.value
+      elif 'groupKey' in message:
+        reason = ErrorReason.GROUP_NOT_FOUND.value
+      elif 'memberKey' in message:
+        reason = ErrorReason.MEMBER_NOT_FOUND.value
+      elif 'Domain not found' in message:
+        reason = ErrorReason.DOMAIN_NOT_FOUND.value
+      elif 'Resource Not Found' in message:
+        reason = ErrorReason.RESOURCE_NOT_FOUND.value
+    elif reason == 'invalid':
+      if 'userId' in message:
+        reason = ErrorReason.USER_NOT_FOUND.value
+      elif 'memberKey' in message:
+        reason = ErrorReason.INVALID_MEMBER.value
+    elif reason == 'failedPrecondition':
+      if 'Bad Request' in message:
+        reason = ErrorReason.BAD_REQUEST.value
+      elif 'Mail service not enabled' in message:
+        reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
+    elif reason == 'required':
+      if 'memberKey' in message:
+        reason = ErrorReason.MEMBER_NOT_FOUND.value
+    elif reason == 'conditionNotMet':
+      if 'Cyclic memberships not allowed' in message:
+        reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
+  except KeyError:
+    reason = f'{http_status}'
+  return (http_status, reason, message)

src/gapi/errors_test.py

@@ -0,0 +1,209 @@
+"""Python unit tests for gapi.errors"""
+
+import json
+
+import unittest
+from unittest.mock import patch
+
+import googleapiclient.errors
+from gapi import errors
+
+
+def create_simple_http_error(status, reason, message):
+  content = errors._create_http_error_dict(status, reason, message)
+  return create_http_error(status, content)
+
+
+def create_http_error(status, content):
+  response = {
+      'status': status,
+      'content-type': 'application/json',
+  }
+  content_as_bytes = json.dumps(content).encode('UTF-8')
+  return googleapiclient.errors.HttpError(response, content_as_bytes)
+
+
+class ErrorsTest(unittest.TestCase):
+
+  def test_get_gapi_error_detail_quota_exceeded(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_invalid_domain(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_invalid_signing_key(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_unknown_error(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_retry_http_error(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_prints_soft_errors(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_exits_on_unrecoverable_errors(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_quota_exceeded_for_current_request(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_quota_exceeded_high_request_rate(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_extracts_user_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: userKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: userKey.')
+
+  def test_get_gapi_error_extracts_group_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: groupKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: groupKey.')
+
+  def test_get_gapi_error_extracts_member_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: memberKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: memberKey.')
+
+  def test_get_gapi_error_extracts_domain_not_found(self):
+    err = create_simple_http_error(404, 'notFound', 'Domain not found.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
+    self.assertEqual(message, 'Domain not found.')
+
+  def test_get_gapi_error_extracts_generic_resource_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: unknownResource.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: unknownResource.')
+
+  def test_get_gapi_error_extracts_invalid_userid(self):
+    err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+    self.assertEqual(message, 'Invalid Input: userId')
+
+  def test_get_gapi_error_extracts_invalid_member(self):
+    err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
+    self.assertEqual(message, 'Invalid Input: memberKey')
+
+  def test_get_gapi_error_extracts_bad_request(self):
+    err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
+    self.assertEqual(message, 'Bad Request')
+
+  def test_get_gapi_error_extracts_service_not_available(self):
+    err = create_simple_http_error(400, 'failedPrecondition',
+                                   'Mail service not enabled')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
+    self.assertEqual(message, 'Mail service not enabled')
+
+  def test_get_gapi_error_extracts_required_member_not_found(self):
+    err = create_simple_http_error(400, 'required',
+                                   'Missing required field: memberKey')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+    self.assertEqual(message, 'Missing required field: memberKey')
+
+  def test_get_gapi_error_extracts_cyclic_memberships_error(self):
+    err = create_simple_http_error(400, 'conditionNotMet',
+                                   'Cyclic memberships not allowed')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason,
+                     errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
+    self.assertEqual(message, 'Cyclic memberships not allowed')
+
+  def test_get_gapi_error_extracts_single_error_with_message(self):
+    status_code = 999
+    response = {'status': status_code}
+    # This error does not have an "errors" key describing each error.
+    content = {'error': {'code': status_code, 'message': 'unknown error'}}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, status_code)
+    self.assertEqual(reason, str(status_code))
+    self.assertEqual(message, content['error']['message'])
+
+  def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
+      self):
+    status_code = 999
+    response = {'status': status_code}
+    # This error only has an error_description_field and an unknown description.
+    content = {'error_description': 'something errored'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    with self.assertRaises(SystemExit) as context:
+      errors.get_gapi_error_detail(err)
+    self.assertEqual(4, context.exception.code)
+
+  def test_get_gapi_error_exits_on_invalid_error_description(self):
+    status_code = 400
+    response = {'status': status_code}
+    content = {'error_description': 'Invalid Value'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, status_code)
+    self.assertEqual(reason, errors.ErrorReason.INVALID.value)
+    self.assertEqual(message, 'Invalid Value')
+
+  def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
+    status_code = 900
+    response = {'status': status_code}
+    content = {'notErrorContentThatIsExpected': 'foo'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    with self.assertRaises(SystemExit) as context:
+      errors.get_gapi_error_detail(err)
+    self.assertEqual(4, context.exception.code)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/linux-gam.spec

@@ -1,8 +1,13 @@
 # -*- mode: python -*-
+
+import sys
+
+sys.modules['FixTk'] = None
+
 a = Analysis(['gam.py'],
              hiddenimports=[],
              hookspath=None,
-             excludes=['_tkinter'],
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
              runtime_hooks=None)
 for d in a.datas:
     if 'pyconfig' in d[0]:

src/macos-gam.spec

@@ -1,8 +1,12 @@
 # -*- mode: python -*-
+import sys
+
+sys.modules['FixTk'] = None
+
 a = Analysis(['gam.py'],
              hiddenimports=[],
              hookspath=None,
-             excludes=['_tkinter'],
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
              runtime_hooks=None)
 for d in a.datas:
     if 'pyconfig' in d[0]:

src/project-apis.txt

@@ -2,11 +2,14 @@ admin.googleapis.com
 alertcenter.googleapis.com
 appsactivity.googleapis.com
 calendar-json.googleapis.com
+chat.googleapis.com
 classroom.googleapis.com
 contacts.googleapis.com
 drive.googleapis.com
+iap.googleapis.com
 gmail.googleapis.com
 groupssettings.googleapis.com
+iam.googleapis.com
 licensing.googleapis.com
 plus.googleapis.com
 reseller.googleapis.com

src/requirements.txt

@@ -1,7 +1,10 @@
+cryptography
 python-dateutil
+distro; sys_platform == 'linux'
+filelock
 google-api-python-client>=1.7.10
-google-auth
+google-auth>=1.11.2
 google-auth-httplib2
-google-auth-oauthlib==0.4.0
+google-auth-oauthlib>=0.4.1
 httplib2>=0.13.0
-passlib
+passlib>=1.7.2; sys_platform == 'win32'

src/tools/a_atleast_b.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+#from packaging import version
+from distutils.version import LooseVersion
+import sys
+
+a = sys.argv[1]
+b = sys.argv[2]
+#result = version.parse(a) >= version.parse(b)
+result = LooseVersion(a) >= LooseVersion(b)
+if result:
+  print('OK: %s is equal or newer than %s' % (a, b))
+else:
+  print('ERROR: %s is older than %s' % (a, b))
+sys.exit(not result)

src/transport.py

@@ -0,0 +1,96 @@
+"""Methods related to network transport."""
+
+import google_auth_httplib2
+import httplib2
+
+from var import GAM_INFO
+from var import GC_CA_FILE
+from var import GC_TLS_MAX_VERSION
+from var import GC_TLS_MIN_VERSION
+from var import GC_Values
+
+
+def create_http(cache=None,
+                timeout=None,
+                override_min_tls=None,
+                override_max_tls=None):
+  """Creates a uniform HTTP transport object.
+
+  Args:
+    cache: The HTTP cache to use.
+    timeout: The cache timeout, in seconds.
+    override_min_tls: The minimum TLS version to require. If not provided, the
+      default is used.
+    override_max_tls: The maximum TLS version to require. If not provided, the
+      default is used.
+
+  Returns:
+    httplib2.Http with the specified options.
+  """
+  tls_minimum_version = override_min_tls if override_min_tls else GC_Values[GC_TLS_MIN_VERSION]
+  tls_maximum_version = override_max_tls if override_max_tls else GC_Values[GC_TLS_MAX_VERSION]
+  return httplib2.Http(ca_certs=GC_Values[GC_CA_FILE],
+                       tls_maximum_version=tls_maximum_version,
+                       tls_minimum_version=tls_minimum_version,
+                       cache=cache,
+                       timeout=timeout)
+
+
+def create_request(http=None):
+  """Creates a uniform Request object with a default http, if not provided.
+
+  Args:
+    http: Optional httplib2.Http compatible object to be used with the request.
+      If not provided, a default HTTP will be used.
+
+  Returns:
+    Request: A google_auth_httplib2.Request compatible Request.
+  """
+  if not http:
+    http = create_http()
+  return Request(http)
+
+
+GAM_USER_AGENT = GAM_INFO
+
+
+def _force_user_agent(user_agent):
+  """Creates a decorator which can force a user agent in HTTP headers."""
+
+  def decorator(request_method):
+    """Wraps a request method to insert a user-agent in HTTP headers."""
+
+    def wrapped_request_method(*args, **kwargs):
+      """Modifies HTTP headers to include a specified user-agent."""
+      if kwargs.get('headers') is not None:
+        if kwargs['headers'].get('user-agent'):
+          if user_agent not in kwargs['headers']['user-agent']:
+            # Save the existing user-agent header and tack on our own.
+            kwargs['headers']['user-agent'] = f'{user_agent} {kwargs["headers"]["user-agent"]}'
+        else:
+          kwargs['headers']['user-agent'] = user_agent
+      else:
+        kwargs['headers'] = {'user-agent': user_agent}
+      return request_method(*args, **kwargs)
+
+    return wrapped_request_method
+
+  return decorator
+
+
+class Request(google_auth_httplib2.Request):
+  """A Request which forces a user agent."""
+
+  @_force_user_agent(GAM_USER_AGENT)
+  def __call__(self, *args, **kwargs):
+    """Inserts the GAM user-agent header in requests."""
+    return super(Request, self).__call__(*args, **kwargs)
+
+
+class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
+  """An AuthorizedHttp which forces a user agent during requests."""
+
+  @_force_user_agent(GAM_USER_AGENT)
+  def request(self, *args, **kwargs):
+    """Inserts the GAM user-agent header in requests."""
+    return super(AuthorizedHttp, self).request(*args, **kwargs)

src/transport_test.py

@@ -0,0 +1,179 @@
+"""Tests for transport."""
+
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import google_auth_httplib2
+import httplib2
+
+import transport
+
+
+class CreateHttpTest(unittest.TestCase):
+
+  def setUp(self):
+    SetGlobalVariables()
+    super(CreateHttpTest, self).setUp()
+
+  def test_create_http_sets_default_values_on_http(self):
+    http = transport.create_http()
+    self.assertIsNone(http.cache)
+    self.assertIsNone(http.timeout)
+    self.assertEqual(http.tls_minimum_version,
+                     transport.GC_Values[transport.GC_TLS_MIN_VERSION])
+    self.assertEqual(http.tls_maximum_version,
+                     transport.GC_Values[transport.GC_TLS_MAX_VERSION])
+    self.assertEqual(http.ca_certs, transport.GC_Values[transport.GC_CA_FILE])
+
+  def test_create_http_sets_tls_min_version(self):
+    http = transport.create_http(override_min_tls='TLSv1_1')
+    self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
+
+  def test_create_http_sets_tls_max_version(self):
+    http = transport.create_http(override_max_tls='TLSv1_3')
+    self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
+
+  def test_create_http_sets_cache(self):
+    fake_cache = {}
+    http = transport.create_http(cache=fake_cache)
+    self.assertEqual(http.cache, fake_cache)
+
+  def test_create_http_sets_cache_timeout(self):
+    http = transport.create_http(timeout=1234)
+    self.assertEqual(http.timeout, 1234)
+
+
+class TransportTest(unittest.TestCase):
+
+  def setUp(self):
+    self.mock_http = MagicMock(spec=httplib2.Http)
+    self.mock_response = MagicMock(spec=httplib2.Response)
+    self.mock_content = MagicMock()
+    self.mock_http.request.return_value = (self.mock_response,
+                                           self.mock_content)
+    self.mock_credentials = MagicMock()
+    self.test_uri = 'http://example.com'
+    super(TransportTest, self).setUp()
+
+  @patch.object(transport, 'create_http')
+  def test_create_request_uses_default_http(self, mock_create_http):
+    request = transport.create_request()
+    self.assertEqual(request.http, mock_create_http.return_value)
+
+  def test_create_request_uses_provided_http(self):
+    request = transport.create_request(http=self.mock_http)
+    self.assertEqual(request.http, self.mock_http)
+
+  def test_create_request_returns_request_with_forced_user_agent(self):
+    request = transport.create_request()
+    self.assertIsInstance(request, transport.Request)
+
+  def test_request_is_google_auth_httplib2_compatible(self):
+    request = transport.create_request()
+    self.assertIsInstance(request, google_auth_httplib2.Request)
+
+  def test_request_call_returns_response_content(self):
+    request = transport.Request(self.mock_http)
+    response = request(self.test_uri)
+    self.assertEqual(self.mock_response.status, response.status)
+    self.assertEqual(self.mock_content, response.data)
+
+  def test_request_call_forces_user_agent_no_provided_headers(self):
+    request = transport.Request(self.mock_http)
+
+    request(self.test_uri)
+    headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', headers)
+    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+  def test_request_call_forces_user_agent_no_agent_in_headers(self):
+    request = transport.Request(self.mock_http)
+    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
+
+    request(self.test_uri, headers=fake_request_headers)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+    self.assertEqual('someData',
+                     final_headers['some-header-thats-not-a-user-agent'])
+
+  def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
+    request = transport.Request(self.mock_http)
+    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+    request(self.test_uri, headers=headers_with_user_agent)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn('existing-user-agent', final_headers['user-agent'])
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+  def test_request_call_same_user_agent_already_in_headers(self):
+    request = transport.Request(self.mock_http)
+    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+    request(self.test_uri, headers=same_user_agent_header)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    # Make sure the header wasn't duplicated
+    self.assertEqual(
+        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))
+
+  def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
+    http = transport.AuthorizedHttp(self.mock_credentials)
+    self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
+
+  def test_authorizedhttp_request_returns_response_content(self):
+    http = transport.AuthorizedHttp(self.mock_credentials, http=self.mock_http)
+    response, content = http.request(self.test_uri)
+    self.assertEqual(self.mock_response, response)
+    self.assertEqual(self.mock_content, content)
+
+  def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    authorized_http.request(self.test_uri)
+    headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', headers)
+    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+  def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
+
+    authorized_http.request(self.test_uri, headers=fake_request_headers)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+    self.assertEqual('someData',
+                     final_headers['some-header-thats-not-a-user-agent'])
+
+  def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
+      self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+    authorized_http.request(self.test_uri, headers=headers_with_user_agent)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn('existing-user-agent', final_headers['user-agent'])
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+  def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+    authorized_http.request(self.test_uri, headers=same_user_agent_header)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    # Make sure the header wasn't duplicated
+    self.assertEqual(
+        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))

src/travis/linux-arm64-before-install.sh

@@ -0,0 +1,75 @@
+export whereibelong=$(pwd)
+export dist=$(lsb_release --codename --short)
+echo "We are running on Ubuntu $dist"
+export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+cpucount=$(nproc --all)
+echo "This device has $cpucount CPUs for compiling..."
+echo -e "nameserver 8.8.8.8\nnameserver 8.8.4.4" > /tmp/resolv.conf
+sudo cp /tmp/resolv.conf /etc
+sudo apt-get -qq --yes update > /dev/null
+sudo apt-get -qq --yes install xz-utils > /dev/null
+SSLVER=$(~/ssl/bin/openssl version)
+SSLRESULT=$?
+PYVER=$(~/python/bin/python3 -V)
+PYRESULT=$?
+if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+  echo "RUNNING: apt dist-upgrade..."
+  sudo apt-get -qq --yes dist-upgrade > /dev/null
+  echo "Installing build tools..."
+  sudo apt-get -qq --yes install build-essential
+
+  echo "Installing deps for python3"
+  sudo cp -v /etc/apt/sources.list /tmp
+  sudo chmod a+rwx /tmp/sources.list
+  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+  sudo cp -v /tmp/sources.list /etc/apt
+  sudo apt-get -qq --yes update > /dev/null
+  sudo apt-get -qq --yes build-dep python3 > /dev/null
+  sudo apt-get -qq --yes install zlib1g-dev > /dev/null
+  sudo apt-get -qq --yes install libffi-dev > /dev/null
+
+  # Compile latest OpenSSL
+  echo "Downloading OpenSSL..."
+  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  echo "Extracting OpenSSL..."
+  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  cd openssl-$BUILD_OPENSSL_VERSION
+  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+  ./config shared --prefix=$HOME/ssl
+  echo "Running make for OpenSSL..."
+  make -j$cpucount -s
+  echo "Running make install for OpenSSL..."
+  make install > /dev/null
+  cd ~
+
+  # Compile latest Python
+  echo "Downloading Python $BUILD_PYTHON_VERSION..."
+  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+  echo "Extracting Python..."
+  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+  cd Python-$BUILD_PYTHON_VERSION
+  echo "Compiling Python $BUILD_PYTHON_VERSION..."
+  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+  unsafe_flags="--enable-optimizations --with-lto"  
+  echo "running configure with safe and unsafe"
+  ./configure $safe_flags $unsafe_flags > /dev/null
+  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+  echo "Installing Python..."
+  make install > /dev/null
+fi
+
+python=~/python/bin/python3
+pip=~/python/bin/pip3
+
+$python -V
+
+cd $whereibelong
+
+echo "Upgrading pip packages..."
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+
+mkdir ~/.ruby
+export GEM_HOME=~/.ruby
+export PATH=$PATH:~/.ruby/bin

src/travis/linux-arm64-install.sh

@@ -0,0 +1,33 @@
+cd src
+if [ "$VMTYPE" == "test" ]; then
+  export gam="$python gam.py"
+  export gampath=$(readlink -e .)
+else
+  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+  export gam="gam/gam"
+  export gampath=$(readlink -e gam)
+  export GAMVERSION=`$gam version simple`
+  cp LICENSE $gampath
+  cp whatsnew.txt $gampath
+  cp GamCommands.txt $gampath
+  this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+  GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
+  rm $gampath/lastupdatecheck.txt
+  tar cfJ $GAM_ARCHIVE gam/
+  echo "PyInstaller GAM info:"
+  du -h gam/gam
+  time $gam version extended
+
+  if [[ "$dist" == "precise" ]]; then
+    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
+    $python -OO -m staticx gam/gam gam/gam-staticx
+    strip gam/gam-staticx
+    rm gam/gam
+    mv gam/gam-staticx gam/gam
+    tar cfJ $GAM_LEGACY_ARCHIVE gam/
+    echo "Legacy StaticX GAM info:"
+    du -h gam/gam
+    time $gam version extended
+  fi
+
+fi

src/travis/linux-x86_64-before-install.sh

@@ -7,86 +7,90 @@ else
   export whereibelong=$(pwd)
   export dist=$(lsb_release --codename --short)
   echo "We are running on Ubuntu $dist"
-  echo "RUNNING: apt update..."
-  sudo apt-get -qq --yes update > /dev/null
-  echo "RUNNING: apt dist-upgrade..."
-#  sudo apt-get -qq --yes dist-upgrade > /dev/null
-  echo "Installing build tools..."
-  sudo apt-get -qq --yes install build-essential
-
-  echo "Installing deps for python3"
-  sudo cp -v /etc/apt/sources.list /tmp
-  sudo chmod a+rwx /tmp/sources.list
-  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
-  sudo cp -v /tmp/sources.list /etc/apt
-  sudo apt-get -qq --yes update > /dev/null
-  sudo apt-get -qq --yes build-dep python3 > /dev/null
-
-  mypath=$HOME
-  echo "My Path is $mypath"
+  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   cpucount=$(nproc --all)
   echo "This device has $cpucount CPUs for compiling..."
+  SSLVER=$(~/ssl/bin/openssl version)
+  SSLRESULT=$?
+  PYVER=$(~/python/bin/python3 -V)
+  PYRESULT=$?
+  if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+    echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+    if [ $SSLRESULT -ne 0 ]; then
+      echo "sslresult -ne 0"
+    fi
+    if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+      echo "sslver not equal to..."
+    fi
+    if [ $PYRESULT -ne 0 ]; then
+      echo "pyresult -ne 0"
+    fi
+    if [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION" ]]; then
+      echo "pyver not equal to..."
+    fi
+    cd ~
+    rm -rf ssl
+    rm -rf python
+    mkdir ssl
+    mkdir python
+    echo "RUNNING: apt update..."
+    sudo apt-get -qq --yes update > /dev/null
+    echo "RUNNING: apt dist-upgrade..."
+    sudo apt-get -qq --yes dist-upgrade > /dev/null
+    echo "Installing build tools..."
+    sudo apt-get -qq --yes install build-essential
+    echo "Installing deps for python3"
+    sudo cp -v /etc/apt/sources.list /tmp
+    sudo chmod a+rwx /tmp/sources.list
+    echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+    sudo cp -v /tmp/sources.list /etc/apt
+    sudo apt-get -qq --yes update > /dev/null
+    sudo apt-get -qq --yes build-dep python3 > /dev/null
 
-  cd ~/pybuild
-  # Compile latest OpenSSL
-  if [ ! -d openssl-$BUILD_OPENSSL_VERSION ]; then
+    # Compile latest OpenSSL
     wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
     echo "Extracting OpenSSL..."
     tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  fi
-  cd openssl-$BUILD_OPENSSL_VERSION
-  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-  ./config shared --prefix=$mypath/ssl
-  echo "Running make for OpenSSL..."
-  make -j$cpucount -s
-  echo "Running make install for OpenSSL..."
-  make install > /dev/null
-  export LD_LIBRARY_PATH=~/ssl/lib
-  cd ~
+    cd openssl-$BUILD_OPENSSL_VERSION
+    echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+    ./config shared --prefix=$HOME/ssl
+    echo "Running make for OpenSSL..."
+    make -j$cpucount -s
+    echo "Running make install for OpenSSL..."
+    make install > /dev/null
+    cd ~
 
-  cd ~/pybuild
-  # Compile latest Python
-  if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
+    # Compile latest Python
     echo "Downloading Python $BUILD_PYTHON_VERSION..."
     curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
     echo "Extracting Python..."
     tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
-  fi
-  cd Python-$BUILD_PYTHON_VERSION
-  #if [[ "$dist" == "bionic" ]]; then
-  #  echo "running bionic make clean"
-  #  make clean
-   # rm Makefile
-  #fi
-  echo "Compiling Python $BUILD_PYTHON_VERSION..."
-  safe_flags="--with-openssl=$mypath/ssl --enable-shared --prefix=$mypath/python --with-ensurepip=upgrade"
-  unsafe_flags="--enable-optimizations --with-lto"
-  
-  if [ ! -e Makefile ]; then
-    echo "running configure with safe and unsafe"
-    ./configure $safe_flags $unsafe_flags > /dev/null
-  fi
-  timeout 1800 make -j$cpucount -s
-  RESULT=$?
-  echo "First make exited with $RESULT"
-  if [ $RESULT != 0 ]; then
-    #echo "Trying Python compile again without unsafe flags..."
-    #make clean
-    #./configure $safe_flags > /dev/null
-    #make -j$cpucount -s
-    echo "Sticking with safe Python for now..."
-  else
-    echo "Installing optimized Python..."
+    cd Python-$BUILD_PYTHON_VERSION
+    echo "Compiling Python $BUILD_PYTHON_VERSION..."
+    safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+    unsafe_flags="--enable-optimizations --with-lto"
+    if [ ! -e Makefile ]; then
+      echo "running configure with safe and unsafe"
+      ./configure $safe_flags $unsafe_flags > /dev/null
+    fi
+    make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+    RESULT=$?
+    echo "First make exited with $RESULT"
+    if [ $RESULT != 0 ]; then
+      echo "Trying Python compile again without unsafe flags..."
+      make clean
+      ./configure $safe_flags > /dev/null
+      make -j$cpucount -s
+      echo "Sticking with safe Python for now..."
+    fi
+    echo "Installing Python..."
     make install > /dev/null
+    cd ~
   fi
-  cd ~
 
-  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   python=~/python/bin/python3
   pip=~/python/bin/pip3
 
-  $python -V
-
   if [[ "$dist" == "precise" ]]; then
     echo "Installing deps for StaticX..."
     sudo apt-get install --yes scons
@@ -99,22 +103,13 @@ else
       make
       sudo make install
     fi
-    if [ ! -d musl=$MUSL_VERSION ]; then
-      echo "Downloading MUSL $MUSL_VERSION"
-      wget https://www.musl-libc.org/releases/musl-$MUSL_VERSION.tar.gz
-      tar xf musl-$MUSL_VERSION.tar.gz
-      cd musl-$MUSL_VERSION
-      ./configure
-      make
-      sudo make install
-    fi
     $pip install git+https://github.com/JonathonReinhart/staticx.git@master
   fi
+
   cd $whereibelong
 fi
 
 echo "Upgrading pip packages..."
-$pip freeze > upgrades.txt
-$pip install --upgrade -r upgrades.txt
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade pyinstaller
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz

src/travis/osx-x86_64-before-install.sh

@@ -3,15 +3,24 @@ whereibelong=$(pwd)
 #echo "Brew installing xz..."
 #brew install xz > /dev/null
 
-cd ~/pybuild
+#brew upgrade
+
+cd ~
 
 if [ ! -f python-$BUILD_PYTHON_VERSION-macosx10.9.pkg ]; then
   wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
 fi
 sudo installer -pkg python-$BUILD_PYTHON_VERSION-macosx10.9.pkg -target /
+
+#brew install openssl@1.1
+#brew upgrade python
+
 export python=python3
 export pip=pip3
 
+echo "Python location:"
+which $python
+
 # Compile latest OpenSSL
 #if [ ! -d openssl-$BUILD_OPENSSL_VERSION ]; then
 #  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
@@ -26,7 +35,7 @@ export pip=pip3
 #echo "Running make install for OpenSSL..."
 #make install > /dev/null
 #export LD_LIBRARY_PATH=~/ssl/lib
-#cd ~/pybuild
+#cd ~
 
 # Compile latest Python
 #if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
@@ -58,14 +67,12 @@ export pip=pip3
 #python=~/python/bin/python3
 #pip=~/python/bin/pip3
 
-
 $python -V
 
 cd $whereibelong
 
 export PATH=/usr/local/opt/python/libexec/bin:$PATH
 $pip install --upgrade pip
-$pip freeze > upgrades.txt
-$pip install --upgrade -r upgrades.txt
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade pyinstaller
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 

src/travis/osx-x86_64-install.sh

@@ -1,4 +1,6 @@
 cd src
+echo "MacOS Version Info According to Python:"
+python -c "import platform; print(platform.mac_ver())"
 $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
 export gam="gam/gam"
 export gampath=gam
@@ -7,6 +9,7 @@ export GAMVERSION=`gam/gam version simple`
 cp LICENSE gam
 cp whatsnew.txt gam
 cp GamCommands.txt gam
-GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.tar.xz
+MACOSVERSION=$(defaults read loginwindow SystemVersionStampAsString)
+GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-MacOS$MACOSVERSION.tar.xz
 rm gam/lastupdatecheck.txt
 tar cfJ $GAM_ARCHIVE gam/

src/travis/windows-x86-before-install.sh

@@ -1,45 +1,34 @@
 echo "Installing Net-Framework-Core..."
 export mypath=$(pwd)
+cd ~
 until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~/pybuild
-#export exefile=Win32OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
-#if [ ! -e $exefile ]; then
-#  echo "Downloading $exefile..."
-#  wget --quiet https://slproweb.com/download/$exefile
-#fi
-#echo "Installing $exefile..."
-#powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
 cinst -y --forcex86 python3
 until cinst -y wixtoolset; do echo "trying again..."; done
-#echo "OpenSSL dlls..."
-#ls -alRF /c/ssl
-#echo "c drive"
-#ls -al /c/
-#echo "Python dlls..."
-#ls -al /c/Python37/DLLs
-#until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
-export PATH=$PATH:/c/Python37/scripts
+export PATH=$PATH:/c/Python38/scripts
 cd $mypath
-pip install --upgrade pip
-pip freeze > upgrades.txt
-pip install --upgrade -r upgrades.txt
-pip install --upgrade -r src/requirements.txt
+export python=/c/Python38/python.exe
+export pip=/c/Python38/scripts/pip.exe
 
-#pip install --upgrade pyinstaller
+$pip install --upgrade pip
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+
+#$pip install --upgrade pyinstaller
 # Install PyInstaller from source and build bootloader
 # to try and avoid getting flagged as malware since
 # lots of malware uses PyInstaller default bootloader
 # https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
 echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
-tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
-cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 
+tar xf develop.tar.gz
+cd pyinstaller-develop/bootloader
 echo "bootloader before:"
 md5sum ../PyInstaller/bootloader/Windows-32bit/*
-/c/python37/python ./waf all --target-arch=32bit
+$python ./waf all --target-arch=32bit
 echo "bootloader after:"
 md5sum ../PyInstaller/bootloader/Windows-32bit/*
 echo "PATH: $PATH"
 cd ..
-/c/python37/python setup.py install
+$python setup.py install
+echo "cd to $mypath"
 cd $mypath

src/travis/windows-x86_64-before-install.sh

@@ -1,7 +1,7 @@
 echo "Installing Net-Framework-Core..."
 export mypath=$(pwd)
 until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~/pybuild
+cd ~
 #export exefile=Win64OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
 #if [ ! -e $exefile ]; then
 #  echo "Downloading $exefile..."
@@ -11,17 +11,36 @@ cd ~/pybuild
 #powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
 cinst -y python3
 until cinst -y wixtoolset; do echo "trying again..."; done
-#echo "OpenSSL dlls..."
-#ls -alRF /c/ssl
-#echo "c drive"
-#ls -al /c
-#echo "Python dlls..."
-#ls -al /c/Python37/DLLs
-#until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
-export PATH=$PATH:/c/Python37/scripts
+#until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/Python37/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
+#until cp -v /c/ssl/libssl-1_1-x64.dll /c/Python37/DLLs/libssl-1_1.dll; do echo "trying again..."; done
+export PATH=$PATH:/c/Python38/scripts
+cd $mypath
+export python=/c/Python38/python.exe
+export pip=/c/Python38/scripts/pip.exe
+
+$pip install --upgrade pip
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+#$pip install --upgrade pyinstaller
+# Install PyInstaller from source and build bootloader
+# to try and avoid getting flagged as malware since
+# lots of malware uses PyInstaller default bootloader
+# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
+echo "Downloading PyInstaller..."
+#wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+#tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
+tar xf develop.tar.gz
+#cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+cd pyinstaller-develop/bootloader
+echo "bootloader before:"
+md5sum ../PyInstaller/bootloader/Windows-64bit/*
+$python ./waf all --target-arch=64bit
+echo "bootloader after:"
+md5sum ../PyInstaller/bootloader/Windows-64bit/*
+echo "PATH: $PATH"
+cd ..
+$python setup.py install
+echo "cd to $mypath..."
+#until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
 cd $mypath
-pip install --upgrade pip
-pip freeze > upgrades.txt
-pip install --upgrade -r upgrades.txt
-pip install --upgrade -r src/requirements.txt
-pip install --upgrade pyinstaller

src/travis/windows-x86_64-install.sh

@@ -1,7 +1,9 @@
 cd src
+echo "compiling GAM with pyinstaller..."
 pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
 export gam="gam/gam"
 export gampath=$(readlink -e gam)
+echo "running compiled GAM..."
 $gam version
 export GAMVERSION=`$gam version simple`
 rm gam/lastupdatecheck.txt

src/utils.py

@@ -9,10 +9,6 @@ ONE_MEGA_BYTES = 1000000
 ONE_GIGA_BYTES = 1000000000
 
 
-def convertUTF8(data):
-  return data
-
-
 class _DeHTMLParser(HTMLParser):
 
   def __init__(self):
@@ -30,7 +26,7 @@ class _DeHTMLParser(HTMLParser):
     if cp:
       self.__text.append(chr(cp))
     else:
-      self.__text.append('&' + name)
+      self.__text.append('&'+name)
 
   def handle_starttag(self, tag, attrs):
     if tag == 'p':
@@ -40,21 +36,20 @@ class _DeHTMLParser(HTMLParser):
     elif tag == 'a':
       for attr in attrs:
         if attr[0] == 'href':
-          self.__text.append('({0}) '.format(attr[1]))
+          self.__text.append(f'({attr[1]}) ')
           break
     elif tag == 'div':
       if not attrs:
         self.__text.append('\n')
-    elif tag in ['http:', 'https']:
-      self.__text.append(' ({0}//{1}) '.format(tag, attrs[0][0]))
+    elif tag in {'http:', 'https'}:
+      self.__text.append(f' ({tag}//{attrs[0][0]}) ')
 
   def handle_startendtag(self, tag, attrs):
     if tag == 'br':
       self.__text.append('\n\n')
 
   def text(self):
-    return re.sub(r'\n{2}\n+', '\n\n',
-                  re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
+    return re.sub(r'\n{2}\n+', '\n\n', re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
 
 
 def dehtml(text):
@@ -89,14 +84,14 @@ def formatFileSize(fileSize):
   if fileSize < ONE_KILO_BYTES:
     return '1kb'
   if fileSize < ONE_MEGA_BYTES:
-    return '{0}kb'.format(fileSize // ONE_KILO_BYTES)
+    return f'{fileSize // ONE_KILO_BYTES}kb'
   if fileSize < ONE_GIGA_BYTES:
-    return '{0}mb'.format(fileSize // ONE_MEGA_BYTES)
-  return '{0}gb'.format(fileSize // ONE_GIGA_BYTES)
+    return f'{fileSize // ONE_MEGA_BYTES}mb'
+  return f'{fileSize // ONE_GIGA_BYTES}gb'
 
 
 def formatMilliSeconds(millis):
   seconds, millis = divmod(millis, 1000)
   minutes, seconds = divmod(seconds, 60)
   hours, minutes = divmod(minutes, 60)
-  return '%02d:%02d:%02d' % (hours, minutes, seconds)
+  return f'{hours:02d}:{minutes:02d}:{seconds:02d}'

src/var.py

@@ -6,21 +6,19 @@ import platform
 import re
 
 gam_author = 'Jay Lee <jay0lee@gmail.com>'
-gam_version = '4.91'
+gam_version = '4.98'
 gam_license = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
-GAM_INFO = 'GAM {0} - {1} / {2} / Python {3}.{4}.{5} {6} / {7} {8} /'.format(gam_version, GAM_URL,
-                                                                             gam_author,
-                                                                             sys.version_info[0], sys.version_info[1],
-                                                                             sys.version_info[2], sys.version_info[3],
-                                                                             platform.platform(), platform.machine())
+GAM_INFO = (f'GAM {gam_version} - {GAM_URL} / {gam_author} / '
+           f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
+           f'{platform.platform()} {platform.machine()}')
 
 GAM_RELEASES = 'https://github.com/jay0lee/GAM/releases'
 GAM_WIKI = 'https://github.com/jay0lee/GAM/wiki'
 GAM_ALL_RELEASES = 'https://api.github.com/repos/jay0lee/GAM/releases'
 GAM_LATEST_RELEASE = GAM_ALL_RELEASES+'/latest'
-GAM_PROJECT_FILEPATH = 'https://raw.githubusercontent.com/jay0lee/GAM/master/'
+GAM_PROJECT_FILEPATH = 'https://raw.githubusercontent.com/jay0lee/GAM/master/src/'
 
 true_values = ['on', 'yes', 'enabled', 'true', '1']
 false_values = ['off', 'no', 'disabled', 'false', '0']
@@ -39,6 +37,8 @@ FN_LAST_UPDATE_CHECK_TXT = 'lastupdatecheck.txt'
 MY_CUSTOMER = 'my_customer'
 # See https://support.google.com/drive/answer/37603
 MAX_GOOGLE_SHEET_CELLS = 5000000
+MAX_LOCAL_GOOGLE_TIME_OFFSET = 30
+
 SKUS = {
   '1010010001': {
     'product': '101001', 'aliases': ['identity', 'cloudidentity'], 'displayName': 'Cloud Identity'},
@@ -47,7 +47,7 @@ SKUS = {
   '1010310002': {
     'product': '101031', 'aliases': ['gsefe', 'e4e', 'gsuiteenterpriseeducation'], 'displayName': 'G Suite Enterprise for Education'},
   '1010310003': {
-    'product': '101031', 'aliases': ['gsefes', 'e4es', 'gsuiteenterpriseeducationstudent'], 'displayName': 'G Suite Enterprise for Education Student'},
+    'product': '101031', 'aliases': ['gsefes', 'e4es', 'gsuiteenterpriseeducationstudent'], 'displayName': 'G Suite Enterprise for Education (Student)'},
   '1010330003': {
     'product': '101033', 'aliases': ['gvstarter', 'voicestarter', 'googlevoicestarter'], 'displayName': 'Google Voice Starter'},
   '1010330004': {
@@ -143,6 +143,7 @@ API_VER_MAPPING = {
   'drive3': 'v3',
   'gmail': 'v1',
   'groupssettings': 'v1',
+  'iam': 'v1',
   'licensing': 'v1',
   'oauth2': 'v2',
   'pubsub': 'v1',
@@ -154,6 +155,8 @@ API_VER_MAPPING = {
   'vault': 'v1',
   }
 
+USERINFO_EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email'
+
 API_SCOPE_MAPPING = {
   'alertcenter': ['https://www.googleapis.com/auth/apps.alerts',],
   'appsactivity': ['https://www.googleapis.com/auth/activity',
@@ -228,6 +231,7 @@ DRIVEFILE_FIELDS_CHOICES_MAP = {
   'cancomment': 'canComment',
   'canreadrevisions': 'canReadRevisions',
   'copyable': 'copyable',
+  'copyrequireswriterpermission': 'copyRequiresWriterPermission',
   'createddate': 'createdDate',
   'createdtime': 'createdDate',
   'description': 'description',
@@ -377,18 +381,33 @@ GOOGLEDOC_VALID_EXTENSIONS_MAP = {
   MIMETYPE_GA_SPREADSHEET: ['.csv', '.ods', '.pdf', '.xlsx', '.zip'],
   }
 
-_MICROSOFT_FORMATS_LIST = [{'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'ext': '.docx'},
-                           {'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'ext': '.dotx'},
-                           {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'ext': '.pptx'},
-                           {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.template', 'ext': '.potx'},
-                           {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'ext': '.xlsx'},
-                           {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'ext': '.xltx'},
-                           {'mime': 'application/msword', 'ext': '.doc'},
-                           {'mime': 'application/msword', 'ext': '.dot'},
-                           {'mime': 'application/vnd.ms-powerpoint', 'ext': '.ppt'},
-                           {'mime': 'application/vnd.ms-powerpoint', 'ext': '.pot'},
-                           {'mime': 'application/vnd.ms-excel', 'ext': '.xls'},
-                           {'mime': 'application/vnd.ms-excel', 'ext': '.xlt'}]
+MACOS_CODENAMES = {
+  6:  'Snow Leopard',
+  7:  'Lion',
+  8:  'Mountain Lion',
+  9:  'Mavericks',
+  10: 'Yosemite',
+  11: 'El Capitan',
+  12: 'Sierra',
+  13: 'High Sierra',
+  14: 'Mojave',
+  15: 'Catalina'
+  }
+
+_MICROSOFT_FORMATS_LIST = [
+  {'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'ext': '.docx'},
+  {'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'ext': '.dotx'},
+  {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'ext': '.pptx'},
+  {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.template', 'ext': '.potx'},
+  {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'ext': '.xlsx'},
+  {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'ext': '.xltx'},
+  {'mime': 'application/msword', 'ext': '.doc'},
+  {'mime': 'application/msword', 'ext': '.dot'},
+  {'mime': 'application/vnd.ms-powerpoint', 'ext': '.ppt'},
+  {'mime': 'application/vnd.ms-powerpoint', 'ext': '.pot'},
+  {'mime': 'application/vnd.ms-excel', 'ext': '.xls'},
+  {'mime': 'application/vnd.ms-excel', 'ext': '.xlt'}
+  ]
 
 DOCUMENT_FORMATS_MAP = {
   'csv': [{'mime': 'text/csv', 'ext': '.csv'}],
@@ -430,6 +449,11 @@ DOCUMENT_FORMATS_MAP = {
                  {'mime': 'application/vnd.oasis.opendocument.text', 'ext': '.odt'}],
   }
 
+REFRESH_PERM_ERRORS = [
+  'invalid_grant: reauth related error (rapt_required)', # no way to reauth today
+  'invalid_grant: Token has been expired or revoked.',
+  ]
+
 DNS_ERROR_CODES_MAP = {
   1: 'DNS Query Format Error',
   2: 'Server failed to complete the DNS request',
@@ -534,17 +558,21 @@ CROS_ARGUMENT_TO_PROPERTY_MAP = {
   'annotateduser': ['annotatedUser',],
   'asset': ['annotatedAssetId',],
   'assetid': ['annotatedAssetId',],
+  'autoupdateexpiration': ['autoUpdateExpiration',],
   'bootmode': ['bootMode',],
   'cpustatusreports': ['cpuStatusReports',],
   'devicefiles': ['deviceFiles',],
   'deviceid': ['deviceId',],
+  'dockmacaddress': ['dockMacAddress',],
   'diskvolumereports': ['diskVolumeReports',],
   'ethernetmacaddress': ['ethernetMacAddress',],
+  'ethernetmacaddress0': ['ethernetMacAddress0',],
   'firmwareversion': ['firmwareVersion',],
   'lastenrollmenttime': ['lastEnrollmentTime',],
   'lastsync': ['lastSync',],
   'location': ['annotatedLocation',],
   'macaddress': ['macAddress',],
+  'manufacturedate': ['manufactureDate',],
   'meid': ['meid',],
   'model': ['model',],
   'notes': ['notes',],
@@ -586,12 +614,16 @@ CROS_SCALAR_PROPERTY_PRINT_ORDER = [
   'osVersion',
   'bootMode',
   'meid',
+  'dockMacAddress',
   'ethernetMacAddress',
+  'ethernetMacAddress0',
   'macAddress',
   'systemRamTotal',
   'lastEnrollmentTime',
   'orderNumber',
+  'manufactureDate',
   'supportEndDate',
+  'autoUpdateExpiration',
   'guessedAUEDate',
   'guessedAUEModel',
   'tpmVersionInfo',
@@ -757,8 +789,6 @@ GM_Globals = {
 #
 # Global variables defined by environment variables/signal files
 #
-# When retrieving lists of Google Drive activities from API, how many should be retrieved in each chunk
-GC_ACTIVITY_MAX_RESULTS = 'activity_max_results'
 # Automatically generate gam batch command if number of users specified in gam users xxx command exceeds this number
 # Default: 0, don't automatically generate gam batch commands
 GC_AUTO_BATCH_MIN = 'auto_batch_min'
@@ -780,19 +810,15 @@ GC_CUSTOMER_ID = 'customer_id'
 GC_DEBUG_LEVEL = 'debug_level'
 # ID Token decoded from OAuth 2.0 refresh token response. Includes hd (domain) and email of authorized user
 GC_DECODED_ID_TOKEN = 'decoded_id_token'
-# When retrieving lists of ChromeOS/Mobile devices from API, how many should be retrieved in each chunk
-GC_DEVICE_MAX_RESULTS = 'device_max_results'
 # Domain obtained from gam.cfg or oauth2.txt
 GC_DOMAIN = 'domain'
 # Google Drive download directory
 GC_DRIVE_DIR = 'drive_dir'
-# When retrieving lists of Drive files/folders from API, how many should be retrieved in each chunk
-GC_DRIVE_MAX_RESULTS = 'drive_max_results'
-# When retrieving lists of Google Group members from API, how many should be retrieved in each chunk
-GC_MEMBER_MAX_RESULTS = 'member_max_results'
 # If no_browser is False, writeCSVfile won't open a browser when todrive is set
 # and doRequestOAuth prints a link and waits for the verification code when oauth2.txt is being created
 GC_NO_BROWSER = 'no_browser'
+# oauth_browser forces usage of web server OAuth flow that proved problematic.
+GC_OAUTH_BROWSER = 'oauth_browser'
 # Disable GAM API caching
 GC_NO_CACHE = 'no_cache'
 # Disable GAM update check
@@ -811,8 +837,6 @@ GC_SHOW_COUNTS_MIN = 'show_counts_min'
 GC_SHOW_GETTINGS = 'show_gettings'
 # GAM config directory containing json discovery files
 GC_SITE_DIR = 'site_dir'
-# When retrieving lists of Users from API, how many should be retrieved in each chunk
-GC_USER_MAX_RESULTS = 'user_max_results'
 # CSV Columns GAM should show on CSV output
 GC_CSV_HEADER_FILTER = 'csv_header_filter'
 # CSV Rows GAM should filter
@@ -826,7 +850,6 @@ GC_CA_FILE = 'ca_file'
 
 tls_min = "TLSv1_2" if hasattr(ssl.SSLContext(), "minimum_version") else None
 GC_Defaults = {
-  GC_ACTIVITY_MAX_RESULTS: 100,
   GC_AUTO_BATCH_MIN: 0,
   GC_BATCH_SIZE: 50,
   GC_CACHE_DIR: '',
@@ -837,22 +860,19 @@ GC_Defaults = {
   GC_CUSTOMER_ID: MY_CUSTOMER,
   GC_DEBUG_LEVEL: 0,
   GC_DECODED_ID_TOKEN: '',
-  GC_DEVICE_MAX_RESULTS: 100,
   GC_DOMAIN: '',
   GC_DRIVE_DIR: '',
-  GC_DRIVE_MAX_RESULTS: 1000,
-  GC_MEMBER_MAX_RESULTS: 200,
   GC_NO_BROWSER: False,
   GC_NO_CACHE: False,
   GC_NO_UPDATE_CHECK: False,
   GC_NUM_THREADS: 25,
+  GC_OAUTH_BROWSER: False,
   GC_OAUTH2_TXT: _FN_OAUTH2_TXT,
   GC_OAUTH2SERVICE_JSON: _FN_OAUTH2SERVICE_JSON,
   GC_SECTION: '',
   GC_SHOW_COUNTS_MIN: 0,
   GC_SHOW_GETTINGS: True,
   GC_SITE_DIR: '',
-  GC_USER_MAX_RESULTS: 500,
   GC_CSV_HEADER_FILTER: '',
   GC_CSV_ROW_FILTER: '',
   GC_TLS_MIN_VERSION: tls_min,
@@ -877,7 +897,6 @@ GC_VAR_TYPE = 'type'
 GC_VAR_LIMITS = 'lmit'
 
 GC_VAR_INFO = {
-  GC_ACTIVITY_MAX_RESULTS: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 500)},
   GC_AUTO_BATCH_MIN: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (0, None)},
   GC_BATCH_SIZE: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 1000)},
   GC_CACHE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
@@ -888,22 +907,19 @@ GC_VAR_INFO = {
   GC_CUSTOMER_ID: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_DEBUG_LEVEL: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (0, None)},
   GC_DECODED_ID_TOKEN: {GC_VAR_TYPE: GC_TYPE_STRING},
-  GC_DEVICE_MAX_RESULTS: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 1000)},
   GC_DOMAIN: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_DRIVE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
-  GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 1000)},
-  GC_MEMBER_MAX_RESULTS: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 10000)},
   GC_NO_BROWSER: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_NO_CACHE: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_NO_UPDATE_CHECK: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_NUM_THREADS: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, None)},
+  GC_OAUTH_BROWSER: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_OAUTH2_TXT: {GC_VAR_TYPE: GC_TYPE_FILE},
   GC_OAUTH2SERVICE_JSON: {GC_VAR_TYPE: GC_TYPE_FILE},
   GC_SECTION: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_SHOW_COUNTS_MIN: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (0, None)},
   GC_SHOW_GETTINGS: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_SITE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
-  GC_USER_MAX_RESULTS: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 500)},
   GC_CSV_HEADER_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
   GC_CSV_ROW_FILTER: {GC_VAR_TYPE: GC_TYPE_ROWFILTER},
   GC_TLS_MIN_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},
@@ -937,58 +953,8 @@ MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = 'Cowardly refusing to perform migration
 MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = 'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.'
 MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}. Please make sure service is enabled for user and run\n\ngam user <user> check serviceaccount\n\nfor further instructions'
 MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON = 'Please run\n\ngam create project\ngam user <user> check serviceaccount\n\nto create and configure a service account.'
-# oauth errors
-OAUTH2_TOKEN_ERRORS = [
-  'access_denied',
-  'access_denied: Requested client not authorized',
-  'internal_failure: Backend Error',
-  'internal_failure: None',
-  'invalid_grant',
-  'invalid_grant: Bad Request',
-  'invalid_grant: Invalid email or User ID',
-  'invalid_grant: Not a valid email',
-  'invalid_grant: Invalid JWT: No valid verifier found for issuer',
-  'invalid_request: Invalid impersonation prn email address',
-  'unauthorized_client: Client is unauthorized to retrieve access tokens using this method',
-  'unauthorized_client: Client is unauthorized to retrieve access tokens using this method, or client not authorized for any of the scopes requested',
-  'unauthorized_client: Unauthorized client or scope in request',
-  ]
-#
-# callGAPI throw reasons
-GAPI_ABORTED = 'aborted'
-GAPI_AUTH_ERROR = 'authError'
-GAPI_BACKEND_ERROR = 'backendError'
-GAPI_BAD_REQUEST = 'badRequest'
-GAPI_CONDITION_NOT_MET = 'conditionNotMet'
-GAPI_CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
-GAPI_DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
-GAPI_DOMAIN_NOT_FOUND = 'domainNotFound'
-GAPI_DUPLICATE = 'duplicate'
-GAPI_FAILED_PRECONDITION = 'failedPrecondition'
-GAPI_FORBIDDEN = 'forbidden'
-GAPI_GROUP_NOT_FOUND = 'groupNotFound'
-GAPI_INTERNAL_ERROR = 'internalError'
-GAPI_INVALID = 'invalid'
-GAPI_INVALID_ARGUMENT = 'invalidArgument'
-GAPI_INVALID_MEMBER = 'invalidMember'
-GAPI_MEMBER_NOT_FOUND = 'memberNotFound'
-GAPI_NOT_FOUND = 'notFound'
-GAPI_NOT_IMPLEMENTED = 'notImplemented'
-GAPI_PERMISSION_DENIED = 'permissionDenied'
-GAPI_QUOTA_EXCEEDED = 'quotaExceeded'
-GAPI_RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
-GAPI_RESOURCE_NOT_FOUND = 'resourceNotFound'
-GAPI_SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
-GAPI_SYSTEM_ERROR = 'systemError'
-GAPI_USER_NOT_FOUND = 'userNotFound'
-GAPI_USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
-#
-GAPI_DEFAULT_RETRY_REASONS = [GAPI_QUOTA_EXCEEDED, GAPI_RATE_LIMIT_EXCEEDED, GAPI_USER_RATE_LIMIT_EXCEEDED, GAPI_BACKEND_ERROR, GAPI_INTERNAL_ERROR]
-GAPI_GMAIL_THROW_REASONS = [GAPI_SERVICE_NOT_AVAILABLE]
-GAPI_GROUP_GET_THROW_REASONS = [GAPI_GROUP_NOT_FOUND, GAPI_DOMAIN_NOT_FOUND, GAPI_DOMAIN_CANNOT_USE_APIS, GAPI_FORBIDDEN, GAPI_BAD_REQUEST]
-GAPI_GROUP_GET_RETRY_REASONS = [GAPI_INVALID, GAPI_SYSTEM_ERROR]
-GAPI_MEMBERS_THROW_REASONS = [GAPI_GROUP_NOT_FOUND, GAPI_DOMAIN_NOT_FOUND, GAPI_DOMAIN_CANNOT_USE_APIS, GAPI_INVALID, GAPI_FORBIDDEN]
-GAPI_MEMBERS_RETRY_REASONS = [GAPI_SYSTEM_ERROR]
+MESSAGE_UPDATE_GAM_TO_64BIT = "You're running a 32-bit version of GAM on a 64-bit version of Windows, upgrade to a windows-x86_64 version of GAM"
+MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from %s by %s'
 
 USER_ADDRESS_TYPES = ['home', 'work', 'other']
 USER_EMAIL_TYPES = ['home', 'work', 'other']
@@ -1199,3 +1165,19 @@ LANGUAGE_CODES_MAP = {
   'ur': 'ur', 'uz': 'uz', 'vi': 'vi', 'wo': 'wo', 'xh': 'xh', 'yi': 'yi', 'yo': 'yo', #Urdu, Uzbek, Vietnamese, Wolof, Xhosa, Yiddish, Yoruba
   'zh-cn': 'zh-CN', 'zh-hk': 'zh-HK', 'zh-tw': 'zh-TW', 'zu': 'zu', #Chinese (Simplified), Chinese (Hong Kong/Traditional), Chinese (Taiwan/Traditional), Zulu
   }
+
+# maxResults exception values for API list calls. Should only be listed if:
+#   - discovery doc does not specify maximum value (we use maximum value if it
+#     exists, not this)
+#   - actual max API returns with maxResults=<bigNum>  > default API returns
+#     when maxResults isn't specified (we should use default otherwise by not
+#     setting maxResults)
+
+MAX_RESULTS_API_EXCEPTIONS = {
+  'calendar.acl.list': 250,
+  'calendar.calendarList.list': 250,
+  'calendar.events.list': 2500,
+  'calendar.settings.list': 250,
+  'directory.chromeosdevices.list': 200,
+  'drive.files.list': 1000,
+  }

src/windows-gam.spec

@@ -1,9 +1,13 @@
 # -*- mode: python -*-
+import sys
+
+sys.modules['FixTk'] = None
+
 a = Analysis(['gam.py'],
              pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
              hiddenimports=[],
              hookspath=None,
-             excludes=['_tkinter'],
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
              runtime_hooks=None)
 for d in a.datas:
     if 'pyconfig' in d[0]: