Handle TYPE_MESSAGE fields with durations or counts as a special case (#1375)

* Handle TYPE_MESSAGE fields with durations or counts as a special case

* Allow schema TYPE_ENUM field values with/without common prefix

.github/ISSUE_TEMPLATE.txt

@@ -5,7 +5,7 @@ Please confirm the following:
 * I am typing the command as described in the GAM Wiki at https://github.com/jay0lee/gam/wiki
 
 Full steps to reproduce the issue:
-1. 
+1.
 2.
 3.
 

.github/ISSUE_TEMPLATE/aa-question.md

@@ -0,0 +1,14 @@
+---
+name: Question about using GAM
+about: Help with using GAM or running it for the first time
+title: Please use the GAM discussion group
+labels: invalid
+assignees: ''
+
+---
+
+If you need help with GAM, please do not file an issue here, it will be closed and ignored.
+
+Please post your question to the GAM discussion group where other admins are ready and willing to help:
+
+https://groups.google.com/g/google-apps-manager

.github/ISSUE_TEMPLATE/za-bug-report.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: jay0lee
+
+---
+
+The issue tracker is for reporting product deficiencies. "How do I?" questions should be posted to the discussion forum at https://groups.google.com/group/google-apps-manager. When in doubt, start at the discussion forum and return here only when instructed to do so.
+
+Please confirm the following:
+* I have upgraded to the latest GAM release from https://git.io/gamreleases and I still have this issue.
+* I am typing the command as described in the GAM Wiki at https://github.com/jay0lee/gam/wiki
+
+Full steps to reproduce the issue:
+1.
+2.
+3.
+
+Expected outcome (what are you trying to do?):
+
+Actual outcome (what errors or bad behavior do you see instead?):

.github/ISSUE_TEMPLATE/zz-feature-request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for GAM
+title: ''
+labels: enhancement
+assignees: jay0lee
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/actions/decrypt.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+gpgfile="$1"
+echo "source file is ${gpgfile}"
+credsfile="$2"
+echo "target file is ${credsfile}"
+if [ -z ${PASSCODE+x} ]; then
+  echo "PASSCODE is unset";
+else
+  echo "PASSCODE is set";
+fi
+
+gpg --quiet --batch --yes --decrypt --passphrase="${PASSCODE}" \
+    --output "${credsfile}" "${gpgfile}"
+
+tar xf "${credsfile}" --directory "${gampath}"

.github/actions/linux-before-install.sh

@@ -0,0 +1,113 @@
+echo "RUNNING: apt update..."
+sudo apt-get -qq --yes update > /dev/null
+sudo apt-get -qq --yes install swig libpcsclite-dev
+if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
+  export python="python"
+  export pip="pip"
+  echo "Travis setup Python $TRAVIS_PYTHON_VERSION"
+  echo "running tests with this version"
+else
+  export whereibelong=$(pwd)
+  echo "We are running on $ImageOS $ImageVersion"
+  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+  cpucount=$(nproc --all)
+  echo "This device has $cpucount CPUs for compiling..."
+  SSLVER=$(~/ssl/bin/openssl version)
+  SSLRESULT=$?
+  PYVER=$(~/python/bin/python3 -V)
+  PYRESULT=$?
+  if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+    echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+    if [ $SSLRESULT -ne 0 ]; then
+      echo "sslresult -ne 0"
+    fi
+    if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+      echo "sslver not equal to..."
+    fi
+    if [ $PYRESULT -ne 0 ]; then
+      echo "pyresult -ne 0"
+    fi
+    if [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION" ]]; then
+      echo "pyver not equal to..."
+    fi
+    cd ~
+    rm -rf ssl
+    rm -rf python
+    mkdir ssl
+    mkdir python
+    echo "RUNNING: apt upgrade..."
+    sudo apt-mark hold openssh-server
+    sudo apt-get --yes upgrade
+    sudo apt-get --yes --with-new-pkgs upgrade
+    echo "Installing build tools..."
+    sudo apt-get -qq --yes install build-essential
+    echo "Installing deps for python3"
+    sudo cp -v /etc/apt/sources.list /tmp
+    sudo chmod a+rwx /tmp/sources.list
+    echo "deb-src http://archive.ubuntu.com/ubuntu/ $TRAVIS_DIST main" >> /tmp/sources.list
+    sudo cp -v /tmp/sources.list /etc/apt
+    sudo apt-get -qq --yes update > /dev/null
+    sudo apt-get -qq --yes build-dep python3 > /dev/null
+
+    # Compile latest OpenSSL
+    wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+    echo "Extracting OpenSSL..."
+    tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+    cd openssl-$BUILD_OPENSSL_VERSION
+    echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+    ./config shared --prefix=$HOME/ssl
+    echo "Running make for OpenSSL..."
+    make -j$cpucount -s
+    echo "Running make install for OpenSSL..."
+    make install > /dev/null
+    cd ~
+
+    # Compile latest Python
+    echo "Downloading Python $BUILD_PYTHON_VERSION..."
+    curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+    echo "Extracting Python..."
+    tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+    cd Python-$BUILD_PYTHON_VERSION
+    echo "Compiling Python $BUILD_PYTHON_VERSION..."
+    safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+    unsafe_flags="--enable-optimizations --with-lto"
+    if [ ! -e Makefile ]; then
+      echo "running configure with safe and unsafe"
+      ./configure $safe_flags $unsafe_flags > /dev/null
+    fi
+    #make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+    make -j$cpucount -s
+    RESULT=$?
+    echo "First make exited with $RESULT"
+    if [ $RESULT != 0 ]; then
+      echo "Trying Python compile again without unsafe flags..."
+      make clean
+      ./configure $safe_flags > /dev/null
+      make -j$cpucount -s
+      echo "Sticking with safe Python for now..."
+    fi
+    echo "Installing Python..."
+    make install > /dev/null
+    cd ~
+  fi
+
+  python=~/python/bin/python3
+  pip=~/python/bin/pip3
+
+  if ([ "${ImageOS}" == "ubuntu16" ]) && [ "${HOSTTYPE}" == "x86_64" ]; then
+    echo "Installing deps for StaticX..."
+    if [ ! -d patchelf-$PATCHELF_VERSION ]; then
+      echo "Downloading PatchELF $PATCHELF_VERSION"
+      wget https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz 
+      tar xf $PATCHELF_VERSION.tar.gz
+      cd patchelf-$PATCHELF_VERSION/
+      ./bootstrap.sh
+      ./configure
+      make
+      sudo make install
+    fi
+    $pip install staticx
+  fi
+
+  cd $whereibelong
+fi

.github/actions/linux-install.sh

@@ -0,0 +1,32 @@
+export gampath="dist/gam"
+rm -rf $gampath
+mkdir -p $gampath
+export gampath=$(readlink -e $gampath)
+$python -OO -m PyInstaller --clean --noupx --strip -F --distpath $gampath gam.spec
+export gam="${gampath}/gam"
+export GAMVERSION=`$gam version simple`
+cp LICENSE $gampath
+cp GamCommands.txt $gampath
+this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+GAM_ARCHIVE="gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-glibc${this_glibc_ver}.tar.xz"
+rm $gampath/lastupdatecheck.txt
+# tar will cd to dist and tar up gam/
+tar -C dist/ --create --file $GAM_ARCHIVE --xz gam
+echo "PyInstaller GAM info:"
+du -h $gam
+time $gam version extended
+if ([ "${ImageOS}" == "ubuntu16" ]) && [ "${HOSTTYPE}" == "x86_64" ]; then
+  GAM_LEGACY_ARCHIVE=gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-legacy.tar.xz
+  $python -OO -m staticx -l /lib/x86_64-linux-gnu/libresolv.so.2 -l /lib/x86_64-linux-gnu/libnss_dns.so.2 $gam $gam-staticx
+  strip $gam-staticx
+  rm $gampath/gam
+  mv $gam-staticx $gam
+  chmod 755 $gam
+  rm $gampath/lastupdatecheck.txt
+  tar -C dist/ --create --file $GAM_LEGACY_ARCHIVE --xz gam
+  echo "Legacy StaticX GAM info:"
+  du -h $gam
+  time $gam version extended
+fi
+echo "GAM packages:"
+ls -l gam-*.tar.xz

.github/actions/macos-before-install.sh

@@ -0,0 +1,136 @@
+mypath=$HOME
+whereibelong=$(pwd)
+cpucount=$(sysctl -n hw.ncpu)
+echo "This device has $cpucount CPUs for compiling..."
+
+#echo "Brew installing xz..."
+#brew install xz > /dev/null
+
+#brew upgrade
+
+brew install coreutils
+brew install bash
+
+# prefer standard GNU tools like date over MacOS defaults
+export PATH="/usr/local/opt/coreutils/libexec/gnubin:$(brew --prefix)/opt/gnu-tar/libexec/gnubin:$PATH"
+
+date --version
+gdate --version
+bash --version
+
+cd ~
+
+# Use official Python.org version of Python which is backwards compatible
+# with older MacOS versions
+if [ "$PLATFORM" == "x86_64" ]; then
+  export pyfile=python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
+else
+  export pyfile=python-$BUILD_PYTHON_VERSION-macos11.pkg
+fi
+
+wget https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$pyfile
+echo "installing Python $BUILD_PYTHON_VERSION..."
+sudo installer -pkg ./$pyfile -target /
+
+# This fixes https://github.com/pyinstaller/pyinstaller/issues/5062
+codesign --remove-signature /Library/Frameworks/Python.framework/Versions/3.9/Python
+
+#if [ ! -f python-$MIN_PYTHON_VERSION-macosx10.9.pkg ]; then
+#  wget --quiet https://www.python.org/ftp/python/$MIN_PYTHON_VERSION/python-$MIN_PYTHON_VERSION-macosx10.9.pkg
+#fi
+#sudo installer -pkg python-$MIN_PYTHON_VERSION-macosx10.9.pkg -target /
+
+#brew install openssl@1.1
+#brew upgrade python
+
+#export python=python3
+#export pip=pip3
+
+#echo "Python location:"
+#which $python
+
+cd ~
+
+#export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+#export openssl=~/ssl/bin/openssl
+#export python=~/python/bin/python3
+#export pip=~/python/bin/pip3
+
+export python=/usr/local/bin/python3
+export pip=/usr/local/bin/pip3
+SSLVER=$($openssl version)
+SSLRESULT=$?
+PYVER=$($python -V)
+PYRESULT=$?
+
+brew install swig
+$pip install pyscard
+
+#wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
+
+#if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+#  echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+#  if [ $SSLRESULT -ne 0 ]; then
+#    echo "sslresult -ne 0"
+#  fi
+#  if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+#    echo "sslver not equal to..."
+#  fi
+#  if [ $PYRESULT -ne 0 ]; then
+#    echo "pyresult -ne 0"
+#  fi
+#  if [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION" ]]; then
+#    echo "pyver not equal to..."
+#  fi
+
+  # Start clean
+ # rm -rf python
+ # rm -rf ssl
+ # mkdir python
+ # mkdir ssl
+
+  # Compile latest OpenSSL
+#  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+#  echo "Extracting OpenSSL..."
+#  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+#  cd openssl-$BUILD_OPENSSL_VERSION
+#  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+#  ./config shared --prefix=$HOME/ssl
+#  echo "Running make for OpenSSL..."
+#  make -j$cpucount -s
+#  echo "Running make install for OpenSSL..."
+#  make install > /dev/null
+#  cd ~
+
+  # Compile latest Python
+#  echo "Downloading Python $BUILD_PYTHON_VERSION..."
+#  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+#  echo "Extracting Python..."
+#  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+#  cd Python-$BUILD_PYTHON_VERSION
+#  echo "Compiling Python $BUILD_PYTHON_VERSION..."
+#  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+#  unsafe_flags="--enable-optimizations --with-lto"
+#  if [ ! -e Makefile ]; then
+#    echo "running configure with safe and unsafe"
+#    ./configure $safe_flags $unsafe_flags > /dev/null
+#  fi
+#  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+#  RESULT=$?
+#  echo "First make exited with $RESULT"
+#  if [ $RESULT != 0 ]; then
+#    echo "Trying Python compile again without unsafe flags..."
+#    make clean
+#    ./configure $safe_flags > /dev/null
+#    make -j$cpucount -s
+#    echo "Sticking with safe Python for now..."
+#  fi
+#  echo "Installing Python..."
+#  make install > /dev/null
+#  cd ~
+#fi
+
+$python -V
+
+cd $whereibelong
+

.github/actions/macos-install.sh

@@ -0,0 +1,18 @@
+echo "MacOS Version Info According to Python:"
+python -c "import platform; print(platform.mac_ver())"
+echo "Xcode versionn:"
+xcodebuild -version
+export gampath=dist/gam
+rm -rf $gampath
+export specfile="gam.spec"
+$python -OO -m PyInstaller --clean --noupx --strip -F --distpath "${gampath}" "${specfile}"
+export gam="${gampath}/gam"
+$gam version extended
+export GAMVERSION=`$gam version simple`
+cp LICENSE "${gampath}"
+cp GamCommands.txt "${gampath}"
+MACOSVERSION=$(defaults read loginwindow SystemVersionStampAsString)
+GAM_ARCHIVE="gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-MacOS${MACOSVERSION}.tar.xz"
+rm "${gampath}/lastupdatecheck.txt"
+# tar will cd to dist/ and tar up gam/
+tar -C dist/ --create --file $GAM_ARCHIVE --xz gam

.github/actions/windows-before-install.sh

@@ -0,0 +1,57 @@
+if [[ "$PLATFORM" == "x86_64" ]]; then
+  export BITS="64"
+  export PYTHONFILE_BITS="-amd64"
+  export OPENSSL_BITS="-x64"
+elif [[ "$PLATFORM" == "x86" ]]; then
+  export BITS="32"
+  export PYTHONFILE_BITS=""
+  export OPENSSL_BITS=""
+  export CHOCOPTIONS="--forcex86"
+fi
+echo "This is a ${BITS}-bit build for ${PLATFORM}"
+
+export mypath=$(pwd)
+cd ~
+
+export python="python"
+export pip="pip"
+
+# pyscard needs swig, keep these two together
+choco install $CHOCOPTIONS swig
+$pip install pyscard
+
+# Python
+#echo "Installing Python..."
+#export python_file=python-${BUILD_PYTHON_VERSION}${PYTHONFILE_BITS}.exe
+#if [ ! -e $python_file ]; then
+#  echo "Downloading $python_file..."
+#  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
+#fi
+#until ./${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python; do echo "trying python again..."; done
+#export python=/c/python/python.exe
+#export pip=/c/python/scripts/pip.exe
+#until [ -f $python ]; do sleep 1; done
+#export PATH=$PATH:/c/python/scripts
+
+# OpenSSL
+#echo "Installing OpenSSL..."
+#export exefile=Win${BITS}OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
+#if [ ! -e $exefile ]; then
+#  echo "Downloading $exefile..."
+#  curl -O https://slproweb.com/download/$exefile
+#fi
+#until ./${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl; do echo "trying openssl again..."; done
+#until cp -v /c/ssl/libcrypto-1_1${OPENSSL_BITS}.dll /c/python/DLLs/; do echo "trying libcrypto copy again..."; sleep 3; done
+#until cp -v /c/ssl/libssl-1_1${OPENSSL_BITS}.dll /c/python/DLLs/; do echo "trying libssl copy again..."; done
+#if [[ "$PLATFORM" == "x86_64" ]]; then
+#  cp -v /c/python/DLLs/libssl-1_1-x64.dll /c/python/DLLs/libssl-1_1.dll
+#  cp -v /c/python/DLLs/libcrypto-1_1-x64.dll /c/python/DLLs/libcrypto-1_1.dll
+#fi
+
+cd $mypath
+
+echo "PATH: $PATH"
+cd ..
+$python setup.py install
+echo "cd to $mypath"
+cd $mypath

.github/actions/windows-install.sh

@@ -0,0 +1,29 @@
+if [[ "$PLATFORM" == "x86_64" ]]; then
+  export WIX_BITS="x64"
+elif [[ "$PLATFORM" == "x86" ]]; then
+  export WIX_BITS="x86"
+fi
+echo "compiling GAM with pyinstaller..."
+export gampath="dist/gam"
+rm -rf $gampath
+mkdir -p $gampath
+export gampath=$(readlink -e $gampath)
+pyinstaller --clean --noupx -F --distpath $gampath gam.spec
+export gam="${gampath}/gam"
+echo "running compiled GAM..."
+$gam version
+export GAMVERSION=$($gam version simple)
+rm $gampath/lastupdatecheck.txt
+cp LICENSE $gampath
+cp GamCommands.txt $gampath
+cp gam-setup.bat $gampath
+GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
+/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE $gampath -xr!.svn
+
+echo "Running WIX candle $WIX_BITS..."
+/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch $WIX_BITS gam.wxs
+echo "Done with WIX candle..."
+echo "Running WIX light..."
+/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
+echo "Done with WIX light..."
+rm *.wixpdb

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - pinned
+  - enhancement
+  - help wanted
+  - security
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/build.yml

@@ -0,0 +1,346 @@
+name: Build and test GAM
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '37 22 * * *'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: src
+
+env:
+  BUILD_PYTHON_VERSION: "3.9.5"
+  MIN_PYTHON_VERSION: "3.9.5"
+  BUILD_OPENSSL_VERSION: "1.1.1k"
+  MIN_OPENSSL_VERSION: "1.1.1k"
+  PATCHELF_VERSION: "0.12"
+  # PYINSTALLER_VERSION can be full commit hash or version like v4.20
+  PYINSTALLER_VERSION: "e20e74c03768d432d48665b8ef1e02511b16e4be"
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-16.04
+            jid: 1
+            goal: "build"
+            gamos: "linux"
+            platform: "x86_64"
+          - os: ubuntu-18.04
+            jid: 2
+            goal: "build"
+            gamos: "linux"
+            platform: "x86_64"
+          - os: ubuntu-20.04
+            jid: 3
+            goal: "build"
+            gamos: "linux"
+            platform: "x86_64"
+#          - os: [self-hosted, linux, ARM]
+#            jid: 10
+#            goal: "build"
+#            gamos: "linux"
+#            platform: "arm"
+#          - os: [self-hosted, linux, ARM64]
+#            jid: 11
+#            goal: "build"
+#            gamos: "linux"
+#            platform: "arm64"
+          - os: macos-10.15
+            jid: 4
+            goal: "build"
+            gamos: "macos"
+            platform: "x86_64"
+          - os: macos-11.0
+            jid: 12
+            goal: "build"
+            gamos: "macos"
+            platform: "universal2"
+          - os: windows-2019
+            jid: 5
+            goal: "build"
+            gamos: "windows"
+            python: 3.9.5 
+            pyarch: "x64" 
+            platform: "x86_64"
+          - os: windows-2019
+            jid: 6
+            goal: "build"
+            gamos: "windows"
+            platform: "x86"
+            python: 3.9.5
+            pyarch: "x86"
+          - os: ubuntu-20.04
+            goal: "test"
+            python: "3.6"
+            jid: 7
+            gamos: "linux"
+            platform: "x86_64"
+          - os: ubuntu-20.04
+            goal: "test"
+            python: "3.7"
+            jid: 8
+            gamos: "linux"
+            platform: "x86_64"
+          - os: ubuntu-20.04
+            goal: "test"
+            python: "3.8"
+            jid: 9
+            gamos: "linux"
+            platform: "x86_64"
+
+    steps:
+
+      - uses: actions/checkout@master
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Cache multiple paths
+        uses: actions/cache@v2
+        if: matrix.goal != 'test'
+        with:
+          path: |
+            ~/python
+            ~/ssl
+          key: ${{ matrix.os }}-${{ matrix.jid }}-20210504
+
+      - name: Set env variables
+        env:
+          GAMOS: ${{ matrix.gamos }}
+          GOAL: ${{ matrix.goal }}
+          JID: ${{ matrix.jid }}
+          PLATFORM: ${{ matrix.platform }}
+        run: |
+          echo "GAMOS=${GAMOS}" >> $GITHUB_ENV
+          echo "GOAL=${GOAL}" >> $GITHUB_ENV
+          echo "JID=${JID}" >> $GITHUB_ENV
+          echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV
+          uname -a
+
+      - name: Use pre-compiled Python for testing and Windows
+        if: matrix.python != ''
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: ${{ matrix.pyarch }}
+
+      - name: Set env variables for pre-compiled Python
+        if: matrix.goal == 'test'
+        run: |
+             export python=$(which python3)
+             export pip=$(which pip3)
+             export gam="${python} -m gam"
+             export gampath="$(readlink -e .)"
+             echo -e "python: $python\npip: $pip\ngam: $gam\ngampath: $gampath"
+             echo "python=${python}" >> $GITHUB_ENV
+             echo "pip=${pip}" >> $GITHUB_ENV
+             echo "gam=${gam}" >> $GITHUB_ENV
+             echo "gampath=${gampath}" >> $GITHUB_ENV
+             echo "RUNNING: apt update..."
+             sudo apt-get -qq --yes update > /dev/null
+             sudo apt-get -qq --yes install swig libpcsclite-dev
+
+      - name: Build and install Python, OpenSSL and PyInstaller
+        if: matrix.goal != 'test' && steps.cache-primes.outputs.cache-hit != 'true'
+        run: |
+             set +e
+             source ../.github/actions/${GAMOS}-before-install.sh
+             echo "PATH=$PATH" >> $GITHUB_ENV # keep gnutools for MacOS
+             echo "python=$python" >> $GITHUB_ENV
+             echo "pip=$pip" >> $GITHUB_ENV
+             echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
+             echo -e "Python: $python\nPip: $pip\nLD_LIB...: $LD_LIBRARY_PATH"
+             export url="https://codeload.github.com/pyinstaller/pyinstaller/tar.gz/${PYINSTALLER_VERSION}"
+             echo "Downloading ${url}"
+             curl -o pyinstaller.tar.gz --compressed "${url}"
+             tar xf pyinstaller.tar.gz
+             cd "pyinstaller-${PYINSTALLER_VERSION}/bootloader"
+             if [ "${PLATFORM}" == "x86" ]; then
+               BITS="32"
+             else
+               BITS="64"
+             fi
+             $python ./waf all --target-arch=${BITS}bit
+             cd ..
+             $python setup.py install
+             #$pip install pyinstaller
+
+      - name: Install pip requirements
+        if: matrix.os != 'self-hosted'
+        run: |
+             set +e
+             $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U --force-reinstall
+
+             $pip install --upgrade -r requirements.txt
+
+      - name: Build GAM with PyInstaller
+        if: matrix.goal != 'test'
+        run: |
+             set +e
+             source ../.github/actions/${GAMOS}-install.sh
+             echo "gampath=$gampath" >> $GITHUB_ENV
+             echo "gam=$gam" >> $GITHUB_ENV
+             echo -e "GAM: ${gam}\nGAMPATH: ${gampath}\nGAMVERSION: ${GAMVERSION}"
+
+      - name: Basic Tests all jobs
+        run: |
+             echo -e "python: $python\npip: $pip\ngam: $gam\ngampath: $gampath\n"
+             $python -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
+             touch "${gampath}/nobrowser.txt"
+             $gam version extended
+             export GAMVERSION=$($gam version simple)
+             echo "GAM Version ${GAMVERSION}"
+             echo "GAMVERSION=${GAMVERSION}" >> $GITHUB_ENV
+
+      - name: Basic Tests build jobs only
+        if: matrix.goal != 'test'
+        run: |
+             export vline=$($gam version | grep "Python ")
+             export python_line=($vline)
+             export this_python=${python_line[1]}
+             $python tools/a_atleast_b.py "${this_python}" "${MIN_PYTHON_VERSION}"
+             export vline=$($gam version extended | grep "OpenSSL ")
+             export openssl_line=($vline)
+             export this_openssl="${openssl_line[1]}"
+             $python tools/a_atleast_b.py "${this_openssl}" "${MIN_OPENSSL_VERSION}"
+
+
+      - name: Live API tests push only
+        if: github.event_name == 'push' || github.event_name == 'schedule'
+        env: # Or as an environment variable
+          PASSCODE: ${{ secrets.PASSCODE }}
+        run: |
+             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.gpg creds.tar
+             export OAUTHFILE="oauth2.txt-gam-gha-${JID}"
+             echo "OAUTHFILE=${OAUTHFILE}" >> $GITHUB_ENV
+             export gam_user="gam-gha-${JID}@pdl.jaylee.us"
+             echo "gam_user=${gam_user}" >> $GITHUB_ENV
+             $gam oauth info
+             $gam info domain
+             $gam oauth refresh
+             $gam info user
+             #$gam info user $gam_user grouptree
+             export tstamp=$(date +%s%3N)
+             export newbase=gha-test-$JID-$tstamp
+             export newuser=$newbase@pdl.jaylee.us
+             export newgroup=$newbase-group@pdl.jaylee.us
+             export newalias=$newbase-alias@pdl.jaylee.us
+             export newbuilding=$newbase-building
+             export newresource=$newbase-resource
+             export GAM_THREADS=5
+             echo email > sample.csv;
+             for i in {01..10}; do
+               echo "${newbase}-bulkuser-$i" >> sample.csv;
+             done
+             $gam create user $newuser firstname GHA lastname $JID password random recoveryphone 12125121110 recoveryemail jay0lee@gmail.com gha.jid $JID
+             $gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "GHA test message"
+             $gam user $gam_user sendemail recipient exchange@pdl.jaylee.us subject "test ${tstamp}" message "test message"
+             $gam create group $newgroup name "GHA $JID group" description "This is a description" isarchived true
+             $gam user $newuser add license gsuitebusiness
+             $gam update group $newgroup add owner $gam_user
+             $gam update group $newgroup add member $newuser
+             $gam csv sample.csv gam create user ~~email~~ firstname "GHA Bulk" lastname ~~email~~ gha.jid $JID
+             $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random
+             $gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""
+             $gam csv sample.csv gam user ~email add license gsuitebusiness
+             $gam csv sample.csv gam user $gam_user sendemail recipient ~~email~~@pdl.jaylee.us subject "test message $newbase" message "GHA test message"
+             $gam csv sample.csv gam update group $newgroup add member ~email
+             $gam info group $newgroup
+             $gam info cigroup $newgroup membertree
+             $gam user $gam_user check serviceaccount
+             # confirm mailbox is provisoned before continuing
+             $gam user $newuser waitformailbox
+             $gam user $newuser imap on
+             $gam user $newuser show imap
+             $gam user $newuser show delegates
+             #$gam user $newuser add contactdelegate "${newbase}-bulkuser-01"
+             #$gam user $newuser print contactdelegates
+             export biohazard=$(echo -e '\xe2\x98\xa3')
+             $gam user $newuser label "$biohazard unicode biohazard $biohazard"
+             $gam user $newuser show labels
+             $gam user $newuser show labels > labels.txt
+             $gam user $gam_user importemail subject "GHA import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED
+             $gam user $gam_user insertemail subject "GHA insert $newbase" file gam.py labels INBOX,UNREAD # yep body is gam code
+             $gam user $gam_user sendemail subject "GHA send $gam_user $newbase" file gam.py recipient admin@pdl.jaylee.us
+             $gam user $gam_user draftemail subject "GHA draft $newbase" message "Draft message test"
+             $gam csvfile sample.csv:email waitformailbox
+             $gam user $newuser delegate to "${newbase}-bulkuser-01"
+             $gam users "$gam_user $newbase-bulkuser-01 $newbase-bulkuser-02 $newbase-bulkuser-03" delete messages query in:anywhere maxtodelete 99999 doit
+             $gam users "$newbase-bulkuser-04 $newbase-bulkuser-05 $newbase-bulkuser-06" trash messages query in:anywhere maxtotrash 99999 doit
+             $gam users "$newbase-bulkuser-07 $newbase-bulkuser-08 $newbase-bulkuser-09" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit
+             $gam user $newuser delete label --ALL_LABELS--
+             $gam create feature name Whiteboard-$newbase
+             $gam create feature name VC-$newbase
+             $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."
+             $gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room
+             $gam info resource $newresource
+             $gam user $newuser show filelist
+             $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id # clear ACLs
+             $gam calendar $gam_user update read domain
+             $gam calendar $gam_user update freebusy default
+             $gam calendar $gam_user add editor $newuser
+             $gam calendar $gam_user showacl
+             $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id
+             $gam calendar $gam_user addevent summary "GHA test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all
+             $gam calendar $gam_user printevents after -0d
+             matterid=uid:$($gam create vaultmatter name "GHA matter $newbase" description "test matter" collaborators $newuser | head -1 | cut -d ' ' -f 3)
+             $gam create vaulthold matter $matterid name "GHA hold $newbase" corpus mail accounts $newuser
+             $gam print vaultmatters matterstate open
+             $gam print vaultholds matter $matterid
+             $gam print vaultcount matter $matterid corpus mail everyone todrive
+             $gam create vaultexport matter $matterid name "GHA export $newbase" corpus mail accounts $newuser
+             $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~
+             $gam csv sample.csv gam user ~email add calendar id:$newresource
+             $gam delete resource $newresource
+             $gam delete feature Whiteboard-$newbase
+             $gam delete feature VC-$newbase
+             $gam delete building $newbuilding
+             $gam delete group $newgroup
+             $gam create alias $newalias user $newuser
+             $gam whatis $newuser
+             $gam user $gam_user show tokens
+             $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~
+             $gam delete hold "GHA hold $newbase" matter $matterid
+             $gam update matter $matterid action close
+             $gam update matter $matterid action delete
+             $gam delete user $newuser
+             $gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail
+             $gam print mobile
+             $gam print devices
+             $gam print browsers
+             export sn="$JID$JID$JID$JID-$(openssl rand -base64 32 | sed 's/[^a-zA-Z0-9]//g')"
+             $gam create device serialnumber $sn devicetype android
+             $gam print cros allfields nolists
+             $gam report usageparameters customer
+             $gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins
+             $gam report customer todrive
+             $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive
+             $gam report admin start -3d todrive
+             $gam print devices nopersonaldevices nodeviceusers filter "serial:$JID$JID$JID$JID-" | $gam csv - gam delete device id ~name
+             $gam print userinvitations
+             $gam print userinvitations | $gam csv - gam send userinvitation ~name
+             export CUSTOMER_ID="C01wfv983"
+             export GA_DOMAIN="pdl.jaylee.us"
+             touch $gampath/enabledasa.txt
+             echo "printer model count:"
+             $gam print printermodels | wc -l
+             #$gam print printers
+             #$gam create printer displayname "${newbase}" uri ipp://localhost:631 driverless description "made by $(date)"
+             rm $gampath/enabledasa.txt
+
+      - name: Upload to Google Drive, build only.
+        if: github.event_name == 'push' && matrix.goal != 'test'
+        run: |
+             ls gam-$GAMVERSION-*
+             for gamfile in gam-$GAMVERSION-*; do
+               echo "Uploading file ${gamfile} to Google Drive..."
+               fileid=$($gam user $gam_user add drivefile localfile $gamfile drivefilename $GAMVERSION-${GITHUB_SHA:0:7}-$gamfile parentid 1N2zbO33qzUQFsGM49-m9AQC1ijzd_ru1 returnidonly)
+               echo "file uploaded as ${fileid}, setting ACL..."
+               $gam user $gam_user add drivefileacl $fileid anyone role reader withlink
+             done

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+    python: python3.7
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: double-quote-string-fixer
+    -   id: check-yaml
+    -   id: check-docstring-first
+    -   id: name-tests-test
+    -   id: requirements-txt-fixer
+    -   id: check-merge-conflict
+
+-   repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.30.0
+    hooks:
+    -   id: yapf
+        args: [--style=google, --in-place]
+
+-   repo: https://github.com/PyCQA/pylint
+    rev: pylint-2.5.0
+    hooks:
+    -   id: pylint
+        args: [--output-format=colorized]

.travis.yml

@@ -1,213 +0,0 @@
-if: tag IS blank
-
-env:
-  global:
-    - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
-    - secure: "J9380tGLOZWa7dSH1y5Il8T5JQpN6ad81gI6VR1HIU0svpRdjgikyDA7ca2MKYDUYYY9yVSkTV6gCl6iIU/9+SKaYugpP+tkvdGYkC2moJdcTgYM/WOnIK9ExQ3BPhN1neGxJjPTwKo1ft27mtZ2I5vuCiBwIcnKWLnKPyW3PD+mWpfqiLuEzkHoAh6G3jC4qbcCrZDeX/knE+PzqESUEi+8k1G8gYcSDWujba9ypSsqZ8T/MXagGla6l7y2Rz+/KZTJmFHwKAA10V+xPLVqxoiqi4ar66yUqy0BamwRXPcseI+ns3Q+4lUpMqVQ5GlRy7LF1xC8myjmcAexXk0F9hg+CMzewKI8UgmQH/ZJvQZEh8s6mW26+CqA4d3zMQkWaR0WtEtpiuH7AGHCflIqvEQ6UiG7ia3B8iZfW2wl0j/kqx4OuHkS3r0pWKVVIIvCj9Ow2BHP7SpiV1AcUGsVxzwbgTh67fitna3Z3c6Uj8ccQlNr7ZIt1az6Wf3w5njijkLOiBpQSLKunTTCTSge/JzBTKUcie3RE9vzirl58gUxAt36nDtPWnory+RttMZrOkBVbTeSxp+IUe8pNwLFPHABsafXsjkfzBOtFmm+0ZXWt2Rlog5NvlemJfQUWDlsL4g+BSakzN+4sIPKzSauWDHyaEeULY7Uprkil6c5zwo="
-    - secure: "szcjWHPr0Bf1KCkyTrV5Fu3ADhWk+pg8YWucjXHdybmhaQIKG7iBNg8LJ5d0OBTwAg31wK4ZgyLVSa2gKrAZ3UeDjykJFsR711xDSQOod51Wrgqu4FbXDewE817DUk3Cwe1l5DCu3/fjEw4vbm8B/qb7iMTRKCq6hJd97FwT5oauP0QHNPer9JjrW4F0Hk9ttkgEU2dXWvBMsTJsDOGNI3ddABE2HskxV4T4thelDYGKBDHhUOAsRwSjXgWy77Tvz98psPIvd+6+WPYNRdRWcPDyAR3Z1O/fNjUymrQI6eMaHoSFrmhDS5lbhjINRfdUmECyfCfIFeLWWiw4g4bq7l+4HBORbei55tAIjhEsxJQoqHi0Q5dD5TFh8IiWqowkFbpvNonMSIpKtB0cyT5jU1G/jRA7MPcIvSrdzHaDkoDNHJgAeZfgjOhzTGYYD19lGIljz5BQBcNFZY2dJbja+Jr4He2CMAOBOdERa4Zn1VyNfOmd8Bn5hu0C9D2ybnSCxjXXq5TRiktR8X7WycVZYfqMZXAwP9FEHVitJ4MZEGUc7S92K5gX4wmjcJjLS+Xo/0nsduQm8PuiMjbcPM7/oGx8Xm1KuSfHdKWMBoaesPaDvRX+YcuiNstXf1DkCWl72TsFABzddlNUMl/s2YSKkCSHAJ5ILqrB28Gx89kzVlg="
-    - secure: "CPsDgSoZIHLyjpUbYEsx1GbB+UZcPXCEsz9qT6XRM+qMFKLlSnHxoJ7gMrJtqfjTh+1gLB3UTjQjFr+jAenqLWzaJh7Zdtpg9BOG6aXC8CAi1h0U5ZMNSA/+5lQxOXuhN8HmXI1r8xPNRFXBdZFea+072/2HJTwmgYlVTTQ8FrbXQJCzs2cFqxnVeFmuG3N49AJuoy3B+P2DMpqPzABbQt7Jf5H9Foq+16iXxhRkYA8/8H2nF7ZE8IdJuRpqhBUoPF/8Mt2QBLIlvNIdIzFEy2O7ZhwL9Dt08AG9v5c91QPzjsLok2e+5hFGaeMpGQJCE1V3uHyrOAeyZs1QYr7qBWbjQCVh0Phxz9yOA9RPfnQdjyJTq4QEj6vVYvCi5K/CGp/DnnLnGyYJKZOt7nBx02fTVI136UXqt29eF5NRkCpnUqah2s0OXkijsw5Y9LsbwiUxELKtCthESOwN8e/ZNvn/gjPfZWIaB0gupRNugL8Xo9Cx6vFmaW8wzm1IutJvo1mWvkWMvuYYjvd4aDyP6s7PFnSX7DoD/pfxoQuzjwHO2OD93nCOx0ofnNojLNooeJPKLikiwS4qKc8exWd+TlnccKSkXO8Y8S+XRgeE652YNlf+9DH79lNLeK0N0W2tRExX5JaEyJQCuvK0WZi02kxvjmUHhwLAOv9ueC2UCRs="
-    - secure: "JMv9mkSQBJXvUznXcyvngFaOfd2fHYEQQTH8BnS03pqAL0HkWSheG/R2HFJlJv1VJ9MDMf+wVzwMvU/kzkOT68nIgyFWnLBkT6fw+Mw7t/dG96/nOh7DPuaTVzKS0xRbMhDaqZOA9GW13TVnpVdIw79vbhQM69N9Gj80j8oM1cHgMi+fkJSDU3EN/vMJOGKSB3DTpyqAG/nYyZ38tLib8ic12za/YL3HIu8QRHa3fr37+cyrVKgGebGg++yK34p8lC1W4apiO30drmHVQhKWrWnmdcssdGmVM7NystMGGAzUwsJcmRFJuREv1LXiijDzjIRVduceeRYgPi1KHB5ZdL9vji0gM1eHYZZefhcJb6WgPDbCtbjdlCId4v+1bNfsh+dMmhM+vBZDtDEt3UC3MBeYTmklQT2w9zCXHuPBQigj7W4zLm6GxnAXife0SQMfmr736QBvSLUJWtd2tRgS+dRG/LvWxrP5Urvfgs8iRtEVZLDpRR+bSjvLs1UEKLN13KMKYuVwieHbxJn2kY7d5wmZVBYdaokl0yrTkZZ6J9xIphxM8GXJU1BnMZY9zn+Xq3jm576QYnNYUwCtUjOis00Ct4UVuuFqIqZ7bJL7GT4AHTENAk+9GtXVCo3/jnv161rxgoSdnUG6VmrpnyA9jSpcIvMVcE129oghBJ2+PkU="
-    - secure: "otePbb9W6CAuwzayL0dDAvCryzF2s5HHAIytvMW/xnzOuRMR03lumj9bhg56YCB/nIuTncAjToYmsMh2Acrp1Xcx8/DzqKse1K9nkfJgsB+EYeXmy8pVJnCxhHeLCzUPJjUfRoPBm+5GHAkeoviUyxenwbOFm3/Uhtay7i2ZU8K1FsECqx4FZXVh8rrGHqFGzHNimqDDvFUQM6f9a+BevEio8+/aDooaFfvVqPixDYXpi2+99DB/0hZKwcUpj6pOUUszATkAl0kJdOSWI6E7bQa63i6SfhcBNHfvbDqhpcAGs1TIXHbXwWGfySABKgT9mkq64CY3bc/QZ6HeWe6P5tJmt4mGBxDMOWMYj4qtCskQCCCiY9+sCx61Sj8W5w8exjQvvvFA088cKtbCKAx8FIil4CuYPtIjDQFkiNI2bC6BmkUen7qe4z7dSF2AZPb6CuSsBjoXL8Ezle3e1pRrkR+SYbxMSaZVcQoG8hinTqaXhlS5gXi97ZzRrJWn3tUB6JWEBeEkIfTJmrJolbLYIMcADNhKenrW8k6nV32JkvMsCgMFCIkJsIZeuQ9ZYgSZ3CXPODux/D2/4/gNa+353Fs3DTlDQxRTgoaYi9UDIRrKZAkoELFclxuGSKMCCheevQ8FMCmXfxUocwLSgjlO7g4rTJa9Kggn7VltucXiVWY="
-    - secure: "WKdB+WpZG/7SKpPpgM6DAwzVg0QQuxFMJEZE324pagJ74xv+GlObTLm6kU4SAZf6TuRSChTXDAD4paJXkxpq3LlXjp9aKxPASbG5PgZMnGL6bZ40c/UUqrwVy2HLnXPKSNuDBX1RidGXmH3u4YvufjguNnlOLvHYfMHIRGjiEL4ZvC38GB/3YDmID1zI6w8NzTCpFvxeNdg1qhhOBUKyt+icRUwBu8DfgLidzTrO0j5jo3UeqO/w3E9t3nnRgSiO25mBYwWySm1QBK1VS3F5iG9jo1J6GKMOFbWi3lOBoqnWotpfwID+p1vMNX35phHwmMrtoBYfMTV4A2CvQpGyQhn58qyMoKnhLz+NIOxlHN83qcu65bTMG+7ji0pgUl3jhp3ZPyWUjRYQpoVAg5f3UAnUzJgBrOUC0N60ukJVacT/kvkO10CLfz+0+eefW53r+GCkTi2m8iDezZn1olinpLs+Mt0M4VfQ52RVtq3WB6uxN7RjgtrK5XGi2zVAsvfHp+vqFQ6uu+iioiWNji88cTlMuKheo+FWwozNQzd8+4Vxe9w01mLek57Q0dpXiKWL3vMS4Qc4T9E6UzutllDyg/c61LW6Afcqw3jL4HN0UNG3zFefno47oKrmB13HcxakLEC8sdNfE1kWGvB7jqD0snXsz/8rz7rf0IqnK0kaCAc="
-    - secure: "Lah0Q4bWyEJXTBLZHCNkuEuU5wlDQzLF6aQwW47RyVtyRDYW8uQRQFkbb/3oj4QAAgXl0sOcDFnRaaWmlOStPIIYKKVY8Mk6ufyDCz8inWoPNJoSbdqbAi761HacGUsEfSNDrPNB7fst47UZaX7WNAO3q1QjCCGSEJQVpJE2MDAjcSJhAgLPt3JRFSOSwvqxcDLZI+wF9AM81OWNp6QisAdI4LPUJK3L4M4dLh9+apFRb1OMld++scWah1SB5Qj6tPzMvX96mp1XfWB1fJXuf8Yvks02H6ZcAXubK2HXe3iJZKVqGB67kShNN52P+wg0ZZO4OP4kZ2PXahnB7hhxIfEfWsNVy/w4ww1N6K907BfT6RDmsgNP1ZP8kpq6H4pTnWgAy9WDxjatzbNFAHtMzakGjYJNFxZJVco2FL0ipNQ7htoVAA4sUb+VbRQv4O/oZTLMNnnco19+TFJzuZuS4Rjxj/zX63gXkj/W7wou3hw+NpI/PL2hUXIc5imXSLfVQwri8Nl+6IOjV2gWR/vR1VhQqbLsTF6TZQKNI4lvbRTs1nNeqNMW1lHizI3r9Kernj2noAsxJK7wFTZ64OUkQvSNphfmVow29JYQKXbpxbmRrdnmfmtnMsxUngpx9WsTPhrprt5hIAqiBspHemrs+H3LiIml6IY/3l9bcPlc0gA="
-    - secure: "sNc7kC0CuH/TCZh2WwEN51GcA1fSpcliCHpFB+WX7ieQiRu3xKn2avby/T7vbvX0viXRER59arFGQF4i/dyr2g4tlZLVRYjPeiApfduKZ7Lb+vZGro3cWesfHG6Abk2VcgZZli1IDrgrHH5qAWnA9xNnKvKBL9NDM73Zj92BmlDFVEzadTii8brWvced/YP3jNXEmM5ZIufgpe2yidBB2bLWYJXb3Cf1MvzMG4tqNAtZTrI32q50mokz/uTqp3MRJ+cR8sOI+2+2xSbT0zZGLSRZf96/7FKtE0QIDxdWAe4XdlHq1CluRVk30Ju5BEn0QzoYLryCIuw3JjDl1Yksw4IA5imljZJlOmWa2l6fX0HNxMw+z0R/1d2HARA8BY7/uQKv4guV3Cf3jpsWoKSsM1WxqOqsuEFOoRQ2eQNJEaSuC6+j/vzNoj61pOuG0R9OC2PFcFCZ9fomIrZMse+7M3WIj4+mp7e+JDK8DgVdUlqkBVCx1Ospseb5pm6lDx8F5NbgqZgGXgyoWVpqZnyYOoOutezMoD6MI2wXzJaepV/L3+LD5f6q3DAa/sRAEEsBFGyMHXiPYbziEiy8Hz09Sz3inT5rzS8OLOinwAI2sHiIYHTl340XfWdYz6AlNhYLCGwwmtkntbjOj5UVW06IgBBx44ujpZSUjv7SOrACPGU="
-    - secure: "t9/mC8eSsxXIB2vjcZGtGISxrSY3Yd+XS4+/i1YG1mxSov3R6UdhVl2blLgrvFfVCSo61YMAzJbXKtZlXPzhLDXxpsSlWiatjNrKKo4D2unnlHsyEMJ4wfz8cJ8pKyynP7Kc1ZuaqTSMcEZgk85tlew8Zy/VH6g3Mc/7DvP4SxEDRsP+DdCplZc0vbxHDaV3iC93bwNRfy1UQypwJJ2WQviRema3Umneg8hVl2V35zbaBoy45ubCkUoCbRCDaUyoHA10GrE1OUOLsioar/dj6K2W3EhAtehHLWrUZhhl07rayrrRDQYDTdebifB/MWlvR5FjM/Dr5M4k2ciss4ol3IR5LypRLD+/YBtzeIuqkDbUkaBowY+oUj6OWlzEbzAUrUNa5mnyR2jhr8ivZUeEEWLxljsu8gWq65mzgiZt/u+kVCnMLciUv0//0nrsNsEMI9pau2ZxbcpItFVKdZYXFdmHWG+qPCgMcbgsUM3xqaIc7fNwbk2Aa6erIGqD2VkwWzD/xksweg4lsgQ0N1tXMfoWWKh4Xj/OFom+S+3w9uSx/jQma7nXm+PKQL8dIo7rqza3fz9biq5T6mhwUrqCpFIJv7mrMwaTT1UfOchjiLL3CGeO7Amv7Avmh3fhMPbypF0sws79d7ewZbyv+oSzn+pxlCdzBU1GYx6veApbzhg="
-    - secure: "ljVx3TgpBJ/ylMKDVmXabi9UNi5YvrRM5UBTrRk7XPu1YFYS4FI1GcGlyvYhToc4fKt4jLhX9qU+s/rZY+odO0x/HpmJglMBCrY+QcWOzuyaP1U5dCET+evuqFdEAZIzLQc4VDjL1aQLZh+OG7bjoBClVAan6a+pmW0yxBC6rNtCWTESG4rY3wOeTpoI0Q1gM7gg5Zkj4Z+yLvYeJdoKHijM7C3/R/VVTqUFqArk+Js7Qb2qTqm03SHP0ahRQA8XSfbPebSkJyX9oLbidanBEaQE6sqnp9Qh+8VGcnn7VkSu6oq2+ZXz4xlSMrH2Iv2JXl68Td51LsLo9BxaMCL68ssgTFfXPSrrcLwholNEt1pXk5nhBl1l6MZ1UwUJyBm+AXZp/4sCK9/P0rGa2d1rOcpOz7nobH7BDktqEJkrR6VzkTMx1aOwtF+JSt7SJQ1RrRdm9uKfOZZsnw17+VgVAHo/ttY0C3cRl10oaF1C/IdliDfa5gJdZ2VSZtJxyewqKwGiZrqCRv2fQyIuGsqfHXsyHVL6q1KfVcHjaXBvh0o6xZ6duieFT4FNHg7clv1qPQV+cLh4L11nugiihRTeYQtKyUnP5YIL+jlcGNM6KqKhF9RN5c+zOqWNmEcz6O8nljY5mFWdIxL6dFfE3+4wpw7snFP0PrWIWl5SmrU5ipY="
-    - secure: "L0ivSUmbOHNmKWVR2zeYcVR+Xr6780dYA829MyksXfg9OrieTS+qVqSODKexFW89dp4Wf8xFdT9f16xSqjA/xSuX12uiISMiTcFKP39ZnQZ//NDkx1T4pZaeNiysqT+2Ys9OaKTWxn8luFtAYp+DaluancQtEw6+M8H3jQyQZimGHl0QB6BK2A5VA2vko+7ebIdwu5Df5E7lc/LG42H+DdSFkSj7Meu5XMyPHSZRAdOE5doO++tqFKzgs2WbDRHRUP4R4LqtDlGn8+5qnQtQKUN4V9UGrKM95R34BhgUlgOqOFAjFDug71/1/rRyv9XnzKkAdTIbxV7nmxSL+APhQDqpwPr01EP9gtZBOycswV/igJYotgqkhzwNAYrmwOA5Ta8S18Ck0feDEqT20w8yKS4QwV2Ihg4q7CqDFu7i+9iSRUCd/RVrGtG0U5Zo1b3+1JcNxXH/ErUeyHPfrk4vktxfHmU+omqwkfvTRy5upn0Ycr57YHOJc/Iyur7vE07HcnBfAwV0d5KO6HJ7M1n6hmMxeyKmf+qiyQQnphySvHHAfa/9Sec7omwwKv4bn33DwFGc8GWvL6cZXWhmGhFvd+LslpZl0vZ+7Bz0tXlAg4t8V48y/ZtyoSpny0HP4UstdA43PttvZ6q2y8LUNk4LBP5btKmp27Fszuj/rldtj5g="
-    - secure: "Dox8JthAJqWT9eh3Jt4Morbf4pGN9OjduJXe/lYMsmFvqNg7b94P2QdumWBPmVjDq4YjDVirMejBA/TNwORgKfg7pI7MOw+qqoHpT9xyPecXi3ecyBay13e16p0GNRlc5pUu5JcU8sgCpttvM0EAw6bOuQIhnnkIFesbOvwoxGYjMzjmWMNuikR3CjKbo0LDtD5NJXT1OMSqrRuh8NM/BoKyn/kCdSaq9wI2GUMbkg09/kFJkQOvtMXPkM7dIhr/9UC0ouMIyqe/MHa8O6Y4xESdqiTql9uz1+eZfHIRrgFlHfxDvkMv87Cx5OuL+O+qeT/a+RYLCRJspMoq94IYHGg2iyEfBO2YAkogl53wiEf2KF2JdiNGT0xId7bxCJj3efTuCAXV1oqaHpJli1Mhvs7zPEtf78B4tkWEgjhGr5pBLIlbhNjS5wtTHJX4BUzoiP+wODj4h7rjPAah42nWF8XOMlboVi56sOCLjiHBOvYObqyhSfiQxoi2XHphsrZqw6H03tr4Kqd9HVmuoSvRiv+NOu24Ubr6MrrQM2/G72TrTx0/aBlt8Dx5nx2oWZ9ZMiDUR3XlvDLUi45SpY5qESXz08nRlcdS9EvUpK7C+77bNvX+A3dIhsxnxuNaf2naf+QnYYbvh7q4Qbrj4v6EMYS90Uky1JHdoc2wMua8J+w="
-    - secure: "Is2Lv/rxKKrXnxFns9KQYseD02tjY9qbgSteVtJavG1cLJDvkTwb6X+Thvgo4cxk5fQPiXScrQaYzHjVVuNleD+dyD1HC/8CU2Xq+tjBhPjdcccHFSbk06DpcETmLGRyMORXG9JkYlgXHLLXtu/9icppWEHgra+zvchVL2YDhofYne8FMNBb/lq0AAC2wgzAS33tW1+57HaYnzl0hf6+Q6lwqoH2/aTfGMRFDgyJ0HK+5IVfLnQJ+OuGFSrj8/0FWSggR5+EXDIddDgovFgaCghMjHYp21bzn0eIAJtuNFFultwk/UC1lT9joXTKEzgLTAh+w13yz1T3x9rNuv6FDKCotBIS/ZDtPmgvyZ8xvB4SzyULnTRSVn7YvspKR6PAO/qxGNudUD5H8tRKer4qKnKjHzSUcVBlRHb4yE6FqqY1Z9RLEcomWO43nJwb6saNHR9BYedyi0gA+EbA+P259QFClW1dWEQ2LQhDa+0VRssOqZ0BQblPFyz+e5Vc9kfAMbOuoss8fjkiYv+twXv7nT27xrVT5okfKDSiy5opZD6d36N5FibZPYiMrVx00YZdkFB+5EqQuJ7lqKUMkZJTeApLzj+h+/4aAOWd3paj5ghv7m+9ReohsNKFHyjaSy97RhMAZjzqgMMdD8rjUSKDhvNKvvQECWHlaXwL129GB0s="
-    - secure: "l7vWxfcu1RgXbStq76Mzz2I5Iu4e31729OyLYqulXZzft3wO+idvgQKy/JSwajiKgOxlpBuI0wrncgIUYshcRvE4yB0y9+QIMDTegJzTADtRSUyVNCIZfTgvtOvzrlW0iCdVsxLBtYcJWJVPdjF2q59ED1jahd1AuJJqX5e61gfr0eZ9+cNiHbX1u3VpmGchFNWQF4KvebE4WKs5xWEds+AtbTODdQq3H6kKQK3fTVJnbz6WMO+bEWgWI7orfSE20lku/3Q3eMLhNOcPwH8WnUoTTvDWol5Cq5NfPhkKF5aV7kIbNXkxswM7yBPAPumBiXdM1BHpfd4+0YQ/fqRtnqxw85HEYpT8dRewHimCl4IccgGCR+G0tK8RNleKL28GWrz86gRVpXMEhOU3ILwb8as3SdQWLwhXXy5uKGJmsdrCNAH4/8eu3XczO5VN2wOo7narYuBgGcl2eLW9TOLyNVKFKxbQnDLBiOybLNoOV42DCRYt7v3Eknkv1Z0dmX6n23q1z9if9kkfAFgRQWsTbNZyWeIwWuX8b2a0Zq0znS3JflSKxzqS7RHADfBOJEVM86AiGkw0XkR4o31yDrY+zOrkJ3GxfV/HpqG7LzCd9tFeexanneDCE1FJhCkDjpnc0Mdyx+1gVf+u2MkgrU0BbCf2EESBAlC/FTRvxTmk/6s="
-    - secure: "bvtQ348bxKwTtB8X0zMxeTsM0jEJozbS6/rzH/88Fk90a+KO8SdXen0Kj9/LahV4duMn2xTTRmxMCVj424FbcVTgBkJpIO7btqTcNASORkmK+9wGTK6Bgb9R5sHLbVrbrMYJzsMQR0MWxE8ibPLlyom+ssUIqr7HAjnRyYSDiKChGhgBfdE/0G9OE/DuQaU/ZkTuEYfc4527QNLJ8Bt1aLDdCHJLxzCojNaTHB215Tz7dmpnJjWa45BXxkMwLTpxIJuY7wA7K+2UBJfLvZv5QiPYyeUlosV7FwhCN9e90+dc2x8HPJbwe/Ysv5dm8/FfNTgTe6IkKw1z9kev9/W1tYCtqMzn9GnlIH2000ZUomhHKbc4UUl8sskF2jGr14rSz7pfaTyvXNvK7nKLKM7mZAO/cAaQvZoGEx0s6B7a7YX38NmJWcM0UUD603n4G9uVZNoxjyr8HWC6pho8MY8/u5aFKMPd+C8DaDWeSzNJDZRyi62v+t2iyyZuQUtVXtnDpv8GQW4tYFIJCkkrTm5VNNgD2x3WRsEr3LsBa3BonOmi3a1VrdIpyidDzBEWFUR59a/C0lU5J2UX5W1vu/Pnr9/8p898sDvvDbluz47OR9xoVOjtmGdt/ENJa4EOY1q/eRIN6zrt/cwwjIKgTcYOVk7DFDMXQuYlDeEFuHSJ5Xs="
-    - secure: "I05ZVphCT2Alu4najchCjDLIgflsdSCR/Z20OkMbcESprwJB5h7nb64kuVkqPQT7GeMcXMnK1xahx0KvE1Gn8OqOlDPt5WgnvcYERnVOKEJabmRXws8nM20AehwAzRoc6uDhaTnmwHRuliPSeuxiwY2Rkyj5hdyvEMzaOSY+qoIOYtDYQu0MM1XIC3cAjSbEfEGuqnKcN0p9VG0l7WscEUtcKPSFBShOlJZWDiHy7lisgO6OEzvAeDpZU/1ZdljXBl0RoDX8l4LqH4SzQqE68kK7qqIhZNmx4hkrh7qtx8tVqVKy/R3IUYhdxrJ84eJxrex6RQb9SgJQRGwi/3+rkKxVS41KiQ4quJHJhnqd2A1+5rDMN/fsjbjtM1I3VIl6w4oaT1C1QtwSjQOvuZCKDLIszVutW0PuGHveNyjwqQSm2PboOkLTg0dNrFIflT2hGMynv1s19WzGY8qQDP2UgODNR4EQVBy0PRk43ne5W/AFtzY1wUUUCBfKi7WYi4uURG/1Xu456GEElbIOwx9F1nC0s3z6ZFVJ3bVbmFxUlHtCyL5wJ3fiVHkAoPSygVARvv2UhHmXvJXXjxBMOvsnZZbpjvcbTeZoguffxMZEMNhuI4W22WjLdVTLxi9uh1VTdm8vkfAeNjMPw53gwPaSz7Jv6A82LFqpmbwAHxNAEy8="
-    - secure: "BSZK3v2RYsNKv/8lBZ/dKagYF+znGXIeDY271A2nCS8DXx+y0octwI5RT08+UqdD1AvhPOPjdnSXG+M2NOSLlfBoXF/0t9S39+qy3EhGgxYAcWcVDXnrOrHEOALg1cQkCA2D/CG+eeG/36SmAn6FINTLaAOilxWGGCeL0KOB5mqPPnPogaYhowIwDkj0Hfsfrw8011XzgbufNr15GIYJ+nP7f41NcgIbnNkqXY6Q1jiWs1DuKXNDr5WGPqztHtVf7RtcpVmShzptAcAMdRqCGKF7atCZhXPbd69j3qf7F533/tCSCIrszR/Wp4BKlJfVCRCk1oiEcfRvprJqeFJ+0WJCVzyD7hXfnkBg2Tb5PmzvxOFM+hsqRu4mY3UdjqXukzAe5O0O6Uo8sqzqQUIjUooRnNQ4GdPd+7wMbyBZn4PJaW6YTi/7zg7mAegqot6uyEGGpWb6iFYrf6DX75GUfTciNktv+ez0AeqYFwNBLgcOsmaq+3V+aR+dIxsxIzC/i5GCTHvMYOIp6Q2tPCuIug0tX6uxm++vMOoMLK+vG1fVQ0Cd8m6yQIWEXUu0VBs1MTlJD+vX6LsK8CGo0Dt3ZS3JmC4TGGo9CBSEkeDvnKvQeX6x3NBIFWvgt+Hjxh4S9U+vW+AaUXpV9k+NEhesC/8Ys0UGbGYsTDzHdx9TMBY="
-    - secure: "sskZYJ/+xmHh5GNvw3QRf10+sBGOjlub29jOTxjXIRYUyZfkhjsF8CR9NP59CBFuqgN716Mj1uEVEcx0aaepr/zWweQSnqa9lZgFD9nDYALVNh1b4oyqfhYG1sw57ZHdBsJOBF4mKBnahly/QQj11Ya25GyIS2IewwCWVNtCOJYietSssG9l0+qc+RPG5Ub/lEKA3VRrtbUuApcIYszt33DumgZ4wzkzICl1SuOy8V15vHVlkBqASJheEDa4Hia+eAMo6e7OCE5KWjl7K2MxTvtszwFi7lgMCyPCOy6DkaULACfBnQeloh42B2Qhd/ta04vuRKg6y4fKSrnegpeOAa9aGxF1ufvG9IvPsRTsu4+w63b9xsQUN9MyDgxcJe0YeUHPgPYL6mqAjCIKvL62LlIyrQ8IAteoh3MC+4Xb8crXaLGINTcLwYvLwxsHMuC/58hdQ23I4DqnyZGAS3L1IhpX4QvYN6xc7H+ptaiQttweoH5VpMAduSrmSnNmvLOc4g2PvbRES6D95moQ7qk7iX6vpu+PevL5HtQCbB8SFyFLTYWsqZaG+4NrjpIi2hLzUT35meHwrvxG/MsfeqOVlnBfa93VnX75vxOkRFNY56sOtrTJaiVZ+rQUdszAZz3KGPbyrSlz3KxV+ZKxZH6/0oFteAJttXFjWQdICnKGDSo="
-    - secure: "sdzU5bPs1w7Nzf3F5Gtk/iq2Kfq3zfLQNGcehLZp1gJXzNf7F6HZLOn6GaXQ/5RVlhqR5nOmzMpVQs88rZv+6PE6YqGMugTxHIQeNmXtGnuEDJSBvGT9Ok8ENxcwKwL93g9SCd9P8mTspxklOsW2Bk3No2+Zlc/aQguBcX44TwYF42KuBU4O7oS6pUg8NjnuQp2zTyhp0ouzyAudatPyu1BLci/3lbx+MehutQw1y4Om6g8tfwf950UONZQdqq9MBluu7yYb1oHkdC1J4qgwdCZkJslWIwQHCH5UE4AW9iVG0qVrLpzBGtV27Kfy/Vf8r2gMYzbiur2h2+zzWSDm8/bk8YLn7u1FBpjGJdJ0pX0ZrZr+hMV2vH3e54NvA5WyRU7tw8mZ1PoDYd/FM5KXIYbscSbSRCqTsbPgyVIHuoOWXeeSe+/Ef1ifZv3HHf6ARfUIWupKfAipxChc7QUMI/HEQ9QPsqgBaooZD9chGsWAgv+8tFxdteqkx4Yh+AuZp1rVykB/9vAamUBebQxy0oeGm65j6X1rksfjAPkfeDYB7L4Ruy7tUwPtvYrAHPoWrf9O0g/qDRsw0vdqp42CjszpIxhuPhVRDx0i0wquqw2LnIU3ejRv2R8d1SecVKBBcVsWLFvc9iR4rNIi5JnxITRtiwL1Xv3Vcgx7WwxExtE="
-    - secure: "f9n1KmU5NuV4jGkrhLNiPD+3Cy7t4D7Rq8YsPlmyB2A6u6IHMuOVP95IwH6Zt0cmMDBrZGthj3/0iu5kzxzqD0m135PT17wSEqsDfDMyKRZJQuYSO/ESVxnSca3afRH7Ds7/ipQVd/ljZgwEnW3JMaOQiIdbbIquNOOTeY0/wsXkreDamXZaKKqUoeadkAV4AkKhM0xcMg+Vni1i71TYPBWrZPLVAu3ZrSvU/cE5mtBUIkbr9EgsEE+WR23QCgtwxKzNxrXetBcPXDsJb98/ABgpoItm5Ko/Zk6pkib44f+iQtn7Y6j3lieELCH5Sn0uy3RrMxkl7xicB7zPYME94NEPHCmshyVsH3RxWfcBG4kauRNBCLYLl5HYF2t1lWZ6In5qlx8xN3Tf0KrbM3vzAEOnnfZt83h3q5OuWl+jzTOcv9xHmeW0lnwEEfS0nxUV3KDqLFBczcUxKBmsA2aWnJZ2HV+kls6OaWZ987m6V2pIGR2uviGT7I4ngjCSOJzbwYbvJbJqYAI97FWP/5pqv97xHLCSSJh6TBO4NMQ7Ib/W1XT6NZyPOjNGSLpd79UA3cTzU2+UYr/7RxZpAKONSAMTJh7CX451XQwgovpFZ2quXs2BzqcVpy8AlUc45ygnFOALOANAkcRP5QFhmff0jpXstjPW83/GksbtaiLhIiQ="
-
-matrix:
-  include:
-    - os: linux
-      name: "Linux 64-bit Xenial"
-      dist: xenial
-      language: bash
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Trusty"
-      dist: trusty
-      language: bash
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Precise"
-      dist: precise
-      language: bash
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Xenial - Python 3.5 Source Testing"
-      dist: xenial
-      language: python
-      python:
-        - "3.5"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: linux
-      name: "Linux 64-bit Xenial - Python 3.6 Source Testing"
-      dist: xenial
-      language: python
-      python:
-        - "3.6"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: linux
-      name: "Linux 64-bit Xenial - Python 3.8-dev Source Testing"
-      dist: xenial
-      language: python
-      python:
-        - "3.8-dev"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: linux
-      name: "Linux 64-bit Xenial - Python nightly Source Testing"
-      dist: xenial
-      language: python
-      python:
-        - "nightly"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: osx
-      name: "MacOS 64-bit"
-      language: generic
-      osx_image: xcode9.2
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: windows
-      name: "Windows 64-bit"
-      language: shell
-      filter_secrets: false
-      env:
-        - GAMOS=windows
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: windows
-      name: "Windows 32-bit"
-      language: shell
-      filter_secrets: false
-      env:
-        - GAMOS=windows
-        - PLATFORM=x86
-        - VMTYPE=build
-
-before_install:
-- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-before-install.sh
-
-install:
-- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
-
-script:
-- $gam version extended
-- $gam version | grep travis # travis should be part of the path (not /tmp or such)
-- if [ "$VMTYPE" == "build" ]; then $gam version | grep "Python 3\.7\.3"; fi # We should be building with Python 3.7.3
-- if [ "$VMTYPE" == "build" ]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should support TLS 1.2 or 1.3 to Google
-- export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
-- export gam_user=gam-travis-$jid@pdl.jaylee.us
-- openssl aes-256-cbc -K $encrypted_ab10ec38326e_key -iv $encrypted_ab10ec38326e_iv -in travis/oauth2service.json.enc -out $gampath/oauth2service.json -d
-- cat travis/cfg_template.json | python travis/svars-write.py &> /dev/null
-- $gam info domain
-- $gam info user
-- export tstamp=$(date +%s%3N);
-  export newbase=travis-test-$jid-$tstamp;
-  export newuser=$newbase@pdl.jaylee.us;
-  export newgroup=$newbase-group@pdl.jaylee.us;
-  export newalias=$newbase-alias@pdl.jaylee.us;
-  export newbuilding=$newbase-building;
-  export newresource=$newbase-resource;
-  export GAM_THREADS=5;
-- echo email > sample.csv;
-  for i in {01..20};
-    do echo $newbase-bulkuser-$i >> sample.csv;
-  done
-- $gam create user $newuser firstname Travis lastname $jid password random travis.jid $jid
-- $gam create group $newgroup name "Travis $jid group" description "This is a description" isarchived true
-- $gam user $newuser add license gsuitebusiness
-- $gam update group $newgroup add owner $gam_user
-- $gam update group $newgroup add member $newuser
-- $gam csv sample.csv gam create user ~~email~~ firstname "Travis Bulk" lastname ~~email~~ travis.jid $jid
-- $gam csv sample.csv gam user ~email add license gsuitebusiness
-- $gam csv sample.csv gam update group $newgroup add member ~email
-- $gam info group $newgroup
-- $gam user $gam_user check serviceaccount
-- $gam user $newuser imap on 
-- $gam user $newuser show imap
-- $gam csv sample.csv gam user $newuser delegate to ~email
-- $gam user $newuser show delegates
-- $gam users "$newbase-bulkuser-01 $newbase-bulkuser-02 $newbase-bulkuser-03" delete messages query in:anywhere maxtodelete 99999 doit
-- $gam users "$newbase-bulkuser-04 $newbase-bulkuser-05 $newbase-bulkuser-06" trash messages query in:anywhere maxtotrash 99999 doit
-- $gam users "$newbase-bulkuser-07 $newbase-bulkuser-08 $newbase-bulkuser-09" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit
-- $gam create feature name Whiteboard-$newbase
-- $gam create feature name VC-$newbase
-- $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."
-- $gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room
-- $gam info resource $newresource
-- $gam user $newuser show filelist
-- $gam calendar $newuser update read domain
-- $gam calendar $newuser add editor $gam_user
-- $gam calendar $newuser showacl
-- $gam printer register
-- source travis/set_printer_csv_filter.sh
-- $gam print printers > printers.csv
-- unset GAM_CSV_ROW_FILTER
-- $gam csv printers.csv gam printer ~id add USER $newgroup
-- $gam csv printers.csv gam printjob ~id submit https://www.github.com/jay0lee/GAM
-- $gam csv printers.csv gam info printer ~id
-- $gam print printjobs
-- $gam csv printers.csv gam printjob ~id fetch
-- $gam csv printers.csv gam delete printer ~id
-#- $gam calendar id:$newresource add editor $newuser
-#- $gam calendar id:$newresource update read domain
-#- $gam calendar id:$newresource showacl
-- $gam csv sample.csv gam user ~email add calendar id:$newresource
-- $gam delete resource $newresource
-- $gam delete feature Whiteboard-$newbase
-- $gam delete feature VC-$newbase
-- $gam delete building $newbuilding
-- $gam delete group $newgroup
-- $gam create alias $newalias user $newuser
-- $gam whatis $newuser
-- $gam user $gam_user show tokens
-- $gam delete user $newuser
-- $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail
-
-before_deploy:
-- export TRAVIS_TAG="preview"
-
-deploy:
-  provider: releases
-  api_key:
-    secure: bzambMcQwyv/o5c5GrKGCsZHgE5R85tg8sNFvPfpISz3+uosCjnBXas7wvCKzT75XUFi2ztfbYak6HdKf4sGnNHk0saEicB3slH+ghPyZbYzp76yvvduhFO2nWW3/F01tL+Yfqqt4/q8wFaWGjrC5km+6GLVyB4lWA/Uyu49qKnz02uSwyhBD/VFbO7DOQ65a1iWk9HngyMsu0Oi7HIbSjSLtxTHedNfOf3waW0NivTTxYXiYGX/MCu3GWhgIGj47a+H3A6FcQ/9QWvnKgnoixdgPBUz7kDb7ktsWwQsILPGStgH7iMuG49ZlXdEFmqwifBri2wvzmFEevBGZjHcupy1IGrNFRG+IUGKMotio+OkLHlLjuv7ZJtqCz/Vf5SNFgNyMSanx6jKEUJuYvndVg99IRXmYVwHFwPu5BAcJACpU6C0AfyGmmSqqwxCd46uXL62ynxNFpHuRfOqlDnmCTfZgjOciJSlDDpf+Xz9fF7+oCoeCi3mrcZVFjhd3tT6Oxw5HrsDtm0ZNld1cdLidaq8H6vOFgHMd0A9yNYZzTzXTvpmxzkXT4Zc7s+PYKN6z5fRZ+pJeckUjRXblvVEfs5HFSymavcOc5AkRwxpvOsTQMNmlnaJCBo5UNs0K/rVmRi5cFmaiwTcBCY0kTllOBJ4zWsfq8seiokWwNUNK2g=
-  file_glob: true
-  overwrite: true
-  file: gam-$GAMVERSION-*
-  skip_cleanup: true
-  draft: true
-  all_branches: true
-  on:
-    repo: jay0lee/GAM
-    condition: $VMTYPE = build

README.md

@@ -1,4 +1,6 @@
-GAM is a command line tool for Google G Suite Administrators to manage domain and user settings quickly and easily. [![Build Status](https://travis-ci.org/jay0lee/GAM.svg?branch=master)](https://travis-ci.org/jay0lee/GAM)
+GAM is a command line tool for Google Workspace (fka G Suite) Administrators to manage domain and user settings quickly and easily.
+
+![Build Status](https://github.com/jay0lee/GAM/workflows/Build%20and%20test%20GAM/badge.svg)
 # Quick Start
 ## Linux / MacOS
 Open a terminal and run:
@@ -12,8 +14,8 @@ Download the MSI Installer from the [GitHub Releases] page. Install the MSI and
 The GAM documentation is hosted in the [GitHub Wiki]
 # Mailing List / Discussion group
 The GAM mailing list / discussion group is hosted on [Google Groups].  You can join the list and interact via email, or just post from the web itself.
-# IM Room
-[![Join the chat at https://gitter.im/jay0lee-GAM/community](https://badges.gitter.im/jay0lee-GAM/community.svg)](https://gitter.im/jay0lee-GAM/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+# Chat Room
+There is a public chat room hosted in Google Chat. [Instructions to join](https://git.io/gam-chat).
 # Author
 GAM is maintained by <a href="mailto:jay0lee@gmail.com">Jay Lee</a>. Please direct "how do I?" questions to [Google Groups].
 

src/.gitignore

@@ -64,7 +64,7 @@ nobrowser.txt
 nocache.txt
 noverifyssl.txt
 gamcache/
-gam/
+dist/
 gam-64/
 *.zip
 *.msi

src/LICENSE

@@ -202,12 +202,12 @@
 
 
 
-APACHE HTTP SERVER SUBCOMPONENTS: 
+APACHE HTTP SERVER SUBCOMPONENTS:
 
 The Apache HTTP Server includes a number of subcomponents with
 separate copyright notices and license terms. Your use of the source
 code for the these subcomponents is subject to the terms and
-conditions of the following licenses. 
+conditions of the following licenses.
 
 For the mod_mime_magic component:
 
@@ -273,7 +273,7 @@ For the  server\util_md5.c component:
  *  Original Code Copyright (C) 1994, Jeff Hostetler, Spyglass, Inc.
  *  Portions of Content-MD5 code Copyright (C) 1993, 1994 by Carnegie Mellon
  *     University (see Copyright below).
- *  Portions of Content-MD5 code Copyright (C) 1991 Bell Communications 
+ *  Portions of Content-MD5 code Copyright (C) 1991 Bell Communications
  *     Research, Inc. (Bellcore) (see Copyright below).
  *  Portions extracted from mpack, John G. Myers - jgm+@cmu.edu
  *  Content-MD5 Code contributed by Martin Hamilton (martin@net.lut.ac.uk)
@@ -319,10 +319,10 @@ For the  server\util_md5.c component:
  * of an authorized representative of Bellcore.  BELLCORE
  * MAKES NO REPRESENTATIONS ABOUT THE ACCURACY OR SUITABILITY
  * OF THIS MATERIAL FOR ANY PURPOSE.  IT IS PROVIDED "AS IS",
- * WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.  
+ * WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
  */
 
-For the  srclib\apr\include\apr_md5.h component: 
+For the  srclib\apr\include\apr_md5.h component:
 /*
  * This is work is derived from material Copyright RSA Data Security, Inc.
  *
@@ -501,21 +501,21 @@ This program is Copyright (C) Zeus Technology Limited 1996.
 This program may be used and copied freely providing this copyright notice
 is not removed.
 
-This software is provided "as is" and any express or implied waranties, 
+This software is provided "as is" and any express or implied waranties,
 including but not limited to, the implied warranties of merchantability and
-fitness for a particular purpose are disclaimed.  In no event shall 
-Zeus Technology Ltd. be liable for any direct, indirect, incidental, special, 
-exemplary, or consequential damaged (including, but not limited to, 
+fitness for a particular purpose are disclaimed.  In no event shall
+Zeus Technology Ltd. be liable for any direct, indirect, incidental, special,
+exemplary, or consequential damaged (including, but not limited to,
 procurement of substitute good or services; loss of use, data, or profits;
 or business interruption) however caused and on theory of liability.  Whether
-in contract, strict liability or tort (including negligence or otherwise) 
+in contract, strict liability or tort (including negligence or otherwise)
 arising in any way out of the use of this software, even if advised of the
 possibility of such damage.
 
      Written by Adam Twiss (adam@zeus.co.uk).  March 1996
 
 Thanks to the following people for their input:
-  Mike Belshe (mbelshe@netscape.com) 
+  Mike Belshe (mbelshe@netscape.com)
   Michael Campanella (campanella@stevms.enet.dec.com)
 
 */
@@ -532,10 +532,10 @@ without limitation the rights to use, copy, modify, merge, publish,
 distribute, sublicense, and/or sell copies of the Software, and to
 permit persons to whom the Software is furnished to do so, subject to
 the following conditions:
-	
+
 The above copyright notice and this permission notice shall be included
 in all copies or substantial portions of the Software.
-	
+
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

src/cbcm-v1.1beta1.json

@@ -0,0 +1,587 @@
+{
+  "auth": {
+    "oauth2": {
+      "scopes": {
+        "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers": {
+          "description": "View and manage your Chrome browsers registered with Cloud Management"
+        },
+        "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers.readonly": {
+          "description": "View your Chrome browsers registered with Cloud Management"
+        }
+      }
+    }
+  },
+  "basePath": "",
+  "baseUrl": "https://www.googleapis.com/admin/directory/v1.1beta1/customer/",
+  "batchPath": "batch",
+  "canonicalName": "cbcm",
+  "discoveryVersion": "v1",
+  "documentationLink": "https://support.google.com/chrome/a/answer/9681204",
+  "fullyEncodeReservedExpansion": true,
+  "icons": {
+    "x16": "http://www.google.com/images/icons/product/search-16.gif",
+    "x32": "http://www.google.com/images/icons/product/search-32.gif"
+  },
+  "id": "cbcm:v1.1beta1",
+  "kind": "discovery#restDescription",
+  "mtlsRootUrl": "https://admin.mtls.googleapis.com/",
+  "name": "cbcm",
+  "ownerDomain": "google.com",
+  "ownerName": "Jay Lee",
+  "packagePath": "cbcm",
+  "parameters": {
+    "$.xgafv": {
+      "description": "V1 error format.",
+      "enum": [
+        "1",
+        "2"
+      ],
+      "enumDescriptions": [
+        "v1 error format",
+        "v2 error format"
+      ],
+      "location": "query",
+      "type": "string"
+    },
+    "access_token": {
+      "description": "OAuth access token.",
+      "location": "query",
+      "type": "string"
+    },
+    "alt": {
+      "default": "json",
+      "description": "Data format for response.",
+      "enum": [
+        "json",
+        "media",
+        "proto"
+      ],
+      "enumDescriptions": [
+        "Responses with Content-Type of application/json",
+        "Media download with context-dependent Content-Type",
+        "Responses with Content-Type of application/x-protobuf"
+      ],
+      "location": "query",
+      "type": "string"
+    },
+    "callback": {
+      "description": "JSONP",
+      "location": "query",
+      "type": "string"
+    },
+    "fields": {
+      "description": "Selector specifying which fields to include in a partial response.",
+      "location": "query",
+      "type": "string"
+    },
+    "key": {
+      "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
+      "location": "query",
+      "type": "string"
+    },
+    "oauth_token": {
+      "description": "OAuth 2.0 token for the current user.",
+      "location": "query",
+      "type": "string"
+    },
+    "prettyPrint": {
+      "default": "true",
+      "description": "Returns response with indentations and line breaks.",
+      "location": "query",
+      "type": "boolean"
+    },
+    "quotaUser": {
+      "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.",
+      "location": "query",
+      "type": "string"
+    },
+    "uploadType": {
+      "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").",
+      "location": "query",
+      "type": "string"
+    },
+    "upload_protocol": {
+      "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").",
+      "location": "query",
+      "type": "string"
+    }
+  },
+  "protocol": "rest",
+  "resources": {
+    "chromebrowsers": {
+      "methods": {
+        "delete": {
+          "description": "Deletes a browser.",
+          "flatPath": "{customer}/devices/chromebrowsers/{deviceId}",
+          "httpMethod": "DELETE",
+          "id": "cbcm.chromebrowsers.delete",
+          "parameterOrder": [
+            "customer",
+            "deviceId"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "deviceId": {
+              "description": "Immutable ID of the browser.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "path": "{customer}/devices/chromebrowsers/{deviceId}",
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers"
+          ]
+        },
+        "get": {
+          "description": "Retrieves a browser.",
+          "flatPath": "{customer}/devices/chromebrowsers/{deviceId}",
+          "httpMethod": "GET",
+          "id": "cbcm.chromebrowsers.get",
+          "parameterOrder": [
+            "customer",
+            "deviceId"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "deviceId": {
+              "description": "Immutable ID of the browser.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "projection": {
+              "description": "Restrict information returned to a set of selected fields. FULL or BASIC.",
+              "location": "query",
+              "type": "string"
+            }
+          },
+          "path": "{customer}/devices/chromebrowsers/{deviceId}",
+          "response": {
+            "$ref": "ChromeBrowser"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers",
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers.readonly"
+          ]
+        },
+        "list": {
+          "description": "Retrieves a paginated list of all the browsers in a domain.",
+          "flatPath": "{customer}/devices/chromebrowsers",
+          "httpMethod": "GET",
+          "id": "cbcm.chromebrowsers.list",
+          "parameterOrder": [
+            "customer"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "maxResults": {
+              "description": "Maximum number of results to return.",
+              "format": "int32",
+              "location": "query",
+              "maximum": "100",
+              "minimum": "1",
+              "type": "integer"
+            },
+            "orderBy": {
+              "description": "property to use for sorting results.",
+              "location": "query",
+              "type": "string"
+            },
+            "orgUnitPath": {
+              "description": "The full path of the organizational unit or its unique ID.",
+              "location": "query",
+              "type": "string"
+            },
+            "pageToken": {
+              "description": "Token to specify the next page in the list.",
+              "location": "query",
+              "type": "string"
+            },
+            "projection": {
+              "description": "Restrict information returned to a set of selected fields. FULL or BASIC.",
+              "location": "query",
+              "type": "string"
+            },
+            "query": {
+              "description": "Search string using the list page query language.",
+              "location": "query",
+              "type": "string"
+            },
+            "sortOrder": {
+              "description": "Whether to return results in ascending or descending order. Must be used with the orderBy parameter.",
+              "location": "query",
+              "type": "string"
+            }
+          },
+          "path": "{customer}/devices/chromebrowsers",
+          "response": {
+            "$ref": "ChromeBrowsers"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers",
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers.readonly"
+          ]
+        },
+        "moveChromeBrowsersToOu": {
+          "description": "Move Chrome Browsers Device between Organization Units",
+          "flatPath": "{customer}/devices/chromebrowsers/moveChromeBrowsersToOu",
+          "httpMethod": "POST",
+          "id": "cbcm.chromebrowsers.moveChromeBrowsersToOu",
+          "parameterOrder": [
+            "customer"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "path": "{customer}/devices/chromebrowsers/moveChromeBrowsersToOu",
+          "request": {
+            "$ref": "MoveChromeBrowsersRequest"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers"
+          ]
+        },
+        "update": {
+          "description": "Updates a browser.",
+          "flatPath": "{customer}/devices/chromebrowsers/{deviceId}",
+          "httpMethod": "PUT",
+          "id": "cbcm.chromebrowsers.update",
+          "parameterOrder": [
+            "customer",
+            "deviceId"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "deviceId": {
+              "description": "Immutable ID of the browser.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "projection": {
+              "description": "BASIC or FULL",
+              "location": "query",
+              "type": "string"
+            }
+          },
+          "path": "{customer}/devices/chromebrowsers/{deviceId}",
+          "request": {
+            "$ref": "ChromeBrowser"
+          },
+          "response": {
+            "$ref": "ChromeBrowser"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers"
+          ]
+        }
+      }
+    },
+    "enrollmentTokens": {
+      "methods": {
+        "list": {
+          "description": "Retrieves a paginated list of all the browser entollment tokens in a domain.",
+          "flatPath": "{customer}/chrome/enrollmentTokens",
+          "httpMethod": "GET",
+          "id": "cbcm.enrollmentTokens.list",
+          "parameterOrder": [
+            "customer"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "pageSize": {
+              "description": "Maximum number of results to return.",
+              "format": "int32",
+              "location": "query",
+              "maximum": "100",
+              "minimum": "1",
+              "type": "integer"
+            },
+            "orgUnitPath": {
+              "description": "The full path of the organizational unit or its unique ID.",
+              "location": "query",
+              "type": "string"
+            },
+            "pageToken": {
+              "description": "Token to specify the next page in the list.",
+              "location": "query",
+              "type": "string"
+            },
+            "query": {
+              "description": "Search string using the list page query language.",
+              "location": "query",
+              "type": "string"
+            }
+          },
+          "path": "{customer}/chrome/enrollmentTokens",
+          "response": {
+            "$ref": "EnrollmentTokens"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers",
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers.readonly"
+          ]
+        },
+        "create": {
+          "description": "Creates a browser enrollment token in a domain.",
+          "flatPath": "{customer}/chrome/enrollmentTokens",
+          "httpMethod": "POST",
+          "id": "cbcm.enrollmentTokens.create",
+          "parameterOrder": [
+            "customer"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+	  },
+          "path": "{customer}/chrome/enrollmentTokens",
+          "request": {
+            "$ref": "CreateEnrollmentTokenRequest"
+          },
+          "response": {
+            "$ref": "EnrollmentToken"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers"
+          ]
+	},
+        "revoke": {
+          "description": "Revokes a browser enrollment token in a domain.",
+          "flatPath": "{customer}/chrome/enrollmentTokens/{tokenPermanentId}:revoke",
+          "httpMethod": "POST",
+          "id": "cbcm.enrollmentTokens.revoke",
+          "parameterOrder": [
+            "customer",
+	    "tokenPermanentId"
+          ],
+          "parameters": {
+            "customer": {
+              "description": "Immutable ID of the G Suite account.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "tokenPermanentId": {
+              "description": "Unique identifier for an enrollment token.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+	  },
+          "path": "{customer}/chrome/enrollmentTokens/{tokenPermanentId}:revoke",
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.directory.device.chromebrowsers"
+          ]
+	}
+      }
+    }
+  },
+  "revision": "20201203",
+  "rootUrl": "https://www.googleapis.com/admin/directory/v1.1beta1/customer/",
+  "schemas": {
+    "ChromeBrowser": {
+      "id": "ChromeBrowser",
+      "properties": {
+        "annotatedAssetId": {
+          "description": "Asset identifier as annotated by the administrator or specified during enrollment.",
+          "type": "string"
+        },
+        "annotatedLocation": {
+          "description": "Address or location of the device as annotated by the administrator.",
+          "type": "string"
+        },
+        "annotatedNotes": {
+          "description": "Notes about this device as annotated by the administrator",
+          "type": "string"
+        },
+        "annotatedUser": {
+          "description": "User of the device as annotated by the administrator.",
+          "type": "string"
+        },
+        "deviceId": {
+          "annotations": {
+            "required": [
+              "cbcm.chromebrowsers.update"
+            ]
+          },
+          "description": "The unique ID of the device.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "ChromeBrowsers": {
+      "id": "ChromeBrowsers",
+      "properties": {
+        "browsers": {
+          "description": "List of Chrome browser objects.",
+          "items": {
+            "$ref": "ChromeBrowser"
+          },
+          "type": "array"
+        },
+        "etag": {
+          "description": "ETag of the resource.",
+          "type": "string"
+        },
+        "kind": {
+          "default": "admin#directory#chromeosdevices",
+          "description": "Kind of resource this is.",
+          "type": "string"
+        },
+        "nextPageToken": {
+          "description": "Token used to access the next page of this result. To access the next page, use this token's value in the `pageToken` query string of this request.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "EnrollmentToken": {
+      "id": "EnrollmentToken",
+      "properties": {
+        "kind": {
+          "default": "admin#directory#chromeEnrollmentToken",
+          "description": "Kind of resource this is.",
+          "type": "string"
+        },
+        "tokenId": {
+          "description": "Enrollment Token ID.",
+          "type": "string"
+        },
+        "tokenPermanentId": {
+          "description": "Enrollment Token Permanent ID.",
+          "type": "string"
+        },
+        "customerId": {
+          "description": "Immutable ID of the G Suite account.",
+          "type": "string"
+        },
+	"orgUnitPath": {
+          "description": "The full path of the organizational unit or its unique ID.",
+          "type": "string"
+        },
+	"creatorId": {
+          "description": "Creator ID.",
+          "type": "string"
+        },
+	"createTime": {
+          "description": "Creation Time.",
+          "type": "string"
+        },
+	"revokerId": {
+          "description": "Revoker ID.",
+          "type": "string"
+        },
+	"revokeTime": {
+          "description": "Revoke Time",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "EnrollmentTokens": {
+      "id": "EnrollmentTokens",
+      "properties": {
+        "chrome_enrollment_tokens": {
+          "description": "List of Chrome browser enrollment token objects.",
+          "items": {
+            "$ref": "EnrollmentToken"
+          },
+          "type": "array"
+        },
+        "kind": {
+          "default": "admin#directory#chromeEnrollmentTokens",
+          "description": "Kind of resource this is.",
+          "type": "string"
+        },
+        "nextPageToken": {
+          "description": "Token used to access the next page of this result. To access the next page, use this token's value in the `pageToken` query string of this request.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "CreateEnrollmentTokenRequest": {
+      "id": "CreateEnrollmentTokenRequest",
+      "properties": {
+	"org_unit_path": {
+          "description": "The full path of the organizational unit or its unique ID.",
+          "type": "string"
+        },
+	"expire_time": {
+          "description": "Expiration Time.",
+          "type": "string"
+        },
+	"token_type": {
+          "annotations": {
+            "required": [
+              "cbcm.enrollmentTokens.create"
+            ]
+          },
+          "description": "CHROME_BROWSER.",
+          "type": "string"
+        }
+      }
+    },
+    "MoveChromeBrowsersRequest": {
+      "properties": {
+        "org_unit_path": {
+          "annotations": {
+            "required": [
+              "cbcm.chromebrowsers.moveChromeBrowsersToOu"
+            ]
+          },
+          "description": "Destination organization unit to move devices to. Full path of the organizational unit or its ID prefixed with id:",
+          "type": "string"
+        },
+        "resource_ids": {
+          "annotations": {
+            "required": [
+              "cbcm.chromebrowsers.moveChromeBrowsersToOu"
+            ]
+          },
+          "description": "List of unique device IDs of Chrome Browser Devices to move. A maximum of 600 browsers may be moved per request.",
+          "type": "array"
+        }
+      }
+    }
+  },
+  "servicePath": "",
+  "title": "Admin SDK API",
+  "version": "cbcm_v1.1beta1"
+}

src/cloudprint-v2.json

@@ -1,486 +0,0 @@
-{
- "kind": "discovery#restDescription",
- "discoveryVersion": "v1",
- "id": "cloudprint:v2",
- "name": "cloudprint",
- "version": "v2",
- "revision": "20150605",
- "title": "Cloud Print API",
- "description": "Lets you access Cloud Print Printers",
- "ownerDomain": "google.com",
- "ownerName": "Google",
- "icons": {
-  "x16": "http://www.google.com/images/icons/product/search-16.gif",
-  "x32": "http://www.google.com/images/icons/product/search-32.gif"
- },
- "documentationLink": "https://developers.google.com/cloud-print",
- "protocol": "rest",
- "baseUrl": "https://www.google.com/",
- "basePath": "/cloudprint/",
- "rootUrl": "https://www.google.com/",
- "servicePath": "/cloudprint/",
- "parameters": {
-  "prettyPrint": {
-   "type": "boolean",
-   "description": "Returns response with indentations and line breaks.",
-   "default": "true",
-   "location": "query"
-  }
- },
- "auth": {
-  "oauth2": {
-   "scopes": {
-    "https://www.googleapis.com/auth/cloudprint": {
-     "description": "Manage Cloud Print"
-    }
-   }
-  }
- },
- "schemas": {
-  "Job": {
-   "id": "Job",
-   "type": "object",
-   "description": "Job Object",
-   "properties": {
-    "title": {
-     "type": "string",
-     "description": "Job Title"
-    },
-    "id": {
-     "type": "string",
-     "description": "Unique ID"
-    }
-   }
-  },
-  "Jobs": {
-   "id": "Jobs",
-   "type": "object",
-   "description": "List of Jobs.",
-   "properties": {
-    "jobs": {
-     "type": "array",
-     "description": "List of job objects.",
-     "items": {
-      "$ref": "Job"
-     }
-    }
-   }
-  },
-  "Printer": {
-   "id": "Printer",
-   "type": "object",
-   "description": "Printer Object",
-   "properties": {
-    "displayName": {
-     "type": "string",
-     "description": "Display Name"
-    },
-    "id": {
-     "type": "string",
-     "description": "Unique ID"
-    }
-   }
-  },
-  "Printers": {
-   "id": "Printers",
-   "type": "object",
-   "description": "List of Printers.",
-   "properties": {
-    "printers": {
-     "type": "array",
-     "description": "List of printer objects.",
-     "items": {
-      "$ref": "Printer"
-     }
-    }
-   }
-  }
- },
- "resources": {
-  "jobs": {
-   "methods": {
-    "delete": {
-     "id": "cloudprint.jobs.delete",
-     "path": "deletejob",
-     "httpMethod": "GET",
-     "parameters": {
-      "jobid": {
-       "type": "string",
-       "location": "query",
-       "required": "true"
-      }
-     }
-    },
-    "fetch": {
-     "id": "cloudprint.jobs.fetch",
-     "path": "fetch",
-     "httpMethod": "GET",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Jobs"
-     }
-    },
-    "getticket": {
-     "id": "cloudprint.jobs.getticket",
-     "path": "ticket",
-     "httpMethod": "GET",
-     "parameters": {
-      "jobid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "use_cjt": {
-       "type": "boolean",
-       "required": "true",
-       "location": "query"
-      }
-     }
-    }, 
-    "list": {
-     "id": "cloudprint.jobs.list",
-     "path": "jobs",
-     "httpMethod": "GET",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "location": "query"
-      },
-      "owner": {
-       "type": "string",
-       "location": "query"
-      },
-      "status": {
-       "type": "string",
-       "location": "query"
-      },
-      "q": {
-       "type": "string",
-       "location": "query"
-      },
-      "offset": {
-       "type": "string",
-       "location": "query"
-      },
-      "limit": {
-       "type": "string",
-       "location": "query"
-      },
-      "sortorder": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Jobs"
-     }
-    },
-    "update": {
-     "id": "cloudprint.jobs.update",
-     "path": "control",
-     "httpMethod": "GET",
-     "parameters": {
-      "jobid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "semantic_state_diff": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Jobs"
-     }
-    },
-    "resubmit": {
-     "id": "cloudprint.jobs.resubmit",
-     "path": "resubmit",
-     "httpMethod": "POST",
-     "description": "resubmit a job to new printer.",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "jobid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "ticket": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Job"
-     }
-    },
-    "submit": {
-     "id": "cloudprint.jobs.submit",
-     "path": "submit",
-     "httpMethod": "POST",
-     "description": "Send a print job to cloud print.",
-     "request": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "title": {
-       "type": "string",
-       "location": "query"
-      },
-      "ticket": {
-       "type": "string",
-       "location": "query"
-      },
-      "content": {
-       "type": "string",
-       "location": "query"
-      },
-      "contentType": {
-       "type": "string",
-       "location": "query"
-      },
-      "tag": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Job"
-     }
-    }
-   }
-  },
-  "printers": {
-   "methods": {
-    "get": {
-     "id": "cloudprint.printers.get",
-     "path": "printer",
-     "httpMethod": "GET",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "extra_fields": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Printer"
-     }
-    },
-    "list": {
-     "id": "cloudprint.printers.list",
-     "path": "search",
-     "httpMethod": "GET",
-     "description": "List all printers",
-     "parameters": {
-      "q": {
-       "type": "string",
-       "description": "Query list of printers",
-       "location": "query"
-      },
-      "type": {
-       "type": "string",
-       "description": "limit results to printers of type",
-       "location": "query"
-       },
-      "connection_status": {
-       "type": "string",
-       "description": "limit results to printers with this status",
-       "location": "query"
-      },
-      "extra_fields": {
-       "type": "string",
-       "description": "include extra fields",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Printers"
-     }
-    },
-    "share": {
-     "id": "cloudprint.printers.share",
-     "path": "share",
-     "httpMethod": "GET",
-     "description": "Share printer with user, group or domain",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "scope": {
-       "type": "string",
-       "location": "query"
-      },
-      "role": {
-       "type": "string",
-       "location": "query"
-      },
-      "type": {
-        "type": "string",
-        "location": "query"
-      },
-      "skip_notification": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "public": {
-       "type": "boolean",
-       "location": "query"
-      }
-     } 
-    },
-    "unshare": {
-     "id": "cloudprint.printers.unshare",
-     "path": "unshare",
-     "httpMethod": "GET",
-     "description": "unshare printer with user, group or domain",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "scope": {
-       "type": "string",
-       "location": "query"
-      },
-      "public": {
-       "type": "string",
-       "location": "query"
-      }
-     }
-    },
-    "delete": {
-     "id": "cloudprint.printers.delete",
-     "path": "delete",
-     "httpMethod": "GET",
-     "description": "delete a printer",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      }
-     }
-    },
-    "update": {
-     "id": "cloudprint.printers.update",
-     "path": "update",
-     "httpMethod": "GET",
-     "description": "update a printer",
-     "parameters": {
-      "isTosAccepted": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "gcpVersion": {
-       "type": "string",
-       "location": "query"
-      },
-      "setupUrl": {
-       "type": "string",
-       "location": "query"
-      },
-      "supportUrl": {
-       "type": "string",
-       "location": "query"
-      },
-      "firmware": {
-       "type": "string",
-       "location": "query"
-      },
-      "currentQuota": {
-       "type": "string",
-       "location": "query"
-      },
-      "type": {
-       "type": "string",
-       "location": "query"
-      },
-      "public": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "status": {
-       "type": "string",
-       "location": "query"
-      },
-      "proxy": {
-       "type": "string",
-       "location": "query"
-      },
-      "manufacturer": {
-       "type": "string",
-       "location": "query"
-      },
-      "defaultDisplayName": {
-       "type": "string",
-       "location": "query"
-      },
-      "displayName": {
-       "type": "string",
-       "location": "query"
-      },
-      "name": {
-       "type": "string",
-       "location": "query"
-      },
-      "uuid": {
-       "type": "string",
-       "location": "query"
-      },
-      "updateUrl": {
-       "type": "string",
-       "location": "query"
-      },
-      "ownerId": {
-       "type": "string",
-       "location": "query"
-      },
-      "model": {
-       "type": "string",
-       "location": "query"
-      },
-      "description": {
-       "type": "string",
-       "location": "query"
-      },
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "quotaEnabled": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "dailyQuota": {
-       "type": "string",
-       "location": "query"
-      }
-     }
-    } 
-   }
-  }
- }
-}

src/contactdelegation-v1.json

@@ -0,0 +1,249 @@
+{
+  "auth": {
+    "oauth2": {
+      "scopes": {
+        "https://www.googleapis.com/auth/admin.contact.delegation": {
+          "description": "View and manage your Contact Delegation"
+        }, 
+        "https://www.googleapis.com/auth/admin.contact.delegation.readonly": {
+          "description": "View your Contact Delegation"
+        }
+      }
+    }
+  }, 
+  "basePath": "",
+  "baseUrl": "https://admin.googleapis.com/admin/contacts/v1/",
+  "batchPath": "batch",
+  "canonicalName": "contactdelegation",
+  "description": "The Contact Delegation API allows Admins to delegate access of one user's, called the delegator, contacts to another user, called the delegate.",
+  "discoveryVersion": "v1",
+  "documentationLink": "https://developers.google.com/admin-sdk/contact-delegation",
+  "fullyEncodeReservedExpansion": true,
+  "icons": {
+    "x16": "http://www.google.com/images/icons/product/search-16.gif",
+    "x32": "http://www.google.com/images/icons/product/search-32.gif"
+  },
+  "id": "contactdelegation:v1",
+  "kind": "discovery#restDescription",
+  "name": "contactdelegation",
+  "ownerDomain": "google.com",
+  "ownerName": "Google",
+  "packagePath": "admin",
+  "parameters": {
+    "$.xgafv": {
+      "description": "V1 error format.",
+      "enum": [
+        "1",
+        "2"
+      ],
+      "enumDescriptions": [
+        "v1 error format",
+        "v2 error format"
+      ],
+      "location": "query",
+      "type": "string"
+    },
+    "access_token": {
+      "description": "OAuth access token.",
+      "location": "query",
+      "type": "string"
+    },
+    "alt": {
+      "default": "json",
+      "description": "Data format for response.",
+      "enum": [
+        "json",
+        "media",
+        "proto"
+      ],
+      "enumDescriptions": [
+        "Responses with Content-Type of application/json",
+        "Media download with context-dependent Content-Type",
+        "Responses with Content-Type of application/x-protobuf"
+      ],
+      "location": "query",
+      "type": "string"
+    },
+    "callback": {
+      "description": "JSONP",
+      "location": "query",
+      "type": "string"
+    },
+    "fields": {
+      "description": "Selector specifying which fields to include in a partial response.",
+      "location": "query",
+      "type": "string"
+    },
+    "key": {
+      "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
+      "location": "query",
+      "type": "string"
+    },
+    "oauth_token": {
+      "description": "OAuth 2.0 token for the current user.",
+      "location": "query",
+      "type": "string"
+    },
+    "prettyPrint": {
+      "default": "true",
+      "description": "Returns response with indentations and line breaks.",
+      "location": "query",
+      "type": "boolean"
+    },
+    "quotaUser": {
+      "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.",
+      "location": "query",
+      "type": "string"
+    },
+    "uploadType": {
+      "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").",
+      "location": "query",
+      "type": "string"
+    },
+    "upload_protocol": {
+      "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").",
+      "location": "query",
+      "type": "string"
+    }
+  },
+  "protocol": "rest",
+  "resources": {
+    "delegates": {
+      "methods": {
+        "create": {
+          "description": "Creates a contact delegations",
+          "flatPath": "users/{user}/delegates",
+          "httpMethod": "POST",
+          "id": "contactdelegations.delegates.create",
+          "parameterOrder": [
+            "user"
+          ],
+          "parameters": {
+            "user": {
+              "description": "Email address of the delegator.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "path": "users/{user}/delegates/{delegate}",
+          "request": {
+            "$ref": "Delegate"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.contact.delegation"
+          ]
+        },
+        "delete": {
+          "description": "Deletes a contact delegation.",
+          "flatPath": "users/{user}/delegates/{delegate}",
+          "httpMethod": "DELETE",
+          "id": "contactdelegations.delegates.delete",
+          "parameterOrder": [
+            "user",
+            "delegate"
+          ],
+          "parameters": {
+            "delegate": {
+              "description": "Email address of the delegate",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "user": {
+              "description": "Email address of the delegator.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "path": "users/{user}/delegates/{delegate}",
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.contact.delegation"
+          ]
+        },
+        "list": {
+          "description": "Lists contact delegates for a user",
+          "flatPath": "users/{user}/delegates",
+          "httpMethod": "GET",
+          "id": "contactdelegations.delegates.list",
+          "parameterOrder": [
+            "user"
+          ],
+          "parameters": {
+            "pageSize": {
+              "description": "Determines how many delegates are returned in each response. ",
+              "format": "int32",
+              "location": "query",
+              "minimum": "1",
+              "type": "integer"
+            },
+            "pageToken": {
+              "description": "Token to specify the next page in the list.",
+              "location": "query",
+              "type": "string"
+            },
+            "user": {
+              "description": "Email address of the delegator.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "path": "users/{user}/delegates",
+          "response": {
+            "$ref": "Delegates"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/admin.contact.delegation",
+            "https://www.googleapis.com/auth/admin.contact.delegation.readonly"
+          ]
+        }
+      }
+    }
+  },
+  "rootUrl": "https://admin.googleapis.com/admin/contacts/v1/",
+  "schemas": {
+    "Delegate": {
+      "description": "JSON template for a delegate.",
+      "id": "Delegate",
+      "properties": {
+        "email": {
+          "description": "Email of the delegate.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "Delegates": {
+      "id": "Delegates",
+      "properties": {
+        "delegates": {
+          "description": "List of delegates.",
+          "items": {
+            "$ref": "Delegate"
+          },
+          "type": "array"
+        },
+        "etag": {
+          "description": "ETag of the resource.",
+          "type": "string"
+        },
+        "kind": {
+          "default": "",
+          "description": "Kind of resource this is.",
+          "type": "string"
+        },
+        "nextPageToken": {
+          "description": "Token used to access the next page of this result. To access the next page, use this token's value in the `pageToken` query string of this request.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    }
+  },
+  "servicePath": "",
+  "title": "Contact Delegation API",
+  "version": "v1",
+  "version_module": true
+}

src/gam-install.sh

@@ -10,6 +10,7 @@ OPTIONS:
    -d      Directory where gam folder will be installed. Default is \$HOME/bin/
    -a      Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m".
    -o      OS we are running (linux, macos). Default is to detect your OS with "uname -s".
+   -b      OS version. Default is to detect on MacOS and Linux.
    -l      Just upgrade GAM to latest version. Skips project creation and auth.
    -p      Profile update (true, false). Should script add gam command to environment. Default is true.
    -u      Admin user email address to use with GAM. Default is to prompt.
@@ -21,20 +22,23 @@ EOF
 target_dir="$HOME/bin"
 gamarch=$(uname -m)
 gamos=$(uname -s)
+osversion=""
 update_profile=true
 upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.23 2.19 2.15"
+gam_glibc_vers="2.31 2.27 2.23"
+gam_macos_vers="10.15.6 10.14.6 10.13.6"
 
-while getopts "hd:a:o:lp:u:r:v:" OPTION
+while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
      case $OPTION in
          h) usage; exit;;
          d) target_dir="$OPTARG";;
          a) gamarch="$OPTARG";;
          o) gamos="$OPTARG";;
+         b) osversion="$OPTARG";;
          l) upgrade_only=true;;
          p) update_profile="$OPTARG";;
          u) adminuser="$OPTARG";;
@@ -48,7 +52,7 @@ done
 target_dir=${target_dir%/}
 
 update_profile() {
-	[ -f "$1" ] || return 1
+	[ $2 -eq 1 ] || [ -f "$1" ] || return 1
 
 	grep -F "$alias_line" "$1" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
@@ -79,13 +83,26 @@ echo -e '\x1B[0m'
 
 version_gt()
 {
-test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+# MacOS < 10.13 doesn't support sort -V
+echo "" | sort -V > /dev/null 2>&1
+vsort_failed=$?
+if [ "${1}" = "${2}" ]; then
+  true
+elif (( $vsort_failed != 0 )); then
+  false
+else
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+fi
 }
 
 case $gamos in
   [lL]inux)
     gamos="linux"
-    this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    if [ "$osversion" == "" ]; then
+      this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    else
+      this_glibc_ver=$osversion
+    fi
     echo "This Linux distribution uses glibc $this_glibc_ver"
     useglibc="legacy"
     for gam_glibc_ver in $gam_glibc_vers; do
@@ -97,25 +114,39 @@ case $gamos in
     done
     case $gamarch in
       x86_64) gamfile="linux-x86_64-$useglibc.tar.xz";;
-      i?86) gamfile="linux-i686.tar.xz";;
-      arm|armv7l) gamfile="linux-armv7l.tar.xz";;
-      arm64|aarch64) gamfile="linux-aarch64.tar.xz";;
+      arm64|aarch64) gamfile="linux-arm64-$useglibc.tar.xz";;
       *)
-        echo_red "ERROR: this installer currently only supports i386, x86_64, arm and arm64 Linux. Looks like you're running on $gamarch. Exiting."
+        echo_red "ERROR: this installer currently only supports x86_64 and arm64 Linux. Looks like you're running on $gamarch. Exiting."
         exit
     esac
     ;;
   [Mm]ac[Oo][sS]|[Dd]arwin)
-    osver=$(sw_vers -productVersion | awk -F'.' '{print $2}')
-    if (( $osver < 13 )); then
-      echo_red "ERROR: GAM currently requires MacOS 10.13 or newer. You are running MacOS 10.$osver. Please upgrade." 
-      exit
-    else
-      echo_green "Good, you're running MacOS 10.$osver..."
-    fi
     gamos="macos"
+    if [ "$osversion" == "" ]; then
+      this_macos_ver=$(sw_vers -productVersion)
+    else
+      this_macos_ver=$osversion
+    fi
+    echo "You are running MacOS $this_macos_ver"
+    use_macos_ver=""
+    for gam_macos_ver in $gam_macos_vers; do
+      if version_gt $this_macos_ver $gam_macos_ver; then
+        use_macos_ver="MacOS$gam_macos_ver"
+        echo_green "Using GAM compiled on $use_macos_ver"
+        break
+      fi
+    done
+    if [ "$use_macos_ver" == "" ]; then
+      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
+      exit
+    fi
     gamfile="macos-x86_64.tar.xz"
     ;;
+  MINGW64_NT*)
+    gamos="windows"
+    echo "You are running Windows"
+    gamfile="-windows-x86_64.zip"
+    ;;
   *)
     echo_red "Sorry, this installer currently only supports Linux and MacOS. Looks like you're runnning on $gamos. Exiting."
     exit
@@ -128,8 +159,14 @@ else
   release_url="https://api.github.com/repos/jay0lee/GAM/releases/tags/v$gamversion"
 fi
 
-echo_yellow "Checking GitHub URL $release_url for $gamversion GAM release..."
-release_json=$(curl -s $release_url 2>&1 /dev/null)
+if [ -z ${GHCLIENT+x} ]; then
+  check_type="unauthenticated"
+else
+  check_type="authenticated"
+fi
+
+echo_yellow "Checking GitHub URL $release_url for $gamversion GAM release ($check_type)..."
+release_json=$(curl -s $GHCLIENT $release_url 2>&1 /dev/null)
 
 echo_yellow "Getting file and download URL..."
 # Python is sadly the nearest to universal way to safely handle JSON with Bash
@@ -161,11 +198,16 @@ try:
 except KeyError:
   print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
-pycmd="python"
+pycmd="python3"
 $pycmd -V >/dev/null 2>&1
 rc=$?
 if (( $rc != 0 )); then
-  pycmd="python3"
+  pycmd="python"
+fi
+$pycmd -V >/dev/null 2>&1
+rc=$?
+if (( $rc != 0 )); then
+  pycmd="python2"
 fi
 $pycmd -V >/dev/null 2>&1
 rc=$?
@@ -187,14 +229,22 @@ fi
 # Temp dir for archive
 #temp_archive_dir=$(mktemp -d)
 temp_archive_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
-echo_yellow "Downloading file $name from $browser_download_url to $temp_archive_dir."
+
+# Clean up after ourselves even if we are killed with CTRL-C
+trap "rm -rf $temp_archive_dir" EXIT
+
+echo_yellow "Downloading file $name from $browser_download_url to $temp_archive_dir ($check_type)..."
 # Save archive to temp w/o losing our path
-(cd $temp_archive_dir && curl -O -L $browser_download_url)
+(cd $temp_archive_dir && curl -O -L $GHCLIENT $browser_download_url)
 
 mkdir -p "$target_dir"
 
 echo_yellow "Extracting archive to $target_dir"
-tar xf $temp_archive_dir/$name -C "$target_dir"
+if [[ "${name}" == *.tar.xz ]]; then
+  tar xf $temp_archive_dir/$name -C "$target_dir"
+else
+  unzip "${temp_archive_dir}/${name}" -d "${target_dir}"
+fi
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: extracting the GAM archive with tar failed with error $rc. Exiting."
@@ -203,9 +253,23 @@ else
   echo_green "Finished extracting GAM archive."
 fi
 
+# Update profile to add gam command
+if [ "$update_profile" = true ]; then
+  alias_line="function gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
+  if [ "$gamos" == "linux" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0
+    update_profile "$HOME/.zshrc" 0
+  elif [ "$gamos" == "macos" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.zshrc" 1
+  fi
+else
+  echo_yellow "skipping profile update."
+fi
+
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
-  "$target_dir/gam/gam" version
+  "$target_dir/gam/gam" version extended
   rc=$?
   if (( $rc != 0 )); then
     echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -216,18 +280,6 @@ if [ "$upgrade_only" = true ]; then
   exit
 fi
 
-# Update profile to add gam command
-if [ "$update_profile" = true ]; then
-  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
-  if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile"
-  elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile"
-  fi
-else
-  echo_yellow "skipping profile update."
-fi
-
 while true; do
   read -p "Can you run a full browser on this machine? (usually Y for MacOS, N for Linux if you SSH into this machine) " yn
   case $yn in
@@ -251,7 +303,7 @@ while true; do
   case $yn in
     [Yy]*)
       if [ "$adminuser" == "" ]; then
-        read -p "Please enter your G Suite admin email address: " adminuser
+        read -p "Please enter your Google Workspace admin email address: " adminuser
       fi
       "$target_dir/gam/gam" create project $adminuser
       rc=$?
@@ -275,7 +327,7 @@ done
 
 admin_authorized=false
 while $project_created; do
-  read -p "Are you ready to authorize GAM to perform G Suite management operations as your admin account? (yes or no) " yn
+  read -p "Are you ready to authorize GAM to perform Google Workspace management operations as your admin account? (yes or no) " yn
   case $yn in
     [Yy]*)
       "$target_dir/gam/gam" oauth create $adminuser
@@ -300,11 +352,11 @@ done
 
 service_account_authorized=false
 while $project_created; do
-  read -p "Are you ready to authorize GAM to manage G Suite user data and settings? (yes or no) " yn
+  read -p "Are you ready to authorize GAM to manage Google Workspace user data and settings? (yes or no) " yn
   case $yn in
     [Yy]*)
       if [ "$regularuser" == "" ]; then
-        read -p "Please enter the email address of a regular G Suite user: " regularuser
+        read -p "Please enter the email address of a regular Google Workspace user: " regularuser
       fi
       echo_yellow "Great! Checking service account scopes.This will fail the first time. Follow the steps to authorize and retry. It can take a few minutes for scopes to PASS after they've been authorized in the admin console."
       "$target_dir/gam/gam" user $adminuser check serviceaccount
@@ -328,7 +380,7 @@ while $project_created; do
 done
 
 echo_green "Here's information about your new GAM installation:"
-"$target_dir/gam/gam" version
+"$target_dir/gam/gam" version extended
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -339,6 +391,3 @@ echo_green "GAM installation and setup complete!"
 if [ "$update_profile" = true ]; then
   echo_green "Please restart your terminal shell or to get started right away run:\n\n$alias_line"
 fi
-
-# Clean up after ourselves even if we are killed with CTRL-C
-trap "rm -rf $temp_archive_dir" EXIT

src/gam-setup.bat

@@ -27,7 +27,7 @@
 @  goto createproject
    )
 @echo(
-@set /p adminemail= "Please enter your G Suite admin email address: "
+@set /p adminemail= "Please enter your Google Workspace admin email address: "
 @gam create project %adminemail%
 @if not ERRORLEVEL 1 goto projectdone
 @echo(
@@ -37,7 +37,7 @@
 
 :adminauth
 @echo(
-@set /p yn= "Are you ready to authorize GAM to perform G Suite management operations as your admin account? [y or n] "
+@set /p yn= "Are you ready to authorize GAM to perform Google Workspace management operations as your admin account? [y or n] "
 @if /I "%yn%"=="n" (
 @  echo(
 @  echo You can authorize an admin later by running:
@@ -59,7 +59,7 @@
 
 :saauth
 @echo(
-@set /p yn= "Are you ready to authorize GAM to manage G Suite user data and settings? [y or n] "
+@set /p yn= "Are you ready to authorize GAM to manage Google Workspace user data and settings? [y or n] "
 @if /I "%yn%"=="n" (
 @  echo(
 @  echo You can authorize a service account later by running:
@@ -73,7 +73,7 @@
 @  goto saauth
    )
 @echo(
-@set /p regularuser= "Please enter the email address of a regular G Suite user: "
+@set /p regularuser= "Please enter the email address of a regular Google Workspace user: "
 @echo Great! Checking service account scopes. This will fail the first time. Follow the steps to authorize and retry. It can take a few minutes for scopes to PASS after they've been authorized in the admin console.
 @gam user %regularuser% check serviceaccount
 @if not ERRORLEVEL 1 goto sadone

src/gam.spec

@@ -0,0 +1,46 @@
+# -*- mode: python -*-
+
+import sys
+
+import importlib
+from PyInstaller.utils.hooks import copy_metadata
+
+sys.modules['FixTk'] = None
+
+# dynamically determine where httplib2/cacerts.txt lives
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+extra_files = [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+
+extra_files += copy_metadata('google-api-python-client')
+extra_files += [('cbcm-v1.1beta1.json', '.')]
+extra_files += [('contactdelegation-v1.json', '.')]
+extra_files += [('versionhistory-v1.json', '.')]
+
+hidden_imports = [
+     'gam.auth.yubikey',
+     ]
+
+a = Analysis(['gam/__main__.py'],
+             hiddenimports=hidden_imports,
+             hookspath=None,
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             datas=extra_files,
+             runtime_hooks=None)
+
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+
+
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam',
+          debug=False,
+          strip=None,
+          upx=False,
+          console=True )

src/gam.wxs

@@ -41,7 +41,7 @@
     <ComponentGroup
         Id="ProductComponents"
         Directory="INSTALLFOLDER"
-        Source="gam-64">
+	Source="dist/gam">
       <Component Id="gam_exe" Guid="886abc07-73c5-4acc-9f71-58daf62aabc1">
         <File Name="gam.exe" KeyPath="yes" />
         <Environment Id="PATH" Name="PATH" Value="[INSTALLFOLDER]" Permanent="yes" Part="last" Action="set" System="yes" />
@@ -49,9 +49,6 @@
       <Component Id="license" Guid="7a15de2e-fb91-4d0a-b8bf-c8b19c68f569">
         <File Name="LICENSE" KeyPath="yes" />
       </Component>
-      <Component Id="whatsnew_txt" Guid="6aa9863c-90d9-412f-9b73-fda82549a950">
-        <File Name="whatsnew.txt" KeyPath="yes" />
-      </Component>
       <Component Id="gam_setup_bat" Guid="ef01f93a-4b50-488a-9c04-ec5e13e66218">
         <File Name="gam-setup.bat" KeyPath="yes" />
       </Component>

src/gam/__main__.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# GAM
+#
+# Copyright 2019, LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GAM is a command line tool which allows Administrators to control their Google Workspace domain and accounts.
+
+With GAM you can programmatically create users, turn on/off services for users like POP and Forwarding and much more.
+For more information, see https://git.io/gam
+"""
+
+import sys
+from multiprocessing import freeze_support
+from multiprocessing import set_start_method
+
+from gam import controlflow
+import gam
+
+
+def main(argv):
+    freeze_support()
+    if sys.platform == 'darwin':
+        # https://bugs.python.org/issue33725 in Python 3.8.0 seems
+        # to break parallel operations with errors about extra -b
+        # command line arguments
+        set_start_method('fork')
+    if sys.version_info[0] < 3 or sys.version_info[1] < 6:
+        controlflow.system_error_exit(
+            5,
+            f'GAM requires Python 3.6 or newer. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.'
+            % sys.version_info[:3])
+    sys.exit(gam.ProcessGAMCommand(sys.argv))
+
+
+# Run from command line
+if __name__ == '__main__':
+    main(sys.argv)

src/gam/auth/__init__.py

@@ -0,0 +1,57 @@
+"""Authentication/Credentials general purpose and convenience methods."""
+
+import json
+import os
+
+from google.auth.jwt import Credentials as JWTCredentials
+
+import gam
+from gam import utils
+
+from gam.auth import oauth
+from gam.var import _FN_OAUTH2_TXT
+from gam.var import _FN_OAUTH2SERVICE_JSON
+from gam.var import GC_OAUTH2_TXT
+from gam.var import GC_OAUTH2SERVICE_JSON
+from gam.var import GC_ENABLE_DASA
+from gam.var import GC_Values
+
+yubikey = utils.LazyLoader('yubikey', globals(), 'gam.auth.yubikey')
+# TODO: Move logic that determines file name into this module. We should be able
+# to discover the file location without accessing a private member or waiting
+# for a global initialization.
+
+
+def get_admin_credentials_filename():
+    """Gets the name of the file that stores the admin account credentials."""
+    # If the environment globals are loaded, use the set global value. It may have
+    # some custom name in it. Otherwise, just use the default name.
+    if GC_Values[GC_ENABLE_DASA]:
+        return GC_Values[GC_OAUTH2SERVICE_JSON] if GC_Values[GC_OAUTH2SERVICE_JSON] else _FN_OAUTH2SERVICE_JSON
+    else:
+        return GC_Values[GC_OAUTH2_TXT] if GC_Values[GC_OAUTH2_TXT] else _FN_OAUTH2_TXT
+
+
+def get_admin_credentials(api=None):
+    """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+    credential_file = get_admin_credentials_filename()
+    if not os.path.isfile(credential_file):
+        raise oauth.InvalidCredentialsFileError
+    with open(credential_file, 'r') as f:
+        creds_data = json.load(f)
+    # Validate that enable DASA matches content of authorization file
+    if GC_Values[GC_ENABLE_DASA] and 'private_key_id' in creds_data:
+        audience = f'https://{api}.googleapis.com/'
+        key_type = creds_data.get('key_type', 'default')
+        if key_type == 'default':
+            return JWTCredentials.from_service_account_info(creds_data,
+                                                            audience=audience)
+        elif key_type == 'yubikey':
+            yksigner = yubikey.YubiKey(creds_data)
+            return JWTCredentials._from_signer_and_info(yksigner,
+                creds_data,
+                audience=audience)
+    elif not GC_Values[GC_ENABLE_DASA] and 'token' in creds_data:
+        return oauth.Credentials.from_credentials_file(credential_file)
+    else:
+        raise oauth.InvalidCredentialsFileError

src/gam/auth/oauth.py

@@ -0,0 +1,560 @@
+"""OAuth2.0 user credentials."""
+
+import datetime
+import json
+import os
+import re
+import threading
+from urllib.parse import urlencode
+
+from filelock import FileLock
+import google_auth_oauthlib.flow
+import google.oauth2.credentials
+import google.oauth2.id_token
+
+from gam import fileutils
+from gam import transport
+from gam.var import GM_Globals
+from gam.var import GM_WINDOWS
+from gam import utils
+
+MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
+                                        'browser:\n\n\t{url}\n')
+MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
+MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
+                                             ' visit:\n\n\t{url}\n\nIf your '
+                                             'browser is on a different machine'
+                                             ' then press CTRL+C and create a '
+                                             'file called nobrowser.txt in the '
+                                             'same folder as GAM.\n')
+MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
+                                ' close this browser window and return to GAM.')
+
+
+class CredentialsError(Exception):
+    """Base error class."""
+    pass
+
+
+class InvalidCredentialsFileError(CredentialsError):
+    """Error raised when a file cannot be opened into a credentials object."""
+    pass
+
+
+class EmptyCredentialsFileError(InvalidCredentialsFileError):
+    """Error raised when a credentials file contains no content."""
+    pass
+
+
+class InvalidClientSecretsFileFormatError(CredentialsError):
+    """Error raised when a client secrets file format is invalid."""
+    pass
+
+
+class InvalidClientSecretsFileError(CredentialsError):
+    """Error raised when client secrets file cannot be read."""
+    pass
+
+
+class Credentials(google.oauth2.credentials.Credentials):
+    """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
+
+    DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+    def __init__(self,
+                 token,
+                 refresh_token=None,
+                 id_token=None,
+                 token_uri=None,
+                 client_id=None,
+                 client_secret=None,
+                 scopes=None,
+                 quota_project_id=None,
+                 expiry=None,
+                 id_token_data=None,
+                 filename=None):
+        """A thread-safe OAuth2.0 credentials object.
+
+    Credentials adds additional utility properties and methods to a
+    standard OAuth2.0 credentials object. When used to store credentials on
+    disk, it implements a file lock to avoid collision during writes.
+
+    Args:
+      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
+        information is provided.
+      refresh_token: String, The OAuth 2.0 refresh token. If specified,
+        credentials can be refreshed.
+      id_token: String, The Open ID Connect ID Token.
+      token_uri: String, The OAuth 2.0 authorization server's token endpoint
+        URI. Must be specified for refresh, can be left as None if the token can
+        not be refreshed.
+      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
+        can be left as None if the token can not be refreshed.
+      client_secret: String, The OAuth 2.0 client secret. Must be specified for
+        refresh, can be left as None if the token can not be refreshed.
+      scopes: Sequence[str], The scopes used to obtain authorization.
+        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
+          not request additional scopes after authorization. The scopes must be
+          derivable from the refresh token if refresh information is provided
+          (e.g. The refresh token scopes are a superset of this or contain a
+          wild card scope like
+            'https://www.googleapis.com/auth/any-api').
+      quota_project_id: String, The project ID used for quota and billing. This
+        project may be different from the project used to create the
+        credentials.
+      expiry: datetime.datetime, The time at which the provided token will
+        expire.
+      id_token_data: Oauth2.0 ID Token data which was previously fetched for
+        this access token against the google.oauth2.id_token library.
+      filename: String, Path to a file that will be used to store the
+        credentials. If provided, a lock file of the same name and a ".lock"
+          extension will be created for concurrency controls. Note: New
+            credentials are not saved to disk until write() or refresh() are
+            called.
+
+    Raises:
+      TypeError: If id_token_data is not the required dict type.
+    """
+        super(Credentials, self).__init__(token=token,
+                                          refresh_token=refresh_token,
+                                          id_token=id_token,
+                                          token_uri=token_uri,
+                                          client_id=client_id,
+                                          client_secret=client_secret,
+                                          scopes=scopes,
+                                          quota_project_id=quota_project_id)
+
+        # Load data not restored by the super class
+        self.expiry = expiry
+        if id_token_data and not isinstance(id_token_data, dict):
+            raise TypeError(f'Expected type id_token_data dict but received '
+                            f'{type(id_token_data)}')
+        self._id_token_data = id_token_data.copy() if id_token_data else None
+
+        # If a filename is provided, use a lock file to control concurrent access
+        # to the resource. If no filename is provided, use a thread lock that has
+        # the same interface as FileLock in order to simplify the implementation.
+        if filename:
+            # Convert relative paths into absolute
+            self._filename = os.path.abspath(filename)
+            lock_file = os.path.abspath(f'{self._filename}.lock')
+            self._lock = FileLock(lock_file)
+        else:
+            self._filename = None
+            self._lock = _FileLikeThreadLock()
+
+    # Use a property to prevent external mutation of the filename.
+    @property
+    def filename(self):
+        return self._filename
+
+    @classmethod
+    def from_authorized_user_info(cls, info, filename=None):
+        """Generates Credentials from JSON containing authorized user info.
+
+    Args:
+      info: Dict, authorized user info in Google format.
+      filename: String, the filename used to store these credentials on disk. If
+        no filename is provided, the credentials will not be saved to disk.
+
+    Raises:
+      ValueError: If missing fields are detected in the info.
+    """
+        # We need all of these keys
+        keys_needed = set(('client_id', 'client_secret'))
+        # We need 1 or more of these keys
+        keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
+        missing = keys_needed.difference(info.keys())
+        has_one_of = set(info) & keys_need_one_of
+        if missing or not has_one_of:
+            raise ValueError(
+                'Authorized user info was not in the expected format, missing '
+                f'fields {", ".join(missing)} and one of '
+                f'{", ".join(keys_need_one_of)}.')
+
+        expiry = info.get('token_expiry')
+        if expiry:
+            # Convert the raw expiry to datetime
+            expiry = datetime.datetime.strptime(expiry,
+                                                Credentials.DATETIME_FORMAT)
+        id_token_data = info.get('decoded_id_token')
+
+        # Provide backwards compatibility with field names when loading from JSON.
+        # Some field names may be different, depending on when/how the credentials
+        # were pickled.
+        return cls(token=info.get('token', info.get('auth_token', '')),
+                   refresh_token=info.get('refresh_token', ''),
+                   id_token=info.get('id_token_jwt', info.get('id_token')),
+                   token_uri=info.get('token_uri'),
+                   client_id=info['client_id'],
+                   client_secret=info['client_secret'],
+                   scopes=info.get('scopes'),
+                   quota_project_id=info.get('quota_project_id'),
+                   expiry=expiry,
+                   id_token_data=id_token_data,
+                   filename=filename)
+
+    @classmethod
+    def from_google_oauth2_credentials(cls, credentials, filename=None):
+        """Generates Credentials from a google.oauth2.Credentials object."""
+        info = json.loads(credentials.to_json())
+        # Add properties which are not exported with the native to_json() output.
+        info['id_token'] = credentials.id_token
+        if credentials.expiry:
+            info['token_expiry'] = credentials.expiry.strftime(
+                Credentials.DATETIME_FORMAT)
+        info['quota_project_id'] = credentials.quota_project_id
+
+        return cls.from_authorized_user_info(info, filename=filename)
+
+    @classmethod
+    def from_credentials_file(cls, filename):
+        """Generates Credentials from a stored Credentials file.
+
+    The same file will be used to save the credentials when the access token is
+    refreshed.
+
+    Args:
+      filename: String, the name of a file containing JSON credentials to load.
+        The same filename will be used to save credentials back to disk.
+
+    Returns:
+      The credentials loaded from disk.
+
+    Raises:
+      InvalidCredentialsFileError: When the credentials file cannot be opened.
+      EmptyCredentialsFileError: When the provided file contains no credentials.
+    """
+        file_content = fileutils.read_file(filename,
+                                           continue_on_error=True,
+                                           display_errors=False)
+        if file_content is None:
+            raise InvalidCredentialsFileError(
+                f'File {filename} could not be opened')
+        info = json.loads(file_content)
+        if not info:
+            raise EmptyCredentialsFileError(
+                f'File {filename} contains no credential data')
+
+        try:
+            # We read the existing data from the passed in file, but we also want to
+            # save future data/tokens in the same place.
+            return cls.from_authorized_user_info(info, filename=filename)
+        except ValueError as e:
+            raise InvalidCredentialsFileError(str(e))
+
+    @classmethod
+    def from_client_secrets(cls,
+                            client_id,
+                            client_secret,
+                            scopes,
+                            access_type='offline',
+                            login_hint=None,
+                            filename=None,
+                            use_console_flow=False):
+        """Runs an OAuth Flow from client secrets to generate credentials.
+
+    Args:
+      client_id: String, The OAuth2.0 Client ID.
+      client_secret: String, The OAuth2.0 Client Secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      filename: String, the path to a file to use to save the credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Returns:
+      Credentials
+    """
+        client_config = {
+            'installed': {
+                'client_id': client_id,
+                'client_secret': client_secret,
+                'redirect_uris': [
+                    'http://localhost', 'urn:ietf:wg:oauth:2.0:oob'
+                ],
+                'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+                'token_uri': 'https://oauth2.googleapis.com/token',
+            }
+        }
+
+        flow = _ShortURLFlow.from_client_config(client_config,
+                                                scopes,
+                                                autogenerate_code_verifier=True)
+        flow_kwargs = {'access_type': access_type}
+        if login_hint:
+            flow_kwargs['login_hint'] = login_hint
+
+        # TODO: Move code for browser detection somewhere in this file so that the
+        # messaging about `nobrowser.txt` is co-located with the logic that uses it.
+        if use_console_flow:
+            flow.run_console(
+                authorization_prompt_message=
+                MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
+                authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
+                **flow_kwargs)
+        else:
+            flow.run_local_server(authorization_prompt_message=
+                                  MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
+                                  success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
+                                  **flow_kwargs)
+        return cls.from_google_oauth2_credentials(flow.credentials,
+                                                  filename=filename)
+
+    @classmethod
+    def from_client_secrets_file(cls,
+                                 client_secrets_file,
+                                 scopes,
+                                 access_type='offline',
+                                 login_hint=None,
+                                 credentials_file=None,
+                                 use_console_flow=False):
+        """Runs an OAuth Flow from secrets stored on disk to generate credentials.
+
+    Args:
+      client_secrets_file: String, path to a file containing a client ID and
+        secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      credentials_file: String, the path to a file to use to save the
+        credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Raises:
+      InvalidClientSecretsFileError: If the client secrets file cannot be
+        opened.
+      InvalidClientSecretsFileFormatError: If the client secrets file does not
+        contain the required data or the data is malformed.
+
+    Returns:
+      Credentials
+    """
+        cs_data = fileutils.read_file(client_secrets_file,
+                                      continue_on_error=True,
+                                      display_errors=False)
+        if not cs_data:
+            raise InvalidClientSecretsFileError(
+                f'File {client_secrets_file} could not be opened')
+        try:
+            cs_json = json.loads(cs_data)
+            client_id = cs_json['installed']['client_id']
+            # Chop off .apps.googleusercontent.com suffix as it's not needed
+            # and we need to keep things short for the Auth URL.
+            client_id = re.sub(r'\.apps\.googleusercontent\.com$', '',
+                               client_id)
+            client_secret = cs_json['installed']['client_secret']
+        except (ValueError, IndexError, KeyError):
+            raise InvalidClientSecretsFileFormatError(
+                f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
+            )
+
+        return cls.from_client_secrets(client_id,
+                                       client_secret,
+                                       scopes,
+                                       access_type=access_type,
+                                       login_hint=login_hint,
+                                       filename=credentials_file,
+                                       use_console_flow=use_console_flow)
+
+    def _fetch_id_token_data(self):
+        """Fetches verification details from Google for the OAuth2.0 token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Raises:
+      CredentialsError: If no id_token is present.
+    """
+        if not self.id_token:
+            raise CredentialsError(
+                'Failed to fetch token data. No id_token present.')
+
+        request = transport.create_request()
+        if self.expired:
+            # The id_token needs to be unexpired, in order to request data about it.
+            self.refresh(request)
+
+        self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
+            self.id_token, request)
+
+    def get_token_value(self, field):
+        """Retrieves data from the OAuth ID token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Args:
+      field: The name of the key/field to fetch
+
+    Returns:
+      The value associated with the given key or 'Unknown' if the key data can
+      not be found in the access token data.
+    """
+        if not self._id_token_data:
+            self._fetch_id_token_data()
+        # Maintain legacy GAM behavior here to return "Unknown" if the field is
+        # otherwise unpopulated.
+        return self._id_token_data.get(field, 'Unknown')
+
+    def to_json(self, strip=None):
+        """Creates a JSON representation of a Credentials.
+
+    Args:
+        strip: Sequence[str], Optional list of members to exclude from the
+          generated JSON.
+
+    Returns:
+        str: A JSON representation of this instance, suitable to pass to
+             from_json().
+    """
+        expiry = self.expiry.strftime(
+            Credentials.DATETIME_FORMAT) if self.expiry else None
+        prep = {
+            'token': self.token,
+            'refresh_token': self.refresh_token,
+            'token_uri': self.token_uri,
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'id_token': self.id_token,
+            # Google auth doesn't currently give us scopes back on refresh.
+            # 'scopes': sorted(self.scopes),
+            'token_expiry': expiry,
+            'decoded_id_token': self._id_token_data,
+        }
+
+        # Remove empty entries
+        prep = {k: v for k, v in prep.items() if v is not None}
+
+        # Remove entries that explicitly need to be removed
+        if strip is not None:
+            prep = {k: v for k, v in prep.items() if k not in strip}
+
+        return json.dumps(prep, indent=2, sort_keys=True)
+
+    def refresh(self, request=None):
+        """Refreshes the credential's access token.
+
+    Args:
+      request: google.auth.transport.Request, The object used to make HTTP
+        requests. If not provided, a default request will be used.
+
+    Raises:
+      google.auth.exceptions.RefreshError: If the credentials could not be
+      refreshed.
+    """
+        with self._lock:
+            if request is None:
+                request = transport.create_request()
+            self._locked_refresh(request)
+            # Save the new tokens back to disk, if these credentials are disk-backed.
+            if self._filename:
+                self._locked_write()
+
+    def _locked_refresh(self, request):
+        """Refreshes the credential's access token while the file lock is held."""
+        assert self._lock.is_locked
+        super(Credentials, self).refresh(request)
+
+    def write(self):
+        """Writes credentials to disk."""
+        with self._lock:
+            self._locked_write()
+
+    def _locked_write(self):
+        """Writes credentials to disk while the file lock is held."""
+        assert self._lock.is_locked
+        if not self.filename:
+            # If no filename was provided to the constructor, these credentials cannot
+            # be saved to disk.
+            raise CredentialsError(
+                'The credentials have no associated filename and cannot be saved '
+                'to disk.')
+        fileutils.write_file(self._filename, self.to_json())
+
+    def delete(self):
+        """Deletes all files on disk related to these credentials."""
+        with self._lock:
+            # Only attempt to remove the file if the lock we're using is a FileLock.
+            if isinstance(self._lock, FileLock):
+                os.remove(self._filename)
+                if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
+                    os.remove(self._lock.lock_file)
+
+    _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+
+    def revoke(self, http=None):
+        """Revokes this credential's access token with the server.
+
+    Args:
+      http: httplib2.Http compatible object for use as a transport. If no http
+        is provided, a default will be used.
+    """
+        with self._lock:
+            if http is None:
+                http = transport.create_http()
+            params = urlencode({'token': self.refresh_token})
+            revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
+            http.request(revoke_uri, 'GET')
+
+
+class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
+    """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
+
+    URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+    def authorization_url(self, http=None, **kwargs):
+        """Gets a shortened authorization URL."""
+        long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
+        short_url = utils.shorten_url(long_url)
+        return short_url, state
+
+
+class _FileLikeThreadLock(object):
+    """A threading.lock which has the same interface as filelock.Filelock."""
+
+    def __init__(self):
+        """A shell object that holds a threading.Lock.
+
+    Since we cannot inherit from built-in classes such as threading.Lock, we
+    just use a shell object and maintain a lock inside of it.
+    """
+        self._lock = threading.Lock()
+
+    def __enter__(self, *args, **kwargs):
+        return self._lock.__enter__(*args, **kwargs)
+
+    def __exit__(self, *args, **kwargs):
+        return self._lock.__exit__(*args, **kwargs)
+
+    def acquire(self, **kwargs):
+        return self._lock.acquire(**kwargs)
+
+    def release(self):
+        return self._lock.release()
+
+    @property
+    def is_locked(self):
+        return self._lock.locked()
+
+    @property
+    def lock_file(self):
+        return None

src/gam/auth/oauth_test.py

@@ -0,0 +1,697 @@
+"""Tests for oauth."""
+
+import datetime
+import json
+import os
+import platform
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import google.oauth2.credentials
+
+from gam.auth import oauth
+
+
+class CredentialsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_token = 'fake_token'
+        self.fake_refresh_token = 'fake_refresh_token'
+        self.fake_id_token = 'fake_id_token'
+        self.fake_token_uri = 'https://fake.token.uri'
+        self.fake_client_id = 'fake_client_id'
+        self.fake_client_secret = 'fake_client_secret'
+        self.fake_scopes = [
+            'fake_api.readonly',
+            'fake_other_api.write',
+        ]
+        self.fake_quota_project_id = 'fake_quota_project_id'
+        self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
+        self.fake_filename = 'fake_filename'
+        self.fake_token_data = {
+            'field': 'value',
+            'another-field': 'another-value',
+        }
+        self.info_with_only_required_fields = {
+            'refresh_token': self.fake_refresh_token,
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+        }
+        super(CredentialsTest, self).setUp()
+
+    def tearDown(self):
+        # Remove any credential files that may have been created.
+        if os.path.exists(self.fake_filename):
+            os.remove(self.fake_filename)
+        if os.path.exists('%s.lock' % self.fake_filename):
+            os.remove('%s.lock' % self.fake_filename)
+        super(CredentialsTest, self).tearDown()
+
+    def test_from_authorized_user_info_only_required_info(self):
+        creds = oauth.Credentials.from_authorized_user_info(
+            self.info_with_only_required_fields)
+        self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+        self.assertEqual(self.fake_client_id, creds.client_id)
+        self.assertEqual(self.fake_client_secret, creds.client_secret)
+        self.assertIsNone(creds.id_token)
+        self.assertIsNone(creds.expiry)
+        self.assertIsNone(creds.filename)
+
+    def test_from_authorized_user_info_all_info_provided(self):
+        info = {
+            'token':
+                self.fake_token,
+            'refresh_token':
+                self.fake_refresh_token,
+            'id_token':
+                self.fake_id_token,
+            'token_uri':
+                self.fake_token_uri,
+            'client_id':
+                self.fake_client_id,
+            'client_secret':
+                self.fake_client_secret,
+            'token_expiry':
+                self.fake_token_expiry.strftime(
+                    oauth.Credentials.DATETIME_FORMAT),
+            'id_token_data':
+                self.fake_token_data,
+        }
+        creds = oauth.Credentials.from_authorized_user_info(info)
+        self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+        self.assertEqual(self.fake_client_id, creds.client_id)
+        self.assertEqual(self.fake_client_secret, creds.client_secret)
+        self.assertEqual(self.fake_id_token, creds.id_token)
+        self.assertEqual(self.fake_token_uri, creds.token_uri)
+        self.assertEqual(self.fake_token_expiry, creds.expiry)
+        self.assertIsNone(creds.filename)
+
+    def test_from_authorized_user_info_missing_required_info(self):
+        info_with_missing_fields = {'token': self.fake_token}
+        with self.assertRaises(ValueError):
+            oauth.Credentials.from_authorized_user_info(
+                info_with_missing_fields)
+
+    def test_from_authorized_user_info_no_expiry_in_info(self):
+        info_with_no_token_expiry = self.info_with_only_required_fields.copy()
+        self.assertIsNone(info_with_no_token_expiry.get('expiry'))
+        creds = oauth.Credentials.from_authorized_user_info(
+            info_with_no_token_expiry)
+        self.assertIsNone(creds.expiry)
+
+    def test_init_saves_filename(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+    def test_init_loads_decoded_id_token_data(self, mock_verify_token):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=self.fake_token_data)
+        self.assertEqual(self.fake_token_data.get('field'),
+                         creds.get_token_value('field'))
+        # Verify the fetching method was not called, since the token
+        # data was supposed to be loaded from the passed in info.
+        self.assertEqual(mock_verify_token.call_count, 0)
+
+    def test_credentials_uses_file_lock_when_filename_provided(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        self.assertIsInstance(creds._lock, oauth.FileLock)
+        self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
+
+    def test_credentials_uses_thread_lock_when_filename_not_provided(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=None)
+        self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
+        self.assertIsNone(creds.filename)
+
+    def test_from_oauth2credentials(self):
+        google_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        creds = oauth.Credentials.from_google_oauth2_credentials(
+            google_creds, filename=self.fake_filename)
+        self.assertEqual(google_creds.token, creds.token)
+        self.assertEqual(google_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(google_creds.client_id, creds.client_id)
+        self.assertEqual(google_creds.client_secret, creds.client_secret)
+        self.assertEqual(google_creds.id_token, creds.id_token)
+        self.assertEqual(google_creds.expiry, creds.expiry)
+        self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+        self.assertIn('could not be opened', str(e.exception))
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_no_serialized_data_in_file_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({})
+        with self.assertRaises(oauth.EmptyCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_missing_any_token_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({
+            # This data is missing a token key/value pair
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+        })
+        with self.assertRaises(oauth.InvalidCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_missing_required_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({
+            # This data is missing a client_secret key/value pair
+            'client_id': self.fake_client_id,
+            'refresh_token': self.fake_refresh_token,
+        })
+        with self.assertRaises(oauth.InvalidCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_console_flow(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                      self.fake_client_secret,
+                                                      self.fake_scopes,
+                                                      use_console_flow=True)
+        self.assertTrue(mock_flow.return_value.run_console.called)
+        self.assertFalse(mock_flow.return_value.run_local_server.called)
+        self.assertEqual(flow_creds.token, creds.token)
+        self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(flow_creds.client_id, creds.client_id)
+        self.assertEqual(flow_creds.client_secret, creds.client_secret)
+        self.assertEqual(flow_creds.id_token, creds.id_token)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_local_server_flow(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                      self.fake_client_secret,
+                                                      self.fake_scopes,
+                                                      use_console_flow=False)
+        self.assertFalse(mock_flow.return_value.run_console.called)
+        self.assertTrue(mock_flow.return_value.run_local_server.called)
+        self.assertEqual(flow_creds.token, creds.token)
+        self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(flow_creds.client_id, creds.client_id)
+        self.assertEqual(flow_creds.client_secret, creds.client_secret)
+        self.assertEqual(flow_creds.id_token, creds.id_token)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_uses_login_hint(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                              self.fake_client_secret,
+                                              self.fake_scopes,
+                                              login_hint='someone@domain.com')
+
+        run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
+        self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
+
+    def test_from_client_secrets_uses_shortened_url_flow(self):
+        with patch.object(oauth._ShortURLFlow,
+                          'from_client_config') as mock_flow:
+            flow_creds = google.oauth2.credentials.Credentials(
+                token=self.fake_token,
+                refresh_token=self.fake_refresh_token,
+                client_id=self.fake_client_id,
+                client_secret=self.fake_client_secret,
+                id_token=self.fake_id_token)
+            mock_flow.return_value.credentials = flow_creds
+            oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                  self.fake_client_secret,
+                                                  self.fake_scopes)
+        self.assertTrue(mock_flow.called)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(
+            self.fake_client_id,
+            self.fake_client_secret,
+            self.fake_scopes,
+            filename=self.fake_filename)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(
+        self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        with self.assertRaises(oauth.InvalidClientSecretsFileError):
+            oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                       self.fake_scopes)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_client_secrets_file_missing_required_json_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({})
+        with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
+            oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                       self.fake_scopes)
+        self.assertIn('Could not extract Client ID or Client Secret',
+                      str(e.exception))
+
+    @patch.object(oauth.Credentials, 'from_client_secrets')
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_client_secrets_file_strips_domain_from_client_id(
+        self, mock_read_file, mock_creds_from_client_secrets):
+        mock_read_file.return_value = json.dumps({
+            'installed': {
+                'client_id':
+                    self.fake_client_id + '.apps.googleusercontent.com',
+                'client_secret':
+                    self.fake_client_secret,
+            }
+        })
+
+        oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                   self.fake_scopes)
+        self.assertEqual(self.fake_client_id,
+                         mock_creds_from_client_secrets.call_args[0][0])
+
+    def test_get_token_value_known_token_field(self):
+        token_data = {'known-field': 'known-value'}
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token_data=token_data)
+        self.assertEqual('known-value', creds.get_token_value('known-field'))
+
+    def test_get_token_value_unknown_field_returns_unknown(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token_data=self.fake_token_data)
+        self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
+
+    @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+    def test_get_token_value_credentials_expired(self,
+                                                 mock_verify_oauth2_token):
+        mock_verify_oauth2_token.return_value = {
+            'fetched-field': 'fetched-value'
+        }
+        time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
+            minutes=5)
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  expiry=time_earlier_than_now,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=None)
+        self.assertTrue(creds.expired)
+        creds.refresh = MagicMock()
+
+        token_value = creds.get_token_value('fetched-field')
+
+        self.assertEqual('fetched-value', token_value)
+        self.assertTrue(creds.refresh.called)
+
+    def test_to_json_contains_all_required_fields(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  refresh_token=self.fake_refresh_token,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=self.fake_token_data,
+                                  token_uri=self.fake_token_uri,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  scopes=self.fake_scopes,
+                                  quota_project_id=self.fake_quota_project_id,
+                                  expiry=self.fake_token_expiry)
+        json_string = creds.to_json()
+        json_data = json.loads(json_string)
+        keys = json_data.keys()
+        self.assertIn('token', keys)
+        self.assertEqual(self.fake_token, json_data['token'])
+        self.assertIn('refresh_token', keys)
+        self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
+        self.assertIn('id_token', keys)
+        self.assertEqual(self.fake_id_token, json_data['id_token'])
+        self.assertIn('token_uri', keys)
+        self.assertEqual(self.fake_token_uri, json_data['token_uri'])
+        self.assertIn('client_id', keys)
+        self.assertEqual(self.fake_client_id, json_data['client_id'])
+        self.assertIn('client_secret', keys)
+        self.assertEqual(self.fake_client_secret, json_data['client_secret'])
+        self.assertNotIn('scopes', keys)  # Scopes are not currently saved
+        self.assertIn('token_expiry', keys)
+        self.assertEqual(
+            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+            json_data['token_expiry'])
+        self.assertIn('decoded_id_token', keys)
+        self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
+
+    def test_credentials_to_json_and_back(self):
+        original_creds = oauth.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            id_token=self.fake_id_token,
+            id_token_data=self.fake_token_data,
+            token_uri=self.fake_token_uri,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            scopes=self.fake_scopes,
+            quota_project_id=self.fake_quota_project_id,
+            expiry=self.fake_token_expiry)
+        pickled_creds = original_creds.to_json()
+        serialized_json = json.loads(pickled_creds)
+        unpickled_creds = oauth.Credentials.from_authorized_user_info(
+            serialized_json)
+        self.assertEqual(original_creds.token, unpickled_creds.token)
+        self.assertEqual(original_creds.refresh_token,
+                         unpickled_creds.refresh_token)
+        self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
+        self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
+        self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
+        self.assertEqual(original_creds.client_secret,
+                         unpickled_creds.client_secret)
+        self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    def test_refresh_calls_super_refresh(self, mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        request = MagicMock()
+
+        creds.refresh(request)
+        self.assertTrue(mock_super_refresh.called)
+        self.assertEqual(request, mock_super_refresh.call_args[0][0])
+
+    def test_refresh_locks_resource_during_refresh(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        lock = creds._lock
+
+        def check_lock_is_locked(*unused_args, **unused_kwargs):
+            self.assertTrue(lock.is_locked)
+
+        # We need to mock the superclass refresh so it doesn't actually try to
+        # refresh our fake token.
+        # At the same time, we'll make sure the lock is held during the refresh.
+        with patch.object(oauth.google.oauth2.credentials.Credentials,
+                          'refresh') as mock_refresh:
+            mock_refresh.side_effect = check_lock_is_locked
+            creds.refresh(request=MagicMock())
+
+        # Make sure our side effect was actually performed.
+        self.assertTrue(mock_refresh.called)
+        # The lock should be released after refresh
+        self.assertFalse(lock.is_locked)
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    @patch.object(oauth.fileutils, 'write_file')
+    def test_refresh_writes_new_credentials_to_disk_after_refresh(
+        self, mock_write_file, mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+
+        def update_access_token(unused_request):
+            creds.token = 'refreshed_access_token'
+
+        mock_super_refresh.side_effect = update_access_token
+
+        self.assertIsNone(creds.token)
+        creds.refresh(request=MagicMock())
+        self.assertEqual('refreshed_access_token', creds.token,
+                         'Access token was not refreshed')
+        text_written_to_file = mock_write_file.call_args[0][1]
+        self.assertIsNotNone(text_written_to_file,
+                             'Nothing was written to file')
+        saved_json = json.loads(text_written_to_file)
+        self.assertEqual('refreshed_access_token', saved_json['token'],
+                         'Refreshed access token was not saved to disk')
+
+    def test_write_writes_credentials_to_disk(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+
+        self.assertFalse(os.path.exists(self.fake_filename))
+        creds.write()
+        self.assertTrue(os.path.exists(self.fake_filename))
+
+    def test_write_raises_error_when_no_credentials_file_is_set(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+
+        self.assertIsNone(creds.filename)
+        with self.assertRaises(oauth.CredentialsError):
+            creds.write()
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    @patch.object(oauth.fileutils, 'write_file')
+    def test_write_locks_resource_during_write(self, mock_write_file,
+                                               unused_mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        lock = creds._lock
+
+        def check_lock_is_locked(*unused_args, **unused_kwargs):
+            self.assertTrue(creds._lock.is_locked)
+
+        mock_write_file.side_effect = check_lock_is_locked
+
+        self.assertFalse(lock.is_locked)
+        creds.refresh(request=MagicMock())
+        self.assertFalse(lock.is_locked)
+        self.assertTrue(mock_write_file.called)
+
+    def test_delete_removes_credentials_file(self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        creds.write()
+        self.assertTrue(os.path.exists(self.fake_filename))
+        creds.delete()
+        self.assertFalse(os.path.exists(self.fake_filename))
+
+    @unittest.skipIf(
+        platform.system() == 'Windows',
+        reason=('On Windows, Filelock deletes the lock file each time the lock '
+                'is released. Delete does not remove it.'))
+    def test_delete_removes_lock_file(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        lock_file = '%s.lock' % creds.filename
+        creds.write()
+        self.assertTrue(os.path.exists(lock_file))
+        creds.delete()
+        self.assertFalse(os.path.exists(lock_file))
+
+    def test_delete_is_noop_when_not_using_filelock(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        self.assertIsNone(creds.filename)
+        creds.delete()  # This should not raise an exception.
+
+    def test_revoke_requests_credential_revoke(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        mock_http = MagicMock()
+
+        creds.revoke(http=mock_http)
+
+        uri = mock_http.request.call_args[0][0]
+        self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
+        params = uri[uri.index('?'):]
+        self.assertIn('token=%s' % creds.refresh_token, params)
+        self.assertEqual('GET', mock_http.request.call_args[0][1])
+
+
+class ShortUrlFlowTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_client_id = 'fake_client_id'
+        self.fake_client_secret = 'fake_client_secret'
+        self.fake_scopes = [
+            'fake_api.readonly',
+            'fake_other_api.write',
+        ]
+        self.fake_client_config = {
+            'installed': {
+                'client_id': self.fake_client_id,
+                'client_secret': self.fake_client_secret,
+                'redirect_uris': [
+                    'http://localhost', 'urn:ietf:wg:oauth:2.0:oob'
+                ],
+                'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+                'token_uri': 'https://oauth2.googleapis.com/token',
+            }
+        }
+        self.long_url = 'http://example.com/some/long/url'
+        self.short_url = 'http://ex.co/short'
+        super(ShortUrlFlowTest, self).setUp()
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = json.dumps({'short_url': self.short_url})
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.short_url, url)
+        self.assertEqual('fake_state', state)
+
+        # Verify request() was called with the expected arguments.
+        self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
+                         mock_http.request.call_args[0][0])
+        self.assertEqual('POST', mock_http.request.call_args[0][1])
+        self.assertIn(self.long_url, mock_http.request.call_args[0][2])
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_request_error(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_http.request.side_effect = Exception()
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 404  # Use a status that is not 200
+        content = json.dumps({'short_url': self.short_url})
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = None
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = json.dumps(
+            {})  # This json content contains no "short-url" key
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/auth/yubikey.py

@@ -0,0 +1,74 @@
+from base64 import b64encode
+import sys
+from threading import Timer
+
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import padding
+from ykman.device import connect_to_device
+from yubikit.piv import KEY_TYPE, SLOT, InvalidPinError, PivSession
+from yubikit.core.smartcard import ApduError
+from gam import controlflow
+
+class YubiKey():
+
+    def __init__(self, service_account_info):
+        key_type = service_account_info.get('yubikey_key_type', 'RSA2048')
+        try:
+            self.key_type = getattr(KEY_TYPE, key_type.upper())
+        except AttributeError:
+            controlflow.system_error_exit(6, f'{key_type} is not a valid value for yubikey_key_type')
+        slot = service_account_info.get('yubikey_slot', 'AUTHENTICATION')
+        try:
+            self.slot = getattr(SLOT, slot.upper())
+        except AttributeError:
+            controlflow.system_error_exit(6, f'{slot} is not a valid value for yubikey_slot')
+        self.serial_number = service_account_info.get('yubikey_serial_number')
+        self.pin = service_account_info.get('yubikey_pin')
+        self.key_id = service_account_info.get('private_key_id')
+
+    def get_certificate(self):
+        try:
+            conn, _, _ = connect_to_device(self.serial_number)
+            session = PivSession(conn)
+            if self.pin:
+                try:
+                    session.verify_pin(self.pin)
+                except InvalidPinError as err:
+                    controlflow.system_error_exit(7, f'YubiKey - {err}')
+            try:
+                cert = session.get_certificate(self.slot)
+                cert_pem = cert.public_bytes(
+                     serialization.Encoding.PEM).decode()
+                publicKeyData = b64encode(cert_pem.encode())
+                if isinstance(publicKeyData, bytes):
+                    publicKeyData = publicKeyData.decode()
+                return publicKeyData
+            except ApduError as err:
+                controlflow.system_error_exit(8, f'YubiKey - {err}')
+        except ValueError as err:
+            controlflow.system_error_exit(9, f'YubiKey - {err}')
+
+    def sign(self, message):
+        if 'mplock' in globals():
+            mplock.acquire()
+        try:
+            conn, _, _ = connect_to_device(self.serial_number)
+            session = PivSession(conn)
+            if self.pin:
+                try:
+                    session.verify_pin(self.pin)
+                except InvalidPinError as err:
+                    controlflow.system_error_exit(7, f'YubiKey - {err}')
+            try:
+                signed = session.sign(slot=self.slot,
+                                      key_type=self.key_type,
+                                      message=message,
+                                      hash_algorithm=hashes.SHA256(),
+                                      padding=padding.PKCS1v15())
+            except ApduError as err:
+                controlflow.system_error_exit(8, f'YubiKey = {err}')
+        except ValueError as err:
+            controlflow.system_error_exit(9, f'YubiKey - {err}')
+        if 'mplock' in globals():
+            mplock.release()
+        return signed

src/gam/controlflow.py

@@ -0,0 +1,96 @@
+"""Methods related to the central control flow of an application."""
+import random
+import sys
+import time
+
+from gam import display
+from gam.var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from gam.var import MESSAGE_INVALID_JSON
+
+
+def system_error_exit(return_code, message):
+    """Raises a system exit with the given return code and message.
+
+  Args:
+    return_code: Int, the return code to yield when the system exits.
+    message: An error message to print before the system exits.
+  """
+    if message:
+        display.print_error(message)
+    sys.exit(return_code)
+
+
+def invalid_argument_exit(argument, command):
+    """Indicate that the argument is not valid for the command.
+
+  Args:
+    argument: the invalid argument
+    command: the base GAM command
+  """
+    system_error_exit(2, f'{argument} is not a valid argument for "{command}"')
+
+
+def missing_argument_exit(argument, command):
+    """Indicate that the argument is missing for the command.
+
+  Args:
+    argument: the missingagrument
+    command: the base GAM command
+  """
+    system_error_exit(2, f'missing argument {argument} for "{command}"')
+
+
+def expected_argument_exit(name, expected, argument):
+    """Indicate that the argument does not have an expected value for the command.
+
+  Args:
+    name: the field name
+    expected: the expected values
+    argument: the invalid argument
+  """
+    system_error_exit(2, f'{name} must be one of {expected}; got {argument}')
+
+
+def csv_field_error_exit(field_name, field_names):
+    """Raises a system exit when a CSV field is malformed.
+
+  Args:
+    field_name: The CSV field name for which a header does not exist in the
+      existing CSV headers.
+    field_names: The known list of CSV headers.
+  """
+    system_error_exit(
+        2,
+        MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                       ','.join(field_names)))
+
+
+def invalid_json_exit(file_name):
+    """Raises a system exit when invalid JSON content is encountered."""
+    system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+
+
+def wait_on_failure(current_attempt_num,
+                    total_num_retries,
+                    error_message,
+                    error_print_threshold=3):
+    """Executes an exponential backoff-style system sleep.
+
+  Args:
+    current_attempt_num: Int, the current number of retries.
+    total_num_retries: Int, the total number of times the current action will be
+      retried.
+    error_message: String, a message to be displayed that will give more context
+      around why the action is being retried.
+    error_print_threshold: Int, the number of attempts which will have their
+      error messages suppressed. Any current_attempt_num greater than
+      error_print_threshold will print the prescribed error.
+  """
+    wait_on_fail = min(2**current_attempt_num,
+                       60) + float(random.randint(1, 1000)) / 1000
+    if current_attempt_num > error_print_threshold:
+        sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+                          f'{int(wait_on_fail)} seconds, Retry: '
+                          f'{current_attempt_num}/{total_num_retries}\n'))
+        sys.stderr.flush()
+    time.sleep(wait_on_fail)

src/gam/controlflow_test.py

@@ -0,0 +1,108 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+from gam import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+    def test_system_error_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.system_error_exit(1, 'exit message')
+
+    def test_system_error_exit_raises_systemexit_with_return_code(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.system_error_exit(100, 'exit message')
+        self.assertEqual(context_manager.exception.code, 100)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_system_error_exit_prints_error_before_exiting(
+        self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.system_error_exit(100, 'exit message')
+        self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+    def test_csv_field_error_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+
+    def test_csv_field_error_exit_exits_code_2(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+        self.assertEqual(context_manager.exception.code, 2)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+        printed_message = mock_print_err.call_args[0][0]
+        self.assertIn('aField', printed_message)
+        self.assertIn('unusedField1', printed_message)
+        self.assertIn('unusedField2', printed_message)
+
+    def test_invalid_json_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.invalid_json_exit('filename')
+
+    def test_invalid_json_exit_exit_exits_code_17(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.invalid_json_exit('filename')
+        self.assertEqual(context_manager.exception.code, 17)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.invalid_json_exit('filename')
+        printed_message = mock_print_err.call_args[0][0]
+        self.assertIn('filename', printed_message)
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+        controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+        controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+        controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+        sleep_calls = mock_sleep.call_args_list
+        self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+        self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+        self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+        total_attempts = 20
+        for attempt in range(1, total_attempts + 1):
+            controlflow.wait_on_failure(
+                attempt,
+                total_attempts,
+                'Attempt #%s' % attempt,
+                # Suppress messages while we make a lot of attempts.
+                error_print_threshold=total_attempts + 1)
+            # Wait time may be between 60 and 61 secs, due to rand addition.
+            self.assertLessEqual(mock_sleep.call_args[0][0], 61)
+
+    # Prevent the system from actually sleeping and thus slowing down the test.
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
+        message = 'An error message to display'
+        with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+            controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+        self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_only_prints_after_threshold(
+        self, unused_mock_sleep):
+        total_attempts = 5
+        threshold = 3
+        with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+            for attempt in range(1, total_attempts + 1):
+                controlflow.wait_on_failure(attempt,
+                                            total_attempts,
+                                            'Attempt #%s' % attempt,
+                                            error_print_threshold=threshold)
+        self.assertEqual(total_attempts - threshold,
+                         mock_stderr_write.call_count)

src/gam/display.py

@@ -0,0 +1,324 @@
+"""Methods related to display of information to the user."""
+
+import csv
+import datetime
+import io
+import sys
+import webbrowser
+
+import dateutil
+import googleapiclient.http
+
+#TODO: get rid of these hacks
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+
+
+def current_count(i, count):
+    return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
+
+
+def current_count_nl(i, count):
+    return f' ({i}/{count})\n' if (
+        count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
+
+
+def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
+    fields = fieldsChoiceMap[fieldName.lower()]
+    if isinstance(fields, list):
+        fieldsList.extend(fields)
+    else:
+        fieldsList.append(fields)
+
+
+# Write a CSV file
+def add_titles_to_csv_file(addTitles, titles):
+    for title in addTitles:
+        if title not in titles:
+            titles.append(title)
+
+
+def add_row_titles_to_csv_file(row, csvRows, titles):
+    csvRows.append(row)
+    for title in row:
+        if title not in titles:
+            titles.append(title)
+
+
+# fieldName is command line argument
+# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
+#ARGUMENT_TO_PROPERTY_MAP = {
+#  u'admincreated': [u'adminCreated'],
+#  u'aliases': [u'aliases', u'nonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles,
+                          titles):
+    for ftList in fieldNameMap[fieldName]:
+        if ftList not in fieldsTitles:
+            fieldsList.append(ftList)
+            fieldsTitles[ftList] = ftList
+            add_titles_to_csv_file([ftList], titles)
+
+
+# fieldName is command line argument
+# fieldNameTitleMap maps fieldName to API field name and CSV file header
+#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
+#  u'admincreated': [u'adminCreated', u'Admin_Created'],
+#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList,
+                                fieldsTitles, titles):
+    ftList = fieldNameTitleMap[fieldName]
+    for i in range(0, len(ftList), 2):
+        if ftList[i] not in fieldsTitles:
+            fieldsList.append(ftList[i])
+            fieldsTitles[ftList[i]] = ftList[i + 1]
+            add_titles_to_csv_file([ftList[i + 1]], titles)
+
+
+def sort_csv_titles(firstTitle, titles):
+    restoreTitles = []
+    for title in firstTitle:
+        if title in titles:
+            titles.remove(title)
+            restoreTitles.append(title)
+    titles.sort()
+    for title in restoreTitles[::-1]:
+        titles.insert(0, title)
+
+
+def QuotedArgumentList(items):
+    return ' '.join([
+        item if item and (item.find(' ') == -1) and
+        (item.find(',') == -1) else '"' + item + '"' for item in items
+    ])
+
+
+def write_csv_file(csvRows, titles, list_type, todrive):
+
+    def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
+        if not rowDate or not isinstance(rowDate, str):
+            return False
+        try:
+            rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
+            if dateMode:
+                rowDate = datetime.datetime(rowTime.year, rowTime.month,
+                                            rowTime.day).isoformat() + 'Z'
+        except ValueError:
+            rowDate = NEVER_TIME
+        if op == '<':
+            return rowDate < filterDate
+        if op == '<=':
+            return rowDate <= filterDate
+        if op == '>':
+            return rowDate > filterDate
+        if op == '>=':
+            return rowDate >= filterDate
+        if op == '!=':
+            return rowDate != filterDate
+        return rowDate == filterDate
+
+    def rowCountFilterMatch(rowCount, op, filterCount):
+        if isinstance(rowCount, str):
+            if not rowCount.isdigit():
+                return False
+            rowCount = int(rowCount)
+        elif not isinstance(rowCount, int):
+            return False
+        if op == '<':
+            return rowCount < filterCount
+        if op == '<=':
+            return rowCount <= filterCount
+        if op == '>':
+            return rowCount > filterCount
+        if op == '>=':
+            return rowCount >= filterCount
+        if op == '!=':
+            return rowCount != filterCount
+        return rowCount == filterCount
+
+    def rowBooleanFilterMatch(rowBoolean, filterBoolean):
+        if not isinstance(rowBoolean, bool):
+            return False
+        return rowBoolean == filterBoolean
+
+    def headerFilterMatch(filters, title):
+        for filterStr in filters:
+            if filterStr.match(title):
+                return True
+        return False
+
+    def rowFilterMatch(filters, columns, row):
+        for c, filterVal in iter(filters.items()):
+            for column in columns[c]:
+                if filterVal[1] == 'regex':
+                    if filterVal[2].search(str(row.get(column, ''))):
+                        return True
+                elif filterVal[1] == 'notregex':
+                    if not filterVal[2].search(str(row.get(column, ''))):
+                        return True
+                elif filterVal[1] in ['date', 'time']:
+                    if rowDateTimeFilterMatch(
+                        filterVal[1] == 'date', row.get(column, ''),
+                        filterVal[2], filterVal[3]):
+                        return True
+                elif filterVal[1] == 'count':
+                    if rowCountFilterMatch(
+                        row.get(column, 0), filterVal[2], filterVal[3]):
+                        return True
+                else:  #boolean
+                    if rowBooleanFilterMatch(
+                        row.get(column, False), filterVal[2]):
+                        return True
+        return False
+
+    if GC_Values[GC_CSV_ROW_FILTER] or GC_Values[GC_CSV_ROW_DROP_FILTER]:
+        if GC_Values[GC_CSV_ROW_FILTER]:
+            keepColumns = {}
+            for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
+                columns = [t for t in titles if filterVal[0].match(t)]
+                if columns:
+                    keepColumns[column] = columns
+                else:
+                    keepColumns[column] = [None]
+                    sys.stderr.write(
+                        f'WARNING: Row filter column pattern "{column}" does not match any output columns\n'
+                        )
+        else:
+            keepColumns = None
+        if GC_Values[GC_CSV_ROW_DROP_FILTER]:
+            dropColumns = {}
+            for column, filterVal in iter(GC_Values[GC_CSV_ROW_DROP_FILTER].items()):
+                columns = [t for t in titles if filterVal[0].match(t)]
+                if columns:
+                    dropColumns[column] = columns
+                else:
+                    dropColumns[column] = [None]
+                    sys.stderr.write(
+                        f'WARNING: Row drop filter column pattern "{column}" does not match any output columns\n'
+                        )
+        else:
+            dropColumns = None
+        rows = []
+        for row in csvRows:
+            if (((keepColumns is None) or
+                 rowFilterMatch(GC_Values[GC_CSV_ROW_FILTER], keepColumns, row)) and
+                ((dropColumns is None) or
+                 not rowFilterMatch(GC_Values[GC_CSV_ROW_DROP_FILTER], dropColumns, row))):
+                rows.append(row)
+        csvRows = rows
+
+    if GC_Values[GC_CSV_HEADER_FILTER] or GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+        if GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+            titles = [
+                t for t in titles if
+                not headerFilterMatch(GC_Values[GC_CSV_HEADER_DROP_FILTER], t)
+            ]
+        if GC_Values[GC_CSV_HEADER_FILTER]:
+            titles = [
+                t for t in titles
+                if headerFilterMatch(GC_Values[GC_CSV_HEADER_FILTER], t)
+            ]
+        if not titles:
+            controlflow.system_error_exit(
+                3,
+                'No columns selected with GAM_CSV_HEADER_FILTER and GAM_CSV_HEADER_DROP_FILTER\n'
+            )
+            return
+    csv.register_dialect('nixstdout', lineterminator='\n')
+    if todrive:
+        write_to = io.StringIO()
+    else:
+        write_to = sys.stdout
+    writer = csv.DictWriter(write_to,
+                            fieldnames=titles,
+                            dialect='nixstdout',
+                            extrasaction='ignore',
+                            quoting=csv.QUOTE_MINIMAL)
+    try:
+        writer.writerow(dict((item, item) for item in writer.fieldnames))
+        writer.writerows(csvRows)
+    except IOError as e:
+        controlflow.system_error_exit(6, e)
+    if todrive:
+        admin_email = gam._get_admin_email()
+        _, drive = gam.buildDrive3GAPIObject(admin_email)
+        if not drive:
+            print(f'''\nGAM is not authorized to create Drive files. Please run:
+gam user {admin_email} check serviceaccount
+and follow recommend steps to authorize GAM for Drive access.''')
+            sys.exit(5)
+        result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
+        columns = len(titles)
+        rows = len(csvRows)
+        cell_count = rows * columns
+        data_size = len(write_to.getvalue())
+        max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
+        if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
+            print(
+                f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}'
+            )
+            mimeType = 'text/csv'
+        else:
+            mimeType = MIMETYPE_GA_SPREADSHEET
+        body = {
+            'description': QuotedArgumentList(sys.argv),
+            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
+            'mimeType': mimeType
+        }
+        result = gapi.call(drive.files(),
+                           'create',
+                           fields='webViewLink',
+                           body=body,
+                           media_body=googleapiclient.http.MediaInMemoryUpload(
+                               write_to.getvalue().encode(),
+                               mimetype='text/csv'))
+        file_url = result['webViewLink']
+        if GC_Values[GC_NO_BROWSER]:
+            msg_txt = f'Drive file uploaded to:\n {file_url}'
+            msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
+            gam.send_email(msg_subj, msg_txt)
+            print(msg_txt)
+        else:
+            webbrowser.open(file_url)
+
+
+def print_error(message):
+    """Prints a one-line error message to stderr in a standard format."""
+    sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
+
+
+def print_warning(message):
+    """Prints a one-line warning message to stderr in a standard format."""
+    sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
+
+
+def print_json(object_value, spacing=''):
+    """Prints Dict or Array to screen in clean human-readable format.."""
+    if isinstance(object_value, list):
+        if len(object_value) == 1 and isinstance(object_value[0],
+                                                 (str, int, bool)):
+            sys.stdout.write(f'{object_value[0]}\n')
+            return
+        if spacing:
+            sys.stdout.write('\n')
+        for i, a_value in enumerate(object_value):
+            if isinstance(a_value, (str, int, bool)):
+                sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
+            else:
+                sys.stdout.write(f' {spacing}{i+1}) ')
+                print_json(a_value, f' {spacing}')
+    elif isinstance(object_value, dict):
+        for key in ['kind', 'etag', 'etags', '@type']:
+            object_value.pop(key, None)
+        for another_object, another_value in object_value.items():
+            sys.stdout.write(f' {spacing}{another_object}: ')
+            print_json(another_value, f' {spacing}')
+    else:
+        sys.stdout.write(f'{object_value}\n')

src/gam/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+from gam import display
+from gam.var import ERROR_PREFIX
+from gam.var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+    def test_print_error_prints_to_stderr(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertIn(message, printed_message)
+
+    def test_print_error_prints_error_prefix(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertLess(
+            printed_message.find(ERROR_PREFIX), printed_message.find(message),
+            'The error prefix does not appear before the error message')
+
+    def test_print_error_ends_message_with_newline(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertRegex(printed_message, '\n$',
+                         'The error message does not end in a newline.')
+
+    def test_print_warning_prints_to_stderr(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertIn(message, printed_message)
+
+    def test_print_warning_prints_error_prefix(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertLess(
+            printed_message.find(WARNING_PREFIX), printed_message.find(message),
+            'The warning prefix does not appear before the error message')
+
+    def test_print_warning_ends_message_with_newline(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertRegex(printed_message, '\n$',
+                         'The warning message does not end in a newline.')

src/gam/fileutils.py

@@ -0,0 +1,183 @@
+"""Common file operations."""
+
+import io
+import os
+import sys
+
+from gam import controlflow
+from gam import display
+from gam.var import GM_Globals
+from gam.var import GM_SYS_ENCODING
+from gam.var import UTF8_SIG
+
+
+def _open_file(filename, mode, encoding=None, newline=None):
+    """Opens a file with no error handling."""
+    # Determine which encoding to use
+    if 'b' in mode:
+        encoding = None
+    elif not encoding:
+        encoding = GM_Globals[GM_SYS_ENCODING]
+    elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
+        encoding = UTF8_SIG
+
+    return open(os.path.expanduser(filename),
+                mode,
+                newline=newline,
+                encoding=encoding)
+
+
+def open_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              strip_utf_bom=False):
+    """Opens a file.
+
+  Args:
+    filename: String, the name of the file to open, or '-' to use stdin/stdout,
+      to read/write, depending on the mode param, respectively.
+    mode: String, the common file mode to open the file with. Default is read.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    strip_utf_bom: Boolean, True if the file being opened should seek past the
+      UTF Byte Order Mark before being returned.
+        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+
+  Returns:
+    The opened file.
+  """
+    try:
+        if filename == '-':
+            # Read from stdin, rather than a file
+            if 'r' in mode:
+                return io.StringIO(str(sys.stdin.read()))
+            return sys.stdout
+
+        # Open a file on disk
+        f = _open_file(filename, mode, newline=newline, encoding=encoding)
+        if strip_utf_bom:
+            utf_bom = u'\ufeff'
+            has_bom = False
+
+            if 'b' in mode:
+                has_bom = f.read(3).decode('UTF-8') == utf_bom
+            elif f.encoding and not f.encoding.lower().startswith('utf'):
+                # Convert UTF BOM into ISO-8859-1 via Bytes
+                utf8_bom_bytes = utf_bom.encode('UTF-8')
+                iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
+                    'iso-8859-1')
+                has_bom = f.read(3).encode('iso-8859-1',
+                                           'replace') == iso_8859_1_bom
+            else:
+                has_bom = f.read(1) == utf_bom
+
+            if not has_bom:
+                f.seek(0)
+
+        return f
+
+    except IOError as e:
+        controlflow.system_error_exit(6, e)
+
+
+def close_file(f, force_flush=False):
+    """Closes a file.
+
+  Args:
+    f: The file to close
+    force_flush: Flush file to disk emptying Python and OS caches. See:
+       https://stackoverflow.com/a/13762137/1503886
+
+  Returns:
+    Boolean, True if the file was successfully closed. False if an error
+        was encountered while closing.
+  """
+    if force_flush:
+        f.flush()
+        os.fsync(f.fileno())
+    try:
+        f.close()
+        return True
+    except IOError as e:
+        display.print_error(e)
+        return False
+
+
+def read_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              continue_on_error=False,
+              display_errors=True):
+    """Reads a file from disk.
+
+  Args:
+    filename: String, the path of the file to open from disk, or "-" to read
+      from stdin.
+    mode: String, the mode in which to open the file.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    The contents of the file, or stdin if filename == "-". Returns None if
+    an error is encountered and continue_on_errors is True.
+  """
+    try:
+        if filename == '-':
+            # Read from stdin, rather than a file.
+            return str(sys.stdin.read())
+
+        with _open_file(filename, mode, newline=newline,
+                        encoding=encoding) as f:
+            return f.read()
+
+    except IOError as e:
+        if continue_on_error:
+            if display_errors:
+                display.print_warning(e)
+            return None
+        controlflow.system_error_exit(6, e)
+    except (LookupError, UnicodeDecodeError, UnicodeError) as e:
+        controlflow.system_error_exit(2, str(e))
+
+
+def write_file(filename,
+               data,
+               mode='w',
+               continue_on_error=False,
+               display_errors=True):
+    """Writes data to a file.
+
+  Args:
+    filename: String, the path of the file to write to disk.
+    data: Serializable data to write to the file.
+    mode: String, the mode in which to open the file and write to it.
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    Boolean, True if the write operation succeeded, or False if not.
+  """
+    try:
+        with _open_file(filename, mode) as f:
+            f.write(data)
+        return True
+
+    except IOError as e:
+        if continue_on_error:
+            if display_errors:
+                display.print_error(e)
+            return False
+        else:
+            controlflow.system_error_exit(6, e)

src/gam/fileutils_test.py

@@ -0,0 +1,244 @@
+"""Tests for fileutils."""
+
+import io
+import os
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import fileutils
+
+
+class FileutilsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_path = '/some/path/to/file'
+        super(FileutilsTest, self).setUp()
+
+    @patch.object(fileutils.sys, 'stdin')
+    def test_open_file_stdin(self, mock_stdin):
+        mock_stdin.read.return_value = 'some stdin content'
+        f = fileutils.open_file('-', mode='r')
+        self.assertIsInstance(f, fileutils.io.StringIO)
+        self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
+
+    def test_open_file_stdout(self):
+        f = fileutils.open_file('-', mode='w')
+        self.assertEqual(fileutils.sys.stdout, f)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_opens_correct_path(self, mock_open):
+        f = fileutils.open_file(self.fake_path)
+        self.assertEqual(self.fake_path, mock_open.call_args[0][0])
+        self.assertEqual(mock_open.return_value, f)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_expands_user_file_path(self, mock_open):
+        file_path = '~/some/path/containing/tilde/shortcut/to/home'
+        fileutils.open_file(file_path)
+        opened_path = mock_open.call_args[0][0]
+        home_path = os.environ.get('HOME')
+        self.assertIsNotNone(home_path)
+        self.assertIn(home_path, opened_path)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_opens_correct_mode(self, mock_open):
+        fileutils.open_file(self.fake_path)
+        self.assertEqual('r', mock_open.call_args[0][1])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_encoding_for_binary(self, mock_open):
+        fileutils.open_file(self.fake_path, mode='b')
+        self.assertIsNone(mock_open.call_args[1]['encoding'])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_default_system_encoding(self, mock_open):
+        fileutils.open_file(self.fake_path)
+        self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
+                         mock_open.call_args[1]['encoding'])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_utf8_encoding_specified(self, mock_open):
+        fileutils.open_file(self.fake_path, encoding='UTF-8')
+        self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
+
+    def test_open_file_strips_utf_bom_in_utf(self):
+        bom_prefixed_data = u'\ufefffoobar'
+        fake_file = io.StringIO(bom_prefixed_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            self.assertEqual('foobar', f.read())
+
+    def test_open_file_strips_utf_bom_in_non_utf(self):
+        bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
+
+        # We need to trick the method under test into believing that a StringIO
+        # instance is a file with an encoding. Since StringIO does not usually have,
+        # an encoding, we'll mock it and add our own encoding, but send the other
+        # methods in use (read and seek) back to the real StringIO object.
+        real_stringio = io.StringIO(bom_prefixed_data)
+        mock_file = MagicMock(spec=io.StringIO)
+        mock_file.read.side_effect = real_stringio.read
+        mock_file.seek.side_effect = real_stringio.seek
+        mock_file.encoding = 'iso-8859-1'
+
+        mock_open = MagicMock(spec=open, return_value=mock_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            self.assertEqual('foobar', f.read())
+
+    def test_open_file_strips_utf_bom_in_binary(self):
+        bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
+        fake_file = io.BytesIO(bom_prefixed_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path,
+                                    mode='rb',
+                                    strip_utf_bom=True)
+            self.assertEqual(b'foobar', f.read())
+
+    def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
+        no_bom_data = 'This data has no BOM'
+        fake_file = io.StringIO(no_bom_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            # Since there was no opening BOM, we should be back at the beginning of
+            # the file.
+            self.assertEqual(fake_file.tell(), 0)
+            self.assertEqual(f.read(), no_bom_data)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_exits_on_io_error(self, mock_open):
+        mock_open.side_effect = IOError('Fake IOError')
+        with self.assertRaises(SystemExit) as context:
+            fileutils.open_file(self.fake_path)
+        self.assertEqual(context.exception.code, 6)
+
+    def test_close_file_closes_file_successfully(self):
+        mock_file = MagicMock()
+        self.assertTrue(fileutils.close_file(mock_file))
+        self.assertEqual(mock_file.close.call_count, 1)
+
+    def test_close_file_with_error(self):
+        mock_file = MagicMock()
+        mock_file.close.side_effect = IOError()
+        self.assertFalse(fileutils.close_file(mock_file))
+        self.assertEqual(mock_file.close.call_count, 1)
+
+    @patch.object(fileutils.sys, 'stdin')
+    def test_read_file_from_stdin(self, mock_stdin):
+        mock_stdin.read.return_value = 'some stdin content'
+        self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_default_params(self, mock_open_file):
+        fake_content = 'some fake content'
+        mock_open_file.return_value.__enter__().read.return_value = fake_content
+        self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
+        self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+        self.assertEqual(mock_open_file.call_args[0][1], 'r')
+        self.assertIsNone(mock_open_file.call_args[1]['newline'])
+
+    @patch.object(fileutils.display, 'print_warning')
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_continues_on_errors_without_displaying(
+        self, mock_open_file, mock_print_warning):
+        mock_open_file.side_effect = IOError()
+        contents = fileutils.read_file(self.fake_path,
+                                       continue_on_error=True,
+                                       display_errors=False)
+        self.assertIsNone(contents)
+        self.assertFalse(mock_print_warning.called)
+
+    @patch.object(fileutils.display, 'print_warning')
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_displays_errors(self, mock_open_file,
+                                       mock_print_warning):
+        mock_open_file.side_effect = IOError()
+        fileutils.read_file(self.fake_path,
+                            continue_on_error=True,
+                            display_errors=True)
+        self.assertTrue(mock_print_warning.called)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_6_when_continue_on_error_is_false(
+        self, mock_open_file):
+        mock_open_file.side_effect = IOError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path, continue_on_error=False)
+        self.assertEqual(context.exception.code, 6)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
+        mock_open_file.return_value.__enter__().read.side_effect = LookupError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
+        mock_open_file.return_value.__enter__().read.side_effect = UnicodeError(
+        )
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
+        fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0,
+                                               1, 'testing only')
+        mock_open_file.return_value.__enter__(
+        ).read.side_effect = fake_decode_error
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_writes_data_to_file(self, mock_open_file):
+        fake_data = 'some fake data'
+        fileutils.write_file(self.fake_path, fake_data)
+        self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+        self.assertEqual(mock_open_file.call_args[0][1], 'w')
+
+        opened_file = mock_open_file.return_value.__enter__()
+        self.assertTrue(opened_file.write.called)
+        self.assertEqual(opened_file.write.call_args[0][0], fake_data)
+
+    @patch.object(fileutils.display, 'print_error')
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_continues_on_errors_without_displaying(
+        self, mock_open_file, mock_print_error):
+        mock_open_file.side_effect = IOError()
+        status = fileutils.write_file(self.fake_path,
+                                      'foo data',
+                                      continue_on_error=True,
+                                      display_errors=False)
+        self.assertFalse(status)
+        self.assertFalse(mock_print_error.called)
+
+    @patch.object(fileutils.display, 'print_error')
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
+        mock_open_file.side_effect = IOError()
+        fileutils.write_file(self.fake_path,
+                             'foo data',
+                             continue_on_error=True,
+                             display_errors=True)
+        self.assertTrue(mock_print_error.called)
+
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_exits_code_6_when_continue_on_error_is_false(
+        self, mock_open_file):
+        mock_open_file.side_effect = IOError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.write_file(self.fake_path,
+                                 'foo data',
+                                 continue_on_error=False)
+        self.assertEqual(context.exception.code, 6)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/__init__.py

@@ -0,0 +1,370 @@
+"""Methods related to execution of GAPI requests."""
+
+import sys
+
+import googleapiclient.errors
+import google.auth.exceptions
+import httplib2
+
+from gam import controlflow
+from gam import display
+from gam.gapi import errors
+from gam import transport
+from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+                     GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
+                     MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
+                     MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
+
+
+def call(service,
+         function,
+         silent_errors=False,
+         soft_errors=False,
+         throw_reasons=None,
+         retry_reasons=None,
+         **kwargs):
+    """Executes a single request on a Google service function.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    silent_errors: Bool, If True, error messages are suppressed when
+      encountered.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A response object for the corresponding Google API call.
+  """
+    if throw_reasons is None:
+        throw_reasons = []
+    if retry_reasons is None:
+        retry_reasons = []
+
+    method = getattr(service, function)
+    retries = 10
+    parameters = dict(
+        list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+    for n in range(1, retries + 1):
+        try:
+            return method(**parameters).execute()
+        except googleapiclient.errors.HttpError as e:
+            http_status, reason, message = errors.get_gapi_error_detail(
+                e,
+                soft_errors=soft_errors,
+                silent_errors=silent_errors,
+                retry_on_http_error=n < 3)
+            if http_status == -1:
+                # The error detail indicated that we should retry this request
+                # We'll refresh credentials and make another pass
+                service._http.credentials.refresh(transport.create_http())
+                continue
+            if http_status == 0:
+                return None
+
+            is_known_error_reason = reason in [
+                r.value for r in errors.ErrorReason
+            ]
+            if is_known_error_reason and errors.ErrorReason(
+                    reason) in throw_reasons:
+                if errors.ErrorReason(
+                        reason) in errors.ERROR_REASON_TO_EXCEPTION:
+                    raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(
+                        reason)](message)
+                raise e
+            if (n != retries) and (is_known_error_reason and errors.ErrorReason(
+                    reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
+                controlflow.wait_on_failure(n, retries, reason)
+                continue
+            if soft_errors:
+                display.print_error(
+                    f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}'
+                )
+                return None
+            controlflow.system_error_exit(
+                int(http_status), f'{http_status}: {message} - {reason}')
+        except google.auth.exceptions.RefreshError as e:
+            handle_oauth_token_error(
+                e, soft_errors or
+                errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+            if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
+                raise errors.GapiServiceNotAvailableError(str(e))
+            display.print_error(
+                f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
+            return None
+        except ValueError as e:
+            if hasattr(service._http,
+                       'cache') and service._http.cache is not None:
+                service._http.cache = None
+                continue
+            controlflow.system_error_exit(4, str(e))
+        except (httplib2.ServerNotFoundError, RuntimeError) as e:
+            if n != retries:
+                service._http.connections = {}
+                controlflow.wait_on_failure(n, retries, str(e))
+                continue
+            controlflow.system_error_exit(4, str(e))
+        except TypeError as e:
+            controlflow.system_error_exit(4, str(e))
+
+
+def get_items(service,
+              function,
+              items='items',
+              throw_reasons=None,
+              retry_reasons=None,
+              **kwargs):
+    """Gets a single page of items from a Google service function that is paged.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the service
+      method's response object.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    The list of items in the first page of a response.
+  """
+    results = call(service,
+                   function,
+                   throw_reasons=throw_reasons,
+                   retry_reasons=retry_reasons,
+                   **kwargs)
+    if results:
+        return results.get(items, [])
+    return []
+
+
+def _get_max_page_size_for_api_call(service, function, **kwargs):
+    """Gets the maximum number of results supported for a single API call.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of the service method to check for max page size.
+    **kwargs: Additional params that will be passed to the request method.
+
+  Returns:
+    Int, A value from discovery if it exists, otherwise value from
+        MAX_RESULTS_API_EXCEPTIONS, otherwise None
+  """
+    method = getattr(service, function)
+    api_id = method(**kwargs).methodId
+    for resource in service._rootDesc.get('resources', {}).values():
+        for a_method in resource.get('methods', {}).values():
+            if a_method.get('id') == api_id:
+                if not a_method.get('parameters') or a_method['parameters'].get(
+                        'pageSize'
+                ) or not a_method['parameters'].get('maxResults'):
+                    # Make sure API call supports maxResults. For now we don't care to
+                    # set pageSize since all known pageSize API calls have
+                    # default pageSize == max pageSize.
+                    return None
+                known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
+                max_results = a_method['parameters']['maxResults'].get(
+                    'maximum', known_api_max)
+                return {'maxResults': max_results}
+
+    return None
+
+
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+
+def got_total_items_msg(items, eol):
+    """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+    return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+
+def got_total_items_first_last_msg(items):
+    """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+    return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}' + '\n'
+
+
+def process_page(page, items, all_items, total_items, page_message, message_attribute):
+    """Process one page of a Google service function response.
+
+  Append a list of items to the aggregate list of items
+
+  Args:
+    page: list of items
+    items: see get_all_pages
+    all_items: aggregate list of items
+    total_items: length of all_items
+    page_message: see get_all_pages
+    message_attribute: get_all_pages
+  Returns:
+    The page token and total number of items
+  """
+    if page:
+        page_token = page.get('nextPageToken')
+        page_items = page.get(items, [])
+        num_page_items = len(page_items)
+        total_items += num_page_items
+        if all_items is not None:
+            all_items.extend(page_items)
+    else:
+        page_token = None
+        num_page_items = 0
+
+    # Show a paging message to the user that indicates paging progress
+    if page_message:
+        show_message = page_message.replace(TOTAL_ITEMS_MARKER,
+                                            str(total_items))
+        if message_attribute:
+            first_item = page_items[0] if num_page_items > 0 else {}
+            last_item = page_items[-1] if num_page_items > 1 else first_item
+            if isinstance(message_attribute, str):
+                first_item = str(first_item.get(message_attribute, ''))
+                last_item = str(last_item.get(message_attribute, ''))
+            else:
+                for attr in message_attribute:
+                    first_item = first_item.get(attr, {})
+                    last_item = last_item.get(attr, {})
+                first_item = str(first_item)
+                last_item = str(last_item)
+            show_message = show_message.replace(FIRST_ITEM_MARKER, first_item)
+            show_message = show_message.replace(LAST_ITEM_MARKER, last_item)
+        sys.stderr.write('\r')
+        sys.stderr.flush()
+        sys.stderr.write(show_message)
+    return (page_token, total_items)
+
+def finalize_page_message(page_message):
+    """ Issue final page_message """
+    if page_message and (page_message[-1] != '\n'):
+        sys.stderr.write('\r\n')
+        sys.stderr.flush()
+
+def get_all_pages(service,
+                  function,
+                  items='items',
+                  page_message=None,
+                  message_attribute=None,
+                  soft_errors=False,
+                  throw_reasons=None,
+                  retry_reasons=None,
+                  **kwargs):
+    """Aggregates and returns all pages of a Google service function response.
+
+  All pages of items are aggregated and returned as a single list.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the method's
+      response object. The items in this field will be aggregated across all
+      pages and returned.
+    page_message: String, a message to be displayed to the user during paging.
+      Template strings allow for dynamic content to be inserted during paging.
+        Supported template strings:
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
+            pages.
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
+            display a unique property of the first item in the current page.
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
+            display a unique property of the last item in the current page.
+    message_attribute: String or list, the name of a signature field within a
+    single returned item which identifies that unique item. This field is used
+    with `page_message` to templatize a paging status message.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A list of all items received from all paged responses.
+  """
+    if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
+        page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
+        if page_key:
+            kwargs.update(page_key)
+    all_items = []
+    page_token = None
+    total_items = 0
+    while True:
+        page = call(service,
+                    function,
+                    soft_errors=soft_errors,
+                    throw_reasons=throw_reasons,
+                    retry_reasons=retry_reasons,
+                    **kwargs)
+        page_token, total_items = process_page(page, items, all_items, total_items, page_message, message_attribute)
+        if not page_token:
+            finalize_page_message(page_message)
+            return all_items
+        kwargs['pageToken'] = page_token
+
+
+# TODO: Make this private once all execution related items that use this method
+# have been brought into this file
+def handle_oauth_token_error(e, soft_errors):
+    """On a token error, exits the application and writes a message to stderr.
+
+  Args:
+    e: google.auth.exceptions.RefreshError, The error to handle.
+    soft_errors: Boolean, if True, suppresses any applicable errors and instead
+      returns to the caller.
+  """
+    token_error = str(e).replace('.', '')
+    if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
+            'Invalid response'):
+        if soft_errors:
+            return
+        if not GM_Globals[GM_CURRENT_API_USER]:
+            display.print_error(
+                MESSAGE_API_ACCESS_DENIED.format(
+                    GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+                    ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+            controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
+        else:
+            controlflow.system_error_exit(
+                19,
+                MESSAGE_SERVICE_NOT_APPLICABLE.format(
+                    GM_Globals[GM_CURRENT_API_USER]))
+    controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
+
+
+def get_enum_values_minus_unspecified(values):
+    return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gam/gapi/__init___test.py

@@ -0,0 +1,520 @@
+"""Tests for gapi."""
+
+import json
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import gam.gapi as gapi
+from gam.gapi import errors
+import httplib2
+
+
+def create_http_error(status, reason, message):
+    """Creates a HttpError object similar to most Google API Errors.
+
+  Args:
+    status: Int, the error's HTTP response status number.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    googleapiclient.errors.HttpError
+  """
+    response = httplib2.Response({
+        'status': status,
+        'content-type': 'application/json',
+    })
+    content = {
+        'error': {
+            'code': status,
+            'errors': [{
+                'reason': str(reason),
+                'message': message,
+            }]
+        }
+    }
+    content_bytes = json.dumps(content).encode('UTF-8')
+    return gapi.googleapiclient.errors.HttpError(response, content_bytes)
+
+
+class GapiTest(unittest.TestCase):
+
+    def setUp(self):
+        SetGlobalVariables()
+        self.mock_service = MagicMock()
+        self.mock_method_name = 'mock_method'
+        self.mock_method = getattr(self.mock_service, self.mock_method_name)
+
+        self.simple_3_page_response = [
+            {
+                'items': [{
+                    'position': 'page1,item1'
+                }, {
+                    'position': 'page1,item2'
+                }, {
+                    'position': 'page1,item3'
+                }],
+                'nextPageToken': 'page2'
+            },
+            {
+                'items': [{
+                    'position': 'page2,item1'
+                }, {
+                    'position': 'page2,item2'
+                }, {
+                    'position': 'page2,item3'
+                }],
+                'nextPageToken': 'page3'
+            },
+            {
+                'items': [{
+                    'position': 'page3,item1'
+                }, {
+                    'position': 'page3,item2'
+                }, {
+                    'position': 'page3,item3'
+                }],
+            },
+        ]
+        self.empty_items_response = {'items': []}
+
+        super(GapiTest, self).setUp()
+
+    def test_call_returns_basic_200_response(self):
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, self.mock_method().execute.return_value)
+
+    def test_call_passes_target_method_params(self):
+        gapi.call(self.mock_service,
+                  self.mock_method_name,
+                  my_param_1=1,
+                  my_param_2=2)
+        self.assertEqual(self.mock_method.call_count, 1)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(method_kwargs.get('my_param_1'), 1)
+        self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+    @patch.object(gapi.errors, 'get_gapi_error_detail')
+    def test_call_retries_with_soft_errors(self, mock_error_detail):
+        mock_error_detail.return_value = (-1, 'aReason', 'some message')
+
+        # Make the request fail first, then return the proper response on the retry.
+        fake_http_error = create_http_error(403, 'aReason', 'unused message')
+        fake_200_response = MagicMock()
+        self.mock_method.return_value.execute.side_effect = [
+            fake_http_error, fake_200_response
+        ]
+
+        response = gapi.call(self.mock_service,
+                             self.mock_method_name,
+                             soft_errors=True)
+        self.assertEqual(response, fake_200_response)
+        self.assertEqual(self.mock_service._http.credentials.refresh.call_count,
+                         1)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+
+    def test_call_throws_for_provided_reason(self):
+        throw_reason = errors.ErrorReason.USER_NOT_FOUND
+        fake_http_error = create_http_error(404, throw_reason, 'forced throw')
+        self.mock_method.return_value.execute.side_effect = fake_http_error
+
+        gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
+        with self.assertRaises(gam_exception):
+            gapi.call(self.mock_service,
+                      self.mock_method_name,
+                      throw_reasons=[throw_reason])
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_request_for_default_retry_reasons(
+        self, mock_wait_on_failure):
+
+        # Test using one of the default retry reasons
+        default_throw_reason = errors.ErrorReason.BACKEND_ERROR
+        self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
+
+        fake_http_error = create_http_error(404, default_throw_reason,
+                                            'message')
+        fake_200_response = MagicMock()
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_http_error, fake_200_response
+        ]
+
+        response = gapi.call(self.mock_service,
+                             self.mock_method_name,
+                             retry_reasons=[])
+        self.assertEqual(response, fake_200_response)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+        # Make sure a backoff technique was used for retry.
+        self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_requests_for_provided_retry_reasons(
+        self, unused_mock_wait_on_failure):
+
+        retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
+        fake_retrieable_error1 = create_http_error(400, retry_reason1,
+                                                   'Forced Error 1')
+        retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
+        fake_retrieable_error2 = create_http_error(400, retry_reason2,
+                                                   'Forced Error 2')
+        non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
+        fake_non_retriable_error = create_http_error(
+            400, non_retriable_reason,
+            'This error should not cause the request to be retried')
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_retrieable_error1, fake_retrieable_error2,
+            fake_non_retriable_error
+        ]
+
+        with self.assertRaises(SystemExit):
+            # The third call should raise the SystemExit when non_retriable_error is
+            # raised.
+            gapi.call(self.mock_service,
+                      self.mock_method_name,
+                      retry_reasons=[retry_reason1, retry_reason2])
+
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
+
+    def test_call_exits_on_oauth_token_error(self):
+        # An error with any OAUTH2_TOKEN_ERROR
+        fake_token_error = gapi.google.auth.exceptions.RefreshError(
+            errors.OAUTH2_TOKEN_ERRORS[0])
+        self.mock_method.return_value.execute.side_effect = fake_token_error
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_exits_on_nonretriable_error(self):
+        error_reason = 'unknownReason'
+        fake_http_error = create_http_error(500, error_reason,
+                                            'Testing unretriable errors')
+        self.mock_method.return_value.execute.side_effect = fake_http_error
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_exits_on_request_valueerror(self):
+        self.mock_method.return_value.execute.side_effect = ValueError()
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_clears_bad_http_cache_on_request_failure(self):
+        self.mock_service._http.cache = 'something that is not None'
+        fake_200_response = MagicMock()
+        self.mock_method.return_value.execute.side_effect = [
+            ValueError(), fake_200_response
+        ]
+
+        self.assertIsNotNone(self.mock_service._http.cache)
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, fake_200_response)
+        # Assert the cache was cleared
+        self.assertIsNone(self.mock_service._http.cache)
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_requests_with_backoff_on_servernotfounderror(
+        self, mock_wait_on_failure):
+        fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
+        fake_200_response = MagicMock()
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_servernotfounderror, fake_200_response
+        ]
+
+        http_connections = self.mock_service._http.connections
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, fake_200_response)
+        # HTTP cached connections should be cleared on receiving this error
+        self.assertNotEqual(http_connections,
+                            self.mock_service._http.connections)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+        # Make sure a backoff technique was used for retry.
+        self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+    def test_get_items_calls_correct_service_function(self):
+        gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertTrue(self.mock_method.called)
+
+    def test_get_items_returns_one_page(self):
+        fake_response = {'items': [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertEqual(page, fake_response['items'])
+
+    def test_get_items_non_default_page_field_name(self):
+        field_name = 'things'
+        fake_response = {field_name: [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_items(self.mock_service,
+                              self.mock_method_name,
+                              items=field_name)
+        self.assertEqual(page, fake_response[field_name])
+
+    def test_get_items_passes_additional_kwargs_to_service(self):
+        gapi.get_items(self.mock_service,
+                       self.mock_method_name,
+                       my_param_1=1,
+                       my_param_2=2)
+        self.assertEqual(self.mock_method.call_count, 1)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(1, method_kwargs.get('my_param_1'))
+        self.assertEqual(2, method_kwargs.get('my_param_2'))
+
+    def test_get_items_returns_empty_list_when_no_items_returned(self):
+        non_items_response = {'noItemsInThisResponse': {}}
+        self.mock_method.return_value.execute.return_value = non_items_response
+        page = gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertIsInstance(page, list)
+        self.assertEqual(0, len(page))
+
+    def test_get_all_pages_returns_all_items(self):
+        page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
+        page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
+        page_3 = {'items': ['3-1', '3-2', '3-3']}
+        self.mock_method.return_value.execute.side_effect = [
+            page_1, page_2, page_3
+        ]
+        response_items = gapi.get_all_pages(self.mock_service,
+                                            self.mock_method_name)
+        self.assertListEqual(
+            response_items, page_1['items'] + page_2['items'] + page_3['items'])
+
+    def test_get_all_pages_includes_next_pagetoken_in_request(self):
+        page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
+        page_2 = {'items': ['2-1', '2-2', '2-3']}
+        self.mock_method.return_value.execute.side_effect = [page_1, page_2]
+
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           pageSize=100)
+        self.assertEqual(self.mock_method.call_count, 2)
+        call_2_kwargs = self.mock_method.call_args_list[1][1]
+        self.assertIn('pageToken', call_2_kwargs)
+        self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
+
+    def test_get_all_pages_uses_default_max_page_size(self):
+        sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
+        sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
+        self.mock_method.return_value.methodId = sample_api_id
+        self.mock_service._rootDesc = {
+            'resources': {
+                'someResource': {
+                    'methods': {
+                        'someMethod': {
+                            'id': sample_api_id,
+                            'parameters': {
+                                'maxResults': {
+                                    'maximum': sample_api_max_results
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+        gapi.get_all_pages(self.mock_service, self.mock_method_name)
+        request_method_kwargs = self.mock_method.call_args[1]
+        self.assertIn('maxResults', request_method_kwargs)
+        self.assertEqual(request_method_kwargs['maxResults'],
+                         gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
+
+    def test_get_all_pages_max_page_size_overrided(self):
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           pageSize=123456)
+        request_method_kwargs = self.mock_method.call_args[1]
+        self.assertIn('pageSize', request_method_kwargs)
+        self.assertEqual(123456, request_method_kwargs['pageSize'])
+
+    def test_get_all_pages_prints_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        self.assertIn(paging_message, messages_written)
+
+    def test_get_all_pages_prints_paging_message_inline(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+
+        # Make sure a return carriage was written between two pages
+        paging_message_call_positions = [
+            i for i, message in enumerate(messages_written)
+            if message == paging_message
+        ]
+        self.assertGreater(len(paging_message_call_positions), 1)
+        printed_between_page_messages = messages_written[
+            paging_message_call_positions[0]:paging_message_call_positions[1]]
+        self.assertIn('\r', printed_between_page_messages)
+
+    def test_get_all_pages_ends_paging_message_with_newline(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        last_page_message_index = len(
+            messages_written) - messages_written[::-1].index(paging_message)
+        last_carriage_return_index = len(
+            messages_written) - messages_written[::-1].index('\r\n')
+        self.assertGreater(last_carriage_return_index, last_page_message_index)
+
+    def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'Total number of items discovered: %%total_items%%'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_item_count = len(self.simple_3_page_response[0]['items'])
+        page_1_message = paging_message.replace('%%total_items%%',
+                                                str(page_1_item_count))
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_item_count = len(self.simple_3_page_response[1]['items'])
+        page_2_message = paging_message.replace(
+            '%%total_items%%', str(page_1_item_count + page_2_item_count))
+        self.assertIn(page_2_message, messages_written)
+
+        page_3_item_count = len(self.simple_3_page_response[2]['items'])
+        page_3_message = paging_message.replace(
+            '%%total_items%%',
+            str(page_1_item_count + page_2_item_count + page_3_item_count))
+        self.assertIn(page_3_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%total_items', message)
+
+    def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'First item in page: %%first_item%%'
+
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message,
+                               message_attribute='position')
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_message = paging_message.replace(
+            '%%first_item%%',
+            self.simple_3_page_response[0]['items'][0]['position'])
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_message = paging_message.replace(
+            '%%first_item%%',
+            self.simple_3_page_response[1]['items'][0]['position'])
+        self.assertIn(page_2_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%first_item', message)
+
+    def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'Last item in page: %%last_item%%'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message,
+                               message_attribute='position')
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_message = paging_message.replace(
+            '%%last_item%%',
+            self.simple_3_page_response[0]['items'][-1]['position'])
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_message = paging_message.replace(
+            '%%last_item%%',
+            self.simple_3_page_response[1]['items'][-1]['position'])
+        self.assertIn(page_2_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%last_item', message)
+
+    def test_get_all_pages_prints_all_attributes_in_paging_message(self):
+        pass
+
+    def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           my_param_1=1,
+                           my_param_2=2)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(method_kwargs.get('my_param_1'), 1)
+        self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+    @patch.object(gapi, 'call')
+    def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
+        throw_for = MagicMock()
+        retry_for = MagicMock()
+        mock_call.return_value = self.empty_items_response
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           throw_reasons=throw_for,
+                           retry_reasons=retry_for)
+        method_kwargs = mock_call.call_args[1]
+        self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
+        self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
+
+    def test_get_all_pages_non_default_items_field_name(self):
+        field_name = 'things'
+        fake_response = {field_name: [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_all_pages(self.mock_service,
+                                  self.mock_method_name,
+                                  items=field_name)
+        self.assertEqual(page, fake_response[field_name])
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/calendar.py

@@ -0,0 +1,988 @@
+import csv
+import sys
+import uuid
+
+# TODO: get rid of these hacks
+import gam
+from gam.var import *
+
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
+
+
+def normalizeCalendarId(calname, checkPrimary=False):
+    if checkPrimary and calname.lower() == 'primary':
+        return calname
+    if not GC_Values[GC_DOMAIN]:
+        GC_Values[GC_DOMAIN] = gam._getValueFromOAuth('hd')
+    return gam.convertUIDtoEmailAddress(calname,
+                                        email_types=['user', 'resource'])
+
+
+def buildCalendarGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+    return (calendarId, gam.buildGAPIServiceObject('calendar', calendarId))
+
+
+def buildCalendarDataGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+
+    # Try to impersonate the calendar owner. If we fail, fall back to using
+    # admin for authentication. Resource calendars cannot be impersonated,
+    # so we need to access them as the admin.
+    cal = None
+    if not calname.endswith('.calendar.google.com'):
+        cal = gam.buildGAPIServiceObject('calendar', calendarId, False)
+    if cal is None:
+        _, cal = buildCalendarGAPIObject(gam._get_admin_email())
+    return (calendarId, cal)
+
+
+def printShowACLs(csvFormat):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    toDrive = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if csvFormat and myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            action = ['showacl', 'printacl'][csvFormat]
+            message = f'gam calendar <email> {action}'
+            controlflow.invalid_argument_exit(sys.argv[i], message)
+    acls = gapi.get_all_pages(cal.acl(), 'list', 'items', calendarId=calendarId)
+    i = 0
+    if csvFormat:
+        titles = []
+        rows = []
+    else:
+        count = len(acls)
+    for rule in acls:
+        i += 1
+        if csvFormat:
+            row = utils.flatten_json(rule, None)
+            for key in row:
+                if key not in titles:
+                    titles.append(key)
+            rows.append(row)
+        else:
+            formatted_acl = formatACLRule(rule)
+            current_count = display.current_count(i, count)
+            print(
+                f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
+    if csvFormat:
+        display.write_csv_file(rows, titles, f'{calendarId} Calendar ACLs',
+                               toDrive)
+
+
+def _getCalendarACLScope(i, body):
+    body['scope'] = {}
+    myarg = sys.argv[i].lower()
+    body['scope']['type'] = myarg
+    i += 1
+    if myarg in ['user', 'group']:
+        body['scope']['value'] = gam.normalizeEmailAddressOrUID(sys.argv[i],
+                                                                noUid=True)
+        i += 1
+    elif myarg == 'domain':
+        if i < len(sys.argv) and \
+           sys.argv[i].lower().replace('_', '') != 'sendnotifications':
+            body['scope']['value'] = sys.argv[i].lower()
+            i += 1
+        else:
+            body['scope']['value'] = GC_Values[GC_DOMAIN]
+    elif myarg != 'default':
+        body['scope']['type'] = 'user'
+        body['scope']['value'] = gam.normalizeEmailAddressOrUID(myarg,
+                                                                noUid=True)
+    return i
+
+
+CALENDAR_ACL_ROLES_MAP = {
+    'editor': 'writer',
+    'freebusy': 'freeBusyReader',
+    'freebusyreader': 'freeBusyReader',
+    'owner': 'owner',
+    'read': 'reader',
+    'reader': 'reader',
+    'writer': 'writer',
+    'none': 'none',
+}
+
+
+def addACL(function):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    myarg = sys.argv[4].lower().replace('_', '')
+    if myarg not in CALENDAR_ACL_ROLES_MAP:
+        controlflow.expected_argument_exit('Role',
+                                           ', '.join(CALENDAR_ACL_ROLES_MAP),
+                                           myarg)
+    body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]}
+    i = _getCalendarACLScope(5, body)
+    sendNotifications = True
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'sendnotifications':
+            sendNotifications = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {function.lower()}')
+    print(f'Calendar: {calendarId}, {function} ACL: {formatACLRule(body)}')
+    gapi.call(cal.acl(),
+              'insert',
+              calendarId=calendarId,
+              body=body,
+              sendNotifications=sendNotifications)
+
+
+def delACL():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    if sys.argv[4].lower() == 'id':
+        ruleId = sys.argv[5]
+        print(f'Removing rights for {ruleId} to {calendarId}')
+        gapi.call(cal.acl(), 'delete', calendarId=calendarId, ruleId=ruleId)
+    else:
+        body = {'role': 'none'}
+        i = 4
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in CALENDAR_ACL_ROLES_MAP:
+            i += 1
+        _getCalendarACLScope(i, body)
+        print(f'Calendar: {calendarId}, Delete ACL: {formatACLScope(body)}')
+        gapi.call(cal.acl(),
+                  'insert',
+                  calendarId=calendarId,
+                  body=body,
+                  sendNotifications=False)
+
+
+def wipeData():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    gapi.call(cal.calendars(), 'clear', calendarId=calendarId)
+
+
+def printEvents():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    q = showDeleted = showHiddenInvitations = timeMin = \
+        timeMax = timeZone = updatedMin = None
+    toDrive = False
+    titles = []
+    csvRows = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            q = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'includedeleted':
+            showDeleted = True
+            i += 1
+        elif myarg == 'includehidden':
+            showHiddenInvitations = True
+            i += 1
+        elif myarg == 'after':
+            timeMin = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'before':
+            timeMax = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'updated':
+            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], 'gam calendar <email> printevents')
+    page_message = gapi.got_total_items_msg(f'Events for {calendarId}', '')
+    results = gapi.get_all_pages(cal.events(),
+                                 'list',
+                                 'items',
+                                 page_message=page_message,
+                                 calendarId=calendarId,
+                                 q=q,
+                                 showDeleted=showDeleted,
+                                 showHiddenInvitations=showHiddenInvitations,
+                                 timeMin=timeMin,
+                                 timeMax=timeMax,
+                                 timeZone=timeZone,
+                                 updatedMin=updatedMin)
+    for result in results:
+        row = {'calendarId': calendarId}
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(result, flattened=row), csvRows, titles)
+    display.sort_csv_titles(['calendarId', 'id', 'summary', 'status'], titles)
+    display.write_csv_file(csvRows, titles, 'Calendar Events', toDrive)
+
+
+def formatACLScope(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]})'
+    return f'(Scope: {rule["scope"]["type"]})'
+
+
+def formatACLRule(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]}, ' \
+               f'Role: {rule["role"]})'
+    return f'(Scope: {rule["scope"]["type"]}, Role: {rule["role"]})'
+
+
+def getSendUpdates(myarg, i, cal):
+    if myarg == 'notifyattendees':
+        sendUpdates = 'all'
+        i += 1
+    elif myarg == 'sendnotifications':
+        sendUpdates = 'all' if gam.getBoolean(sys.argv[i +
+                                                       1], myarg) else 'none'
+        i += 2
+    else:  # 'sendupdates':
+        sendUpdatesMap = {}
+        for val in cal._rootDesc['resources']['events']['methods']['delete'][
+                'parameters']['sendUpdates']['enum']:
+            sendUpdatesMap[val.lower()] = val
+        sendUpdates = sendUpdatesMap.get(sys.argv[i + 1].lower(), False)
+        if not sendUpdates:
+            controlflow.expected_argument_exit('sendupdates',
+                                               ', '.join(sendUpdatesMap),
+                                               sys.argv[i + 1])
+        i += 2
+    return (sendUpdates, i)
+
+
+def moveOrDeleteEvent(moveOrDelete):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    sendUpdates = 'none'
+    doit = False
+    kwargs = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg in ['id', 'eventid']:
+            eventId = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['query', 'eventquery']:
+            controlflow.system_error_exit(
+                2, f'query is no longer supported for {moveOrDelete}event. ' \
+                   f'Use "gam calendar <email> printevents query <query> | ' \
+                   f'gam csv - gam {moveOrDelete}event id ~id" instead.')
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        elif moveOrDelete == 'move' and myarg == 'destination':
+            kwargs['destination'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {moveOrDelete}event')
+    if doit:
+        print(f' going to {moveOrDelete} eventId {eventId}')
+        gapi.call(cal.events(),
+                  moveOrDelete,
+                  calendarId=calendarId,
+                  eventId=eventId,
+                  sendUpdates=sendUpdates,
+                  **kwargs)
+    else:
+        print(
+            f' would {moveOrDelete} eventId {eventId}. Add doit to command ' \
+            f'to actually {moveOrDelete} event')
+
+
+def infoEvent():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    eventId = sys.argv[4]
+    result = gapi.call(cal.events(),
+                       'get',
+                       calendarId=calendarId,
+                       eventId=eventId)
+    display.print_json(result)
+
+
+def addOrUpdateEvent(action):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    # only way for non-Google calendars to get updates is via email
+    kwargs = {}
+    body = {}
+    if action == 'add':
+        i = 4
+        func = 'insert'
+    else:
+        eventId = sys.argv[4]
+        kwargs = {'eventId': eventId}
+        i = 5
+        func = 'patch'
+        requires_full_update = [
+            'attendee', 'optionalattendee', 'removeattendee',
+            'replacedescription'
+        ]
+        for arg in sys.argv[i:]:
+            if arg.replace('_', '').lower() in requires_full_update:
+                func = 'update'
+                body = gapi.call(cal.events(),
+                                 'get',
+                                 calendarId=calendarId,
+                                 eventId=eventId)
+                break
+    sendUpdates, body = getEventAttributes(i, calendarId, cal, body, action)
+    result = gapi.call(cal.events(),
+                       func,
+                       conferenceDataVersion=1,
+                       supportsAttachments=True,
+                       calendarId=calendarId,
+                       sendUpdates=sendUpdates,
+                       body=body,
+                       fields='id',
+                       **kwargs)
+    print(f'Event {result["id"]} {action} finished')
+
+
+def _remove_attendee(attendees, remove_email):
+    return [
+        attendee for attendee in attendees
+        if not attendee['email'].lower() == remove_email
+    ]
+
+
+def getEventAttributes(i, calendarId, cal, body, action):
+    # Default to external only so non-Google
+    # calendars are notified of changes
+    sendUpdates = 'externalOnly'
+    action = 'update' if body else 'add'
+    timeZone = None
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg == 'attendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({'email': sys.argv[i + 1]})
+            i += 2
+        elif myarg == 'removeattendee' and action == 'update':
+            remove_email = sys.argv[i + 1].lower()
+            if 'attendees' in body:
+                body['attendees'] = _remove_attendee(body['attendees'],
+                                                     remove_email)
+            i += 2
+        elif myarg == 'optionalattendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({
+                'email': sys.argv[i + 1],
+                'optional': True
+            })
+            i += 2
+        elif myarg == 'anyonecanaddself':
+            body['anyoneCanAddSelf'] = True
+            i += 1
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'replacedescription' and action == 'update':
+            search = sys.argv[i + 1]
+            replace = sys.argv[i + 2]
+            if 'description' in body:
+                body['description'] = re.sub(search, replace,
+                                             body['description'])
+            i += 3
+        elif myarg == 'start':
+            if sys.argv[i + 1].lower() == 'allday':
+                body['start'] = {'date': utils.get_yyyymmdd(sys.argv[i + 2])}
+                i += 3
+            else:
+                start_time = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+                body['start'] = {'dateTime': start_time}
+                i += 2
+        elif myarg == 'end':
+            if sys.argv[i + 1].lower() == 'allday':
+                body['end'] = {'date': utils.get_yyyymmdd(sys.argv[i + 2])}
+                i += 3
+            else:
+                end_time = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+                body['end'] = {'dateTime': end_time}
+                i += 2
+        elif myarg == 'guestscantinviteothers':
+            body['guestsCanInviteOthers'] = False
+            i += 1
+        elif myarg == 'guestscaninviteothers':
+            body['guestsCanInviteTohters'] = gam.getBoolean(
+                sys.argv[i + 1], 'guestscaninviteothers')
+            i += 2
+        elif myarg == 'guestscantseeothers':
+            body['guestsCanSeeOtherGuests'] = False
+            i += 1
+        elif myarg == 'guestscanseeothers':
+            body['guestsCanSeeOtherGuests'] = gam.getBoolean(
+                sys.argv[i + 1], 'guestscanseeothers')
+            i += 2
+        elif myarg == 'guestscanmodify':
+            body['guestsCanModify'] = gam.getBoolean(sys.argv[i + 1],
+                                                     'guestscanmodify')
+            i += 2
+        elif myarg == 'id':
+            if action == 'update':
+                controlflow.invalid_argument_exit(
+                    'id', 'gam calendar <calendar> updateevent')
+            body['id'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'available':
+            body['transparency'] = 'transparent'
+            i += 1
+        elif myarg == 'transparency':
+            validTransparency = ['opaque', 'transparent']
+            if sys.argv[i + 1].lower() in validTransparency:
+                body['transparency'] = sys.argv[i + 1].lower()
+            else:
+                controlflow.expected_argument_exit('transparency',
+                                                   ', '.join(validTransparency),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg == 'visibility':
+            validVisibility = ['default', 'public', 'private']
+            if sys.argv[i + 1].lower() in validVisibility:
+                body['visibility'] = sys.argv[i + 1].lower()
+            else:
+                controlflow.expected_argument_exit('visibility',
+                                                   ', '.join(validVisibility),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg == 'tentative':
+            body['status'] = 'tentative'
+            i += 1
+        elif myarg == 'status':
+            validStatus = ['confirmed', 'tentative', 'cancelled']
+            if sys.argv[i + 1].lower() in validStatus:
+                body['status'] = sys.argv[i + 1].lower()
+            else:
+                controlflow.expected_argument_exit('visibility',
+                                                   ', '.join(validStatus),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg == 'source':
+            body['source'] = {'title': sys.argv[i + 1], 'url': sys.argv[i + 2]}
+            i += 3
+        elif myarg == 'noreminders':
+            body['reminders'] = {'useDefault': False}
+            i += 1
+        elif myarg == 'reminder':
+            minutes = \
+            gam.getInteger(sys.argv[i+1], myarg, minVal=0,
+                                maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+            reminder = {'minutes': minutes, 'method': sys.argv[i + 2]}
+            body.setdefault('reminders', {'overrides': [], 'useDefault': False})
+            body['reminders']['overrides'].append(reminder)
+            i += 3
+        elif myarg == 'recurrence':
+            body.setdefault('recurrence', [])
+            body['recurrence'].append(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'privateproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['private'][sys.argv[i +
+                                                           1]] = sys.argv[i + 2]
+            i += 3
+        elif myarg == 'sharedproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['shared'][sys.argv[i + 1]] = sys.argv[i +
+                                                                             2]
+            i += 3
+        elif myarg == 'colorindex':
+            body['colorId'] = gam.getInteger(sys.argv[i + 1], myarg,
+                                             CALENDAR_EVENT_MIN_COLOR_INDEX,
+                                             CALENDAR_EVENT_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'hangoutsmeet':
+            body['conferenceData'] = {
+                'createRequest': {
+                    'requestId': f'{str(uuid.uuid4())}'
+                }
+            }
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {action}event')
+    if ('recurrence' in body) and (('start' in body) or ('end' in body)):
+        if not timeZone:
+            timeZone = gapi.call(cal.calendars(),
+                                 'get',
+                                 calendarId=calendarId,
+                                 fields='timeZone')['timeZone']
+        if 'start' in body:
+            body['start']['timeZone'] = timeZone
+        if 'end' in body:
+            body['end']['timeZone'] = timeZone
+    return (sendUpdates, body)
+
+
+def modifySettings():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'timezone':
+            body['timeZone'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam calendar <email> modify')
+    gapi.call(cal.calendars(), 'patch', calendarId=calendarId, body=body)
+
+
+def changeAttendees(users):
+    do_it = True
+    i = 5
+    allevents = False
+    start_date = end_date = None
+    while len(sys.argv) > i:
+        myarg = sys.argv[i].lower()
+        if myarg == 'csv':
+            csv_file = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'dryrun':
+            do_it = False
+            i += 1
+        elif myarg == 'start':
+            start_date = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'end':
+            end_date = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'allevents':
+            allevents = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], 'gam <users> update calattendees')
+    attendee_map = {}
+    f = fileutils.open_file(csv_file)
+    csvFile = csv.reader(f)
+    for row in csvFile:
+        attendee_map[row[0].lower()] = row[1].lower()
+    fileutils.close_file(f)
+    for user in users:
+        sys.stdout.write(f'Checking user {user}\n')
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        page_token = None
+        while True:
+            events_page = gapi.call(cal.events(),
+                                    'list',
+                                    calendarId=user,
+                                    pageToken=page_token,
+                                    timeMin=start_date,
+                                    timeMax=end_date,
+                                    showDeleted=False,
+                                    showHiddenInvitations=False)
+            print(f'Got {len(events_page.get("items", []))}')
+            for event in events_page.get('items', []):
+                if event['status'] == 'cancelled':
+                    # print u' skipping cancelled event'
+                    continue
+                try:
+                    event_summary = event['summary']
+                except (KeyError, UnicodeEncodeError, UnicodeDecodeError):
+                    event_summary = event['id']
+                try:
+                    organizer = event['organizer']['email'].lower()
+                    if not allevents and organizer != user:
+                        #print(f' skipping not-my-event {event_summary}')
+                        continue
+                except KeyError:
+                    pass  # no email for organizer
+                needs_update = False
+                try:
+                    for attendee in event['attendees']:
+                        try:
+                            if attendee['email'].lower() in attendee_map:
+                                old_email = attendee['email'].lower()
+                                new_email = attendee_map[
+                                    attendee['email'].lower()]
+                                print(f' SWITCHING attendee {old_email} to ' \
+                                    f'{new_email} for {event_summary}')
+                                event['attendees'].remove(attendee)
+                                event['attendees'].append({'email': new_email})
+                                needs_update = True
+                        except KeyError:  # no email for that attendee
+                            pass
+                except KeyError:
+                    continue  # no attendees
+                if needs_update:
+                    body = {}
+                    body['attendees'] = event['attendees']
+                    print(f'UPDATING {event_summary}')
+                    if do_it:
+                        gapi.call(cal.events(),
+                                  'patch',
+                                  calendarId=user,
+                                  eventId=event['id'],
+                                  sendNotifications=False,
+                                  body=body)
+                    else:
+                        print(' not pulling the trigger.')
+                # else:
+                #  print(f' no update needed for {event_summary}')
+            try:
+                page_token = events_page['nextPageToken']
+            except KeyError:
+                break
+
+
+def deleteCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    for user in users:
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        gapi.call(cal.calendarList(),
+                  'delete',
+                  soft_errors=True,
+                  calendarId=calendarId)
+
+
+CALENDAR_REMINDER_MAX_MINUTES = 40320
+
+CALENDAR_MIN_COLOR_INDEX = 1
+CALENDAR_MAX_COLOR_INDEX = 24
+
+CALENDAR_EVENT_MIN_COLOR_INDEX = 1
+CALENDAR_EVENT_MAX_COLOR_INDEX = 11
+
+
+def getCalendarAttributes(i, body, function):
+    colorRgbFormat = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'selected':
+            body['selected'] = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg == 'hidden':
+            body['hidden'] = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg == 'summary':
+            body['summaryOverride'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'colorindex':
+            body['colorId'] = gam.getInteger(sys.argv[i + 1],
+                                             myarg,
+                                             minVal=CALENDAR_MIN_COLOR_INDEX,
+                                             maxVal=CALENDAR_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'backgroundcolor':
+            body['backgroundColor'] = gam.getColor(sys.argv[i + 1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'foregroundcolor':
+            body['foregroundColor'] = gam.getColor(sys.argv[i + 1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'reminder':
+            body.setdefault('defaultReminders', [])
+            method = sys.argv[i + 1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_REMINDER_METHODS:
+                    controlflow.expected_argument_exit(
+                        'Method', ', '.join(CALENDAR_REMINDER_METHODS +
+                                            CLEAR_NONE_ARGUMENT), method)
+                minutes = gam.getInteger(sys.argv[i + 2],
+                                         myarg,
+                                         minVal=0,
+                                         maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+                body['defaultReminders'].append({
+                    'method': method,
+                    'minutes': minutes
+                })
+                i += 3
+            else:
+                i += 2
+        elif myarg == 'notification':
+            body.setdefault('notificationSettings', {'notifications': []})
+            method = sys.argv[i + 1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_NOTIFICATION_METHODS:
+                    controlflow.expected_argument_exit(
+                        'Method', ', '.join(CALENDAR_NOTIFICATION_METHODS +
+                                            CLEAR_NONE_ARGUMENT), method)
+                eventType = sys.argv[i + 2].lower()
+                if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP:
+                    controlflow.expected_argument_exit(
+                        'Event', ', '.join(CALENDAR_NOTIFICATION_TYPES_MAP),
+                        eventType)
+                notice = {
+                    'method': method,
+                    'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]
+                }
+                body['notificationSettings']['notifications'].append(notice)
+                i += 3
+            else:
+                i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              f'gam {function} calendar')
+    return colorRgbFormat
+
+
+def addCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    body = {'id': calendarId, 'selected': True, 'hidden': False}
+    colorRgbFormat = getCalendarAttributes(6, body, 'add')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f'Subscribing {user} to calendar {calendarId}{current_count}')
+        gapi.call(cal.calendarList(),
+                  'insert',
+                  soft_errors=True,
+                  body=body,
+                  colorRgbFormat=colorRgbFormat)
+
+
+def updateCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    body = {}
+    colorRgbFormat = getCalendarAttributes(6, body, 'update')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f"Updating {user}'s subscription to calendar ' \
+              f'{calendarId}{current_count}")
+        calId = calendarId if calendarId != 'primary' else user
+        gapi.call(cal.calendarList(),
+                  'patch',
+                  soft_errors=True,
+                  calendarId=calId,
+                  body=body,
+                  colorRgbFormat=colorRgbFormat)
+
+
+def _showCalendar(userCalendar, j, jcount):
+    current_count = display.current_count(j, jcount)
+    summary = userCalendar.get('summaryOverride', userCalendar['summary'])
+    print(f'  Calendar: {userCalendar["id"]}{current_count}')
+    print(f'    Summary: {summary}')
+    print(f'    Description: {userCalendar.get("description", "")}')
+    print(f'    Access Level: {userCalendar["accessRole"]}')
+    print(f'    Timezone: {userCalendar["timeZone"]}')
+    print(f'    Location: {userCalendar.get("location", "")}')
+    print(f'    Hidden: {userCalendar.get("hidden", "False")}')
+    print(f'    Selected: {userCalendar.get("selected", "False")}')
+    print(f'    Color ID: {userCalendar["colorId"]}, ' \
+          f'Background Color: {userCalendar["backgroundColor"]}, ' \
+          f'Foreground Color: {userCalendar["foregroundColor"]}')
+    print('    Default Reminders:')
+    for reminder in userCalendar.get('defaultReminders', []):
+        print(f'      Method: {reminder["method"]}, ' \
+              f'Minutes: {reminder["minutes"]}')
+    print('    Notifications:')
+    if 'notificationSettings' in userCalendar:
+        notifications = userCalendar['notificationSettings'].get(
+            'notifications', [])
+        for notification in notifications:
+            print(f'      Method: {notification["method"]}, ' \
+                  f'Type: {notification["type"]}')
+
+
+def infoCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.call(cal.calendarList(),
+                           'get',
+                           soft_errors=True,
+                           calendarId=calendarId)
+        if result:
+            print(f'User: {user}, Calendar:{display.current_count(i, count)}')
+            _showCalendar(result, 1, 1)
+
+
+def printShowCalendars(users, csvFormat):
+    if csvFormat:
+        todrive = False
+        titles = []
+        csvRows = []
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, f"gam <users> {['show', 'print'][csvFormat]} calendars")
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.get_all_pages(cal.calendarList(),
+                                    'list',
+                                    'items',
+                                    soft_errors=True)
+        jcount = len(result)
+        if not csvFormat:
+            print(f'User: {user}, Calendars:{display.current_count(i, count)}')
+            if jcount == 0:
+                continue
+            j = 0
+            for userCalendar in result:
+                j += 1
+                _showCalendar(userCalendar, j, jcount)
+        else:
+            if jcount == 0:
+                continue
+            for userCalendar in result:
+                row = {'primaryEmail': user}
+                display.add_row_titles_to_csv_file(
+                    utils.flatten_json(userCalendar, flattened=row), csvRows,
+                    titles)
+    if csvFormat:
+        display.sort_csv_titles(['primaryEmail', 'id'], titles)
+        display.write_csv_file(csvRows, titles, 'Calendars', todrive)
+
+
+def showCalSettings(users):
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        feed = gapi.get_all_pages(cal.settings(),
+                                  'list',
+                                  'items',
+                                  soft_errors=True)
+        if feed:
+            current_count = display.current_count(i, count)
+            print(f'User: {user}, Calendar Settings:{current_count}')
+            settings = {}
+            for setting in feed:
+                settings[setting['id']] = setting['value']
+            for attr, value in sorted(settings.items()):
+                print(f'  {attr}: {value}')
+
+
+def transferSecCals(users):
+    target_user = sys.argv[5]
+    remove_source_user = sendNotifications = True
+    i = 6
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'keepuser':
+            remove_source_user = False
+            i += 1
+        elif myarg == 'sendnotifications':
+            sendNotifications = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam <users> transfer seccals')
+    if remove_source_user:
+        target_user, target_cal = buildCalendarGAPIObject(target_user)
+        if not target_cal:
+            return
+    for user in users:
+        user, source_cal = buildCalendarGAPIObject(user)
+        if not source_cal:
+            continue
+        calendars = gapi.get_all_pages(source_cal.calendarList(),
+                                       'list',
+                                       'items',
+                                       soft_errors=True,
+                                       minAccessRole='owner',
+                                       showHidden=True,
+                                       fields='items(id),nextPageToken')
+        for calendar in calendars:
+            calendarId = calendar['id']
+            if calendarId.find('@group.calendar.google.com') != -1:
+                body = {
+                    'role': 'owner',
+                    'scope': {
+                        'type': 'user',
+                        'value': target_user
+                    }
+                }
+                gapi.call(source_cal.acl(),
+                          'insert',
+                          calendarId=calendarId,
+                          body=body,
+                          sendNotifications=sendNotifications)
+                if remove_source_user:
+                    body = {
+                        'role': 'none',
+                        'scope': {
+                            'type': 'user',
+                            'value': user
+                        }
+                    }
+                    gapi.call(target_cal.acl(),
+                              'insert',
+                              calendarId=calendarId,
+                              body=body,
+                              sendNotifications=sendNotifications)

src/gam/gapi/cbcm.py

@@ -0,0 +1,303 @@
+"""Chrome Browser Cloud Management API calls"""
+
+import csv
+import os.path
+import sys
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+from gam import utils
+
+
+def _get_customerid():
+    ''' returns customer id without C prefix'''
+    customer_id = GC_Values[GC_CUSTOMER_ID]
+    if customer_id[0] == 'C':
+        customer_id = customer_id[1:]
+    return customer_id
+
+
+def build():
+    return gam.buildGAPIObject('cbcm')
+
+
+def delete():
+    cbcm = build()
+    device_id = sys.argv[3]
+    customer_id = _get_customerid()
+    gapi.call(cbcm.chromebrowsers(), 'delete', deviceId=device_id,
+              customer=customer_id)
+    print(f'Deleted browser {device_id}')
+
+
+def info():
+    cbcm = build()
+    device_id = sys.argv[3]
+    projection = 'BASIC'
+    fields = None
+    customer_id = _get_customerid()
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['basic', 'full']:
+            projection = myarg.upper()
+            i += 1
+        elif myarg == 'fields':
+            fields = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam info browser')
+    browser = gapi.call(cbcm.chromebrowsers(), 'get',
+                        customer=customer_id,
+                        fields=fields, deviceId=device_id,
+                        projection=projection)
+    display.print_json(browser)
+
+
+def move():
+    cbcm = build()
+    body = {'resource_ids': []}
+    customer_id = _get_customerid()
+    i = 3
+    resource_ids = []
+    batch_size = 600
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'ids':
+            resource_ids.extend(sys.argv[i + 1].split(','))
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            page_message = gapi.got_total_items_msg('Browsers', '...\n')
+            browsers = gapi.get_all_pages(cbcm.chromebrowsers(), 'list',
+                         'browsers', page_message=page_message,
+                         customer=customer_id,
+                         query=query, projection='BASIC',
+                         fields='browsers(deviceId),nextPageToken')
+            ids = [browser['deviceId'] for browser in browsers]
+            resource_ids.extend(ids)
+            i += 2
+        elif myarg == 'file':
+            with fileutils.open_file(sys.argv[i+1], strip_utf_bom=True) as filed:
+                for row in filed:
+                    rid = row.strip()
+                    if rid:
+                        resource_ids.append(rid)
+            i += 2
+        elif myarg == 'csvfile':
+            drive, fname_column = os.path.splitdrive(sys.argv[i+1])
+            if fname_column.find(':') == -1:
+                controlflow.system_error_exit(
+                    2, 'Expected csvfile FileName:FieldName')
+            (filename, column) = fname_column.split(':')
+            with fileutils.open_file(drive + filename) as filed:
+                input_file = csv.DictReader(filed, restval='')
+                if column not in input_file.fieldnames:
+                    controlflow.csv_field_error_exit(column,
+                                                     input_file.fieldnames)
+                for row in input_file:
+                    rid = row[column].strip()
+                    if rid:
+                        resource_ids.append(rid)
+            i += 2
+        elif myarg in ['ou', 'orgunit', 'org']:
+            org_unit = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            body['org_unit_path'] = org_unit
+            i += 2
+        elif myarg == 'batchsize':
+            batch_size = int(sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam move browsers')
+    if 'org_unit_path' not in body:
+        controlflow.missing_argument_exit('ou', 'gam move browsers')
+    elif not resource_ids:
+        controlflow.missing_argument_exit('query or ids',
+                                          'gam move browsers')
+    # split moves into max 600 devices per batch
+    for chunk in range(0, len(resource_ids), batch_size):
+        body['resource_ids'] = resource_ids[chunk:chunk + batch_size]
+        print(f' moving {len(body["resource_ids"])} browsers to ' \
+                       f'{body["org_unit_path"]}')
+        gapi.call(cbcm.chromebrowsers(), 'moveChromeBrowsersToOu',
+                  customer=customer_id, body=body)
+
+
+def print_():
+    cbcm = build()
+    customer_id = _get_customerid()
+    projection = 'BASIC'
+    orgUnitPath = query = None
+    fields = None
+    titles = []
+    csv_rows = []
+    todrive = False
+    sort_headers = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg in ['ou', 'org', 'orgunit']:
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1], pathOnly=True, absolutePath=True)
+            i += 2
+        elif myarg == 'projection':
+            projection = sys.argv[i + 1].upper()
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'sortheaders':
+            sort_headers = True
+            i += 1
+        elif myarg == 'fields':
+            fields = sys.argv[i + 1].replace(',', ' ').split()
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print browsers')
+    if fields:
+        fields.append('deviceId')
+        fields = f'browsers({",".join(set(fields))}),nextPageToken'
+    page_message = gapi.got_total_items_msg('Browsers', '...\n')
+    browsers = gapi.get_all_pages(cbcm.chromebrowsers(), 'list',
+                         'browsers', page_message=page_message,
+                         customer=customer_id,
+                         orgUnitPath=orgUnitPath, query=query, projection=projection,
+                         fields=fields)
+    for browser in browsers:
+        browser = utils.flatten_json(browser)
+        for a_key in browser:
+            if a_key not in titles:
+                titles.append(a_key)
+        csv_rows.append(browser)
+    if sort_headers:
+        display.sort_csv_titles(['deviceId',], titles)
+    display.write_csv_file(csv_rows, titles, 'Browsers', todrive)
+
+
+attributes = {
+    'assetid': 'annotatedAssetId',
+    'location': 'annotatedLocation',
+    'notes': 'annotatedNotes',
+    'user': 'annotatedUser'
+    }
+attribute_fields = ','.join(list(attributes.values()))
+
+def update():
+    cbcm = build()
+    customer_id = _get_customerid()
+    device_id = sys.argv[3]
+    body = {'deviceId': device_id}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in attributes:
+            body[attributes[myarg]] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam update browser')
+    browser = gapi.call(cbcm.chromebrowsers(), 'get', deviceId=device_id,
+                        customer=customer_id,
+                        projection='BASIC', fields=attribute_fields)
+    browser.update(body)
+    result = gapi.call(cbcm.chromebrowsers(), 'update', deviceId=device_id,
+                       customer=customer_id, body=browser,
+                       projection='BASIC', fields="deviceId")
+    print(f'Updated browser {result["deviceId"]}')
+
+
+def createtoken():
+    cbcm = build()
+    customer_id = _get_customerid()
+    body = {'token_type': 'CHROME_BROWSER'}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['ou', 'orgunit', 'org']:
+            body['org_unit_path'] = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['expire', 'expires']:
+            body['expire_time'] = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam create browsertoken')
+    browser = gapi.call(cbcm.enrollmentTokens(), 'create',
+                        customer=customer_id, body=body)
+    print(f'Created browser enrollment token {browser["token"]}')
+
+
+def revoketoken():
+    cbcm = build()
+    customer_id = _get_customerid()
+    token_permanent_id = sys.argv[3]
+    gapi.call(cbcm.enrollmentTokens(), 'revoke', tokenPermanentId=token_permanent_id,
+              customer=customer_id)
+    print(f'Deleted browser enrollment token {token_permanent_id}')
+
+
+def printshowtokens(csvFormat):
+    cbcm = build()
+    customer_id = _get_customerid()
+    query = None
+    fields = None
+    if csvFormat:
+        titles = ['token']
+        csv_rows = []
+        todrive = False
+        sort_headers = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif csvFormat and myarg == 'sortheaders':
+            sort_headers = True
+            i += 1
+        elif myarg == 'fields':
+            fields = sys.argv[i + 1].replace(',', ' ').split()
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                f"gam {['show', 'print'][csvFormat]} browsertokens")
+    if fields:
+        fields.append('token')
+        fields = f'chromeEnrollmentTokens({",".join(set(fields))}),nextPageToken'
+    page_message = gapi.got_total_items_msg('Chrome Browser Enrollment Tokens', '...\n')
+    browsers = gapi.get_all_pages(cbcm.enrollmentTokens(), 'list',
+                         'chromeEnrollmentTokens', page_message=page_message,
+                         customer=customer_id,
+                         query=query, fields=fields)
+    if not csvFormat:
+        count = len(browsers)
+        print(f'Show {count} Chrome Browser Enrollment Tokens')
+        i = 0
+        for browser in browsers:
+            i += 1
+            print(f'  Chrome Browser Enrollment Token: {browser["token"]}{gam.currentCount(i, count)}')
+            browser.pop('kind', None)
+            for field in browser:
+                print(f'    {field}: {browser[field]}')
+    else:
+        for browser in browsers:
+            browser = utils.flatten_json(browser)
+            for a_key in browser:
+                if a_key not in titles:
+                    titles.append(a_key)
+            csv_rows.append(browser)
+        if sort_headers:
+            display.sort_csv_titles(['token',], titles)
+        display.write_csv_file(csv_rows, titles, 'Chrome Browser Enrollment Tokens', todrive)

src/gam/gapi/chromehistory.py

@@ -0,0 +1,229 @@
+"""Chrome Version History API calls"""
+
+import re
+import sys
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam import utils
+
+
+def build():
+    return gam.buildGAPIObjectNoAuthentication('versionhistory')
+
+
+CHROME_HISTORY_ENTITY_CHOICES = {
+  'platforms',
+  'channels',
+  'versions',
+  'releases',
+  }
+
+CHROME_VERSIONHISTORY_ORDERBY_CHOICE_MAP = {
+  'versions':  {
+    'channel': 'channel',
+    'name': 'name',
+    'platform': 'platform',
+    'version': 'version'
+    },
+  'releases': {
+    'channel': 'channel',
+    'endtime': 'endtime',
+    'fraction': 'fraction',
+    'name': 'name',
+    'platform': 'platform',
+    'starttime': 'starttime',
+    'version': 'version'
+    }
+  }
+
+CHROME_VERSIONHISTORY_TITLES = {
+  'platforms': ['platform'],
+  'channels':  ['channel', 'platform'],
+  'versions': ['version', 'channel', 'platform',
+               'major_version', 'minor_version', 'build', 'patch'],
+  'releases': ['version', 'channel', 'platform',
+               'major_version', 'minor_version', 'build', 'patch',
+               'fraction', 'serving.startTime','serving.endTime']
+  }
+
+def get_relative_milestone(channel='stable', minus=0):
+    '''
+    takes a channel and minus like stable and -1.
+    returns current given milestone number
+    '''
+    cv = build()
+    parent = f'chrome/platforms/all/channels/{channel}/versions/all'
+    releases = gapi.get_all_pages(cv.platforms().channels().versions().releases(),
+                                  'list',
+                                  'releases',
+                                  parent=parent,
+                                  fields='releases/version,nextPageToken')
+    milestones = []
+    # Note that milestones are usually sequential but some numbers
+    # may be skipped. For example, there was no Chrome 82 stable.
+    # Thus we need to do more than find the latest version and subtract.
+    for release in releases:
+        milestone = release.get('version').split('.')[0]
+        if milestone not in milestones:
+            milestones.append(milestone)
+    milestones.sort(reverse=True)
+    try:
+        return milestones[minus]
+    except IndexError:
+        return ''
+
+def get_platform_map(cv=None):
+    '''returns dict mapping of platform choices'''
+    if cv is None:
+        cv = build()
+    result = gapi.get_all_pages(cv.platforms(),
+                                'list',
+                                'platforms',
+                                parent='chrome')
+    platforms = [p.get('platformType', '').lower() for p in result]
+    platform_map = {'all': 'all'}
+    for cplatform in platforms:
+        key = cplatform.replace('_', '')
+        platform_map[key] = cplatform
+    return platform_map
+
+
+def get_channel_map(cv=None):
+    '''returns dict mapping of channel choices'''
+    if cv is None:
+        cv = build()
+    result = gapi.get_all_pages(cv.platforms().channels(),
+                                'list',
+                                'channels',
+                                parent='chrome/platforms/all')
+    channels = [c.get('channelType', '').lower() for c in result]
+    channels = list(set(channels))
+    channel_map = {'all': 'all'}
+    for channel in channels:
+        key = channel.replace('_', '')
+        channel_map[key] = channel
+    return channel_map
+
+def printHistory():
+    cv = build()
+    entityType = sys.argv[3].lower().replace('_', '')
+    if entityType not in CHROME_HISTORY_ENTITY_CHOICES:
+        msg = f'{entityType} is not a valid argument to "gam print chromehistory"'
+        controlflow.system_error_exit(3, msg)
+    todrive = False
+    csvRows = []
+    cplatform = 'all'
+    channel = 'all'
+    version = 'all'
+    kwargs = {}
+    orderByList = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif entityType != 'platforms' and myarg == 'platform':
+            cplatform = sys.argv[i + 1].lower().replace('_', '')
+            platform_map = get_platform_map(cv)
+            if cplatform not in platform_map:
+                controlflow.expected_argument_exit('platform',
+                                                   ', '.join(platform_map),
+                                                   cplatform)
+            cplatform = platform_map[cplatform]
+            i += 2
+        elif entityType in {'versions', 'releases'} and myarg == 'channel':
+            channel = sys.argv[i + 1].lower().replace('_', '')
+            channel_map = get_channel_map(cv)
+            if channel not in channel_map:
+                controlflow.expected_argument_exit('channel',
+                                                   ', '.join(channel_map),
+                                                   channel)
+            channel = channel_map[channel]
+            i += 2
+        elif entityType == 'releases' and myarg == 'version':
+            version = sys.argv[i + 1]
+            i += 2
+        elif entityType in {'versions', 'releases'} and myarg == 'orderby':
+            fieldName = sys.argv[i + 1].lower().replace('_', '')
+            i += 2
+            if fieldName in CHROME_VERSIONHISTORY_ORDERBY_CHOICE_MAP[entityType]:
+                fieldName = CHROME_VERSIONHISTORY_ORDERBY_CHOICE_MAP[entityType][fieldName]
+                orderBy = ''
+                if i < len(sys.argv):
+                    orderBy = sys.argv[i].lower()
+                    if orderBy in SORTORDER_CHOICES_MAP:
+                        orderBy = SORTORDER_CHOICES_MAP[orderBy]
+                        i += 1
+                if orderBy != 'DESCENDING':
+                    orderByList.append(fieldName)
+                else:
+                    orderByList.append(f'{fieldName} desc')
+            else:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(CHROME_VERSIONHISTORY_ORDERBY_CHOICE_MAP[entityType]),
+                                                   fieldName)
+        elif entityType in {'versions', 'releases'} and myarg == 'filter':
+            kwargs['filter'] = sys.argv[i + 1]
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromehistory {entityType}"'
+            controlflow.system_error_exit(3, msg)
+    if orderByList:
+        kwargs['orderBy'] = ','.join(orderByList)
+    if entityType == 'platforms':
+        svc = cv.platforms()
+        parent = 'chrome'
+    elif entityType == 'channels':
+        svc = cv.platforms().channels()
+        parent = f'chrome/platforms/{cplatform}'
+    elif entityType == 'versions':
+        svc = cv.platforms().channels().versions()
+        parent = f'chrome/platforms/{cplatform}/channels/{channel}'
+    else: #elif entityType == 'releases'
+        svc = cv.platforms().channels().versions().releases()
+        parent = f'chrome/platforms/{cplatform}/channels/{channel}/versions/{version}'
+    reportTitle = f'Chrome Version History {entityType.capitalize()}'
+    page_message = gapi.got_total_items_msg(reportTitle, '...\n')
+    gam.printGettingAllItems(reportTitle, None)
+    citems = gapi.get_all_pages(svc, 'list', entityType,
+                                page_message=page_message,
+                                parent=parent,
+                                fields=f'nextPageToken,{entityType}',
+                                **kwargs)
+    for citem in citems:
+        for key in list(citem):
+            if key.endswith('Type'):
+                newkey = key[:-4]
+                citem[newkey] = citem.pop(key)
+        if 'channel' in citem:
+            citem['channel'] = citem['channel'].lower()
+        else:
+            channel_match = re.search(r"\/channels\/([^/]*)", citem['name'])
+            if channel_match:
+                try:
+                    citem['channel'] = channel_match.group(1)
+                except IndexError:
+                    pass
+        if 'platform' in citem:
+            citem['platform'] = citem['platform'].lower()
+        else:
+            platform_match = re.search(r"\/platforms\/([^/]*)", citem['name'])
+            if platform_match:
+                try:
+                    citem['platform'] = platform_match.group(1)
+                except IndexError:
+                    pass
+
+        if citem.get('version', '').count('.') == 3:
+            citem['major_version'], \
+            citem['minor_version'], \
+            citem['build'], \
+            citem['patch'] = citem['version'].split('.')
+        citem.pop('name')
+        csvRows.append(utils.flatten_json(citem))
+    display.write_csv_file(csvRows, CHROME_VERSIONHISTORY_TITLES[entityType], reportTitle, todrive)

src/gam/gapi/chromemanagement.py

@@ -0,0 +1,265 @@
+"""Chrome Management API calls"""
+
+import sys
+
+import gam
+from gam.var import GC_CUSTOMER_ID, GC_Values, MY_CUSTOMER
+from gam.var import CROS_START_ARGUMENTS, CROS_END_ARGUMENTS
+from gam.var import YYYYMMDD_FORMAT
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+from gam.gapi.directory.cros import _getFilterDate
+
+
+def _get_customerid():
+    customer = GC_Values[GC_CUSTOMER_ID]
+    if customer != MY_CUSTOMER and customer[0] != 'C':
+        customer = 'C' + customer
+    return f'customers/{customer}'
+
+
+def _get_orgunit(orgunit):
+    if orgunit.startswith('orgunits/'):
+        return orgunit
+    _, orgunitid = gapi_directory_orgunits.getOrgUnitId(orgunit)
+    return f'{orgunitid[3:]}'
+
+
+def build():
+    return gam.buildGAPIObject('chromemanagement')
+
+
+CHROME_APPS_ORDERBY_CHOICE_MAP = {
+  'appname': 'app_name',
+  'apptype': 'appType',
+  'installtype': 'install_type',
+  'numberofpermissions': 'number_of_permissions',
+  'totalinstallcount': 'total_install_count',
+  }
+CHROME_APPS_TITLES = [
+  'appId', 'displayName',
+  'browserDeviceCount', 'osUserCount',
+  'appType', 'description',
+  'appInstallType', 'appSource',
+  'disabled', 'homepageUri',
+  'permissions'
+  ]
+
+def printApps():
+    cm = build()
+    customer = _get_customerid()
+    todrive = False
+    titles = CHROME_APPS_TITLES
+    csvRows = []
+    orgunit = None
+    pfilter = None
+    orderBy = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg == 'filter':
+            pfilter = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i + 1].lower().replace('_', '')
+            if orderBy not in CHROME_APPS_ORDERBY_CHOICE_MAP:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(CHROME_APPS_ORDERBY_CHOICE_MAP),
+                                                   orderBy)
+            orderBy = CHROME_APPS_ORDERBY_CHOICE_MAP[orderBy]
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromeapps"'
+            controlflow.system_error_exit(3, msg)
+    if orgunit:
+        orgUnitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit, None)
+        titles.append('orgUnitPath')
+    else:
+        orgUnitPath = '/'
+    gam.printGettingAllItems('Chrome Installed Applications', pfilter)
+    page_message = gapi.got_total_items_msg('Chrome Installed Applications', '...\n')
+    apps = gapi.get_all_pages(cm.customers().reports(),
+                              'countInstalledApps',
+                              'installedApps',
+                              page_message=page_message,
+                              customer=customer, orgUnitId=orgunit,
+                              filter=pfilter, orderBy=orderBy)
+    for app in apps:
+        if orgunit:
+            app['orgUnitPath'] = orgUnitPath
+        if 'permissions'in app:
+            app['permissions'] = ' '.join(app['permissions'])
+        csvRows.append(app)
+    display.write_csv_file(csvRows, titles, 'Chrome Installed Applications', todrive)
+
+
+CHROME_APP_DEVICES_APPTYPE_CHOICE_MAP = {
+  'extension': 'EXTENSION',
+  'app': 'APP',
+  'theme': 'THEME',
+  'hostedapp': 'HOSTED_APP',
+  'androidapp': 'ANDROID_APP',
+  }
+CHROME_APP_DEVICES_ORDERBY_CHOICE_MAP = {
+  'deviceid': 'deviceId',
+  'machine': 'machine',
+  }
+CHROME_APP_DEVICES_TITLES = [
+  'appId', 'appType', 'deviceId', 'machine'
+  ]
+
+def printAppDevices():
+    cm = build()
+    customer = _get_customerid()
+    todrive = False
+    titles = CHROME_APP_DEVICES_TITLES
+    csvRows = []
+    orgunit = None
+    appId = None
+    appType = None
+    startDate = None
+    endDate = None
+    pfilter = None
+    orderBy = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg == 'appid':
+            appId = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'apptype':
+            appType = sys.argv[i + 1].lower().replace('_', '')
+            if appType not in CHROME_APP_DEVICES_APPTYPE_CHOICE_MAP:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(CHROME_APP_DEVICES_APPTYPE_CHOICE_MAP),
+                                                   appType)
+            appType = CHROME_APP_DEVICES_APPTYPE_CHOICE_MAP[appType]
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1]).strftime(YYYYMMDD_FORMAT)
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1]).strftime(YYYYMMDD_FORMAT)
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i + 1].lower().replace('_', '')
+            if orderBy not in CHROME_APP_DEVICES_ORDERBY_CHOICE_MAP:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(CHROME_APP_DEVICES_ORDERBY_CHOICE_MAP),
+                                                   orderBy)
+            orderBy = CHROME_APP_DEVICES_ORDERBY_CHOICE_MAP[orderBy]
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromeappdevices"'
+            controlflow.system_error_exit(3, msg)
+    if not appId:
+        controlflow.system_error_exit(3, 'You must specify an appid')
+    if not appType:
+        controlflow.system_error_exit(3, 'You must specify an apptype')
+    if endDate:
+        pfilter = f'last_active_date<={endDate}'
+    if startDate:
+        if pfilter:
+            pfilter += ' AND '
+        else:
+            pfilter = ''
+        pfilter += f'last_active_date>={startDate}'
+    if orgunit:
+        orgUnitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit, None)
+        titles.append('orgUnitPath')
+    else:
+        orgUnitPath = '/'
+    gam.printGettingAllItems('Chrome Installed Application Devices', pfilter)
+    page_message = gapi.got_total_items_msg('Chrome Installed Application Devices', '...\n')
+    devices = gapi.get_all_pages(cm.customers().reports(),
+                                 'findInstalledAppDevices',
+                                 'devices',
+                                 page_message=page_message,
+                                 appId=appId, appType=appType,
+                                 customer=customer, orgUnitId=orgunit,
+                                 filter=pfilter, orderBy=orderBy)
+    for device in devices:
+        if orgunit:
+            device['orgUnitPath'] = orgUnitPath
+        device['appId'] = appId
+        device['appType'] = appType
+        csvRows.append(device)
+    display.write_csv_file(csvRows, titles, 'Chrome Installed Application Devices', todrive)
+
+
+CHROME_VERSIONS_TITLES = [
+    'version', 'count', 'channel', 'deviceOsVersion', 'system'
+    ]
+def printVersions():
+    cm = build()
+    customer = _get_customerid()
+    todrive = False
+    titles = CHROME_VERSIONS_TITLES
+    csvRows = []
+    orgunit = None
+    startDate = None
+    endDate = None
+    pfilter = None
+    reverse = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1]).strftime(YYYYMMDD_FORMAT)
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1]).strftime(YYYYMMDD_FORMAT)
+            i += 2
+        elif myarg == 'recentfirst':
+            reverse = True
+            i += 1
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromeversions"'
+            controlflow.system_error_exit(3, msg)
+    if endDate:
+        pfilter = f'last_active_date<={endDate}'
+    if startDate:
+        if pfilter:
+            pfilter += ' AND '
+        else:
+            pfilter = ''
+        pfilter += f'last_active_date>={startDate}'
+    if orgunit:
+        orgUnitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit, None)
+        titles.append('orgUnitPath')
+    else:
+        orgUnitPath = '/'
+    gam.printGettingAllItems('Chrome Versions', pfilter)
+    page_message = gapi.got_total_items_msg('Chrome Versions', '...\n')
+    versions = gapi.get_all_pages(cm.customers().reports(),
+                                  'countChromeVersions',
+                                  'browserVersions',
+                                  page_message=page_message,
+                                  customer=customer, orgUnitId=orgunit, filter=pfilter)
+    for version in sorted(versions, key=lambda k: k.get('version', 'Unknown'), reverse=reverse):
+        if orgunit:
+            version['orgUnitPath'] = orgUnitPath
+        if 'version' not in version:
+            version['version'] = 'Unknown'
+        csvRows.append(version)
+    display.write_csv_file(csvRows, titles, 'Chrome Versions', todrive)

src/gam/gapi/chromepolicy.py

@@ -0,0 +1,382 @@
+"""Chrome Browser Cloud Management API calls"""
+
+import re
+import sys
+
+import googleapiclient.errors
+
+import gam
+from gam.var import GC_CUSTOMER_ID, GC_Values, MY_CUSTOMER
+from gam import controlflow
+from gam import gapi
+from gam.gapi import errors as gapi_errors
+from gam.gapi import chromehistory as gapi_chromehistory
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+from gam import utils
+
+
+def _get_customerid():
+    customer = GC_Values[GC_CUSTOMER_ID]
+    if customer != MY_CUSTOMER and customer[0] != 'C':
+        customer = 'C' + customer
+    return f'customers/{customer}'
+
+
+def _get_orgunit(orgunit):
+    if orgunit.startswith('orgunits/'):
+        return orgunit
+    _, orgunitid = gapi_directory_orgunits.getOrgUnitId(orgunit)
+    return f'orgunits/{orgunitid[3:]}'
+
+
+def build():
+    return gam.buildGAPIObject('chromepolicy')
+
+
+def printshow_policies():
+    svc = build()
+    customer = _get_customerid()
+    orgunit = None
+    printer_id = None
+    app_id = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg == 'printerid':
+            printer_id = sys.argv[i+1]
+            i += 2
+        elif myarg == 'appid':
+            app_id = sys.argv[i+1]
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromepolicy"'
+            controlflow.system_error_exit(3, msg)
+    if not orgunit:
+        controlflow.system_error_exit(3, 'You must specify an orgunit')
+    body = {
+             'policyTargetKey': {
+               'targetResource': orgunit,
+             }
+           }
+    if printer_id:
+        body['policyTargetKey']['additionalTargetKeys'] = {'printer_id': printer_id}
+        namespaces = ['chrome.printers']
+    elif app_id:
+        body['policyTargetKey']['additionalTargetKeys'] = {'app_id': app_id}
+        namespaces = ['chrome.users.apps',
+                      'chrome.devices.managedGuest.apps',
+                      'chrome.devices.kiosk.apps']
+    else:
+        namespaces = [
+            'chrome.users',
+#           Not yet implemented:
+#           'chrome.devices',
+#           'chrome.devices.managedGuest',
+#           'chrome.devices.kiosk',
+            ]
+    throw_reasons = [gapi_errors.ErrorReason.FOUR_O_O,]
+    orgunitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit[9:], None)
+    header = f'Organizational Unit: {orgunitPath}'
+    if printer_id:
+        header += f', printerid: {printer_id}'
+    elif app_id:
+        header += f', appid: {app_id}'
+    print(header)
+    for namespace in namespaces:
+        body['policySchemaFilter'] = f'{namespace}.*'
+        try:
+            policies = gapi.get_all_pages(svc.customers().policies(), 'resolve',
+                                          items='resolvedPolicies',
+                                          throw_reasons=throw_reasons,
+                                          customer=customer,
+                                          body=body)
+        except googleapiclient.errors.HttpError:
+            policies = []
+        for policy in sorted(policies, key=lambda k: k.get('value', {}).get('policySchema', '')):
+            print()
+            name = policy.get('value', {}).get('policySchema', '')
+            schema = CHROME_SCHEMA_TYPE_MESSAGE.get(name)
+            print(name)
+            values = policy.get('value', {}).get('value', {})
+            for setting, value in values.items():
+                # Handle TYPE_MESSAGE fields with durations or counts as a special case
+                if schema and setting == schema['casedField']:
+                  value = value.get(schema['type'], '')
+                  if value:
+                      if value.endswith('s'):
+                          value = value[:-1]
+                      value = int(value) // schema['scale']
+                elif isinstance(value, str) and value.find('_ENUM_') != -1:
+                    value = value.split('_ENUM_')[-1]
+                print(f'  {setting}: {value}')
+
+
+def build_schemas(svc=None, sfilter=None):
+    if not svc:
+        svc = build()
+    parent = _get_customerid()
+    schemas = gapi.get_all_pages(svc.customers().policySchemas(), 'list',
+            items='policySchemas', parent=parent, filter=sfilter)
+    schema_objects = {}
+    for schema in schemas:
+        schema_name = schema.get('name', '').split('/')[-1]
+        schema_dict = {
+                'name': schema_name,
+                'description': schema.get('policyDescription', ''),
+                'settings': {},
+                }
+        field_descriptions = schema.get('fieldDescriptions', [])
+        for mtype in schema.get('definition', {}).get('messageType', {}):
+            for setting in mtype.get('field', {}):
+                setting_name = setting.get('name', '')
+                setting_dict = {
+                                 'name': setting_name,
+                                 'constraints': None,
+                                 'descriptions':  [],
+                                 'type': setting.get('type'),
+                               }
+                if setting_dict['type'] == 'TYPE_STRING' and \
+                   setting.get('label') == 'LABEL_REPEATED':
+                    setting_dict['type'] = 'TYPE_LIST'
+                if setting_dict['type'] == 'TYPE_ENUM':
+                    type_name = setting['typeName']
+                    for an_enum in schema['definition']['enumType']:
+                        if an_enum['name'] == type_name:
+                            setting_dict['enums'] = [enum['name'] for enum in an_enum['value']]
+                            setting_dict['enum_prefix'] = utils.commonprefix(setting_dict['enums'])
+                            prefix_len = len(setting_dict['enum_prefix'])
+                            setting_dict['enums'] = [enum[prefix_len:] for enum \
+                                                     in setting_dict['enums'] \
+                                                     if not enum.endswith('UNSPECIFIED')]
+                            setting_dict['descriptions'] = ['']*len(setting_dict['enums'])
+                            if field_descriptions:
+                                for i, an in enumerate(setting_dict['enums']):
+                                    for fdesc in field_descriptions:
+                                      if fdesc.get('field') == setting_name:
+                                          for d in fdesc.get('knownValueDescriptions', []):
+                                              if d['value'][prefix_len:] == an:
+                                                  setting_dict['descriptions'][i] = d['description']
+                                                  break
+                                          break
+                            break
+                elif setting_dict['type'] == 'TYPE_MESSAGE':
+                    continue
+                else:
+                    setting_dict['enums'] = None
+                    for fdesc in schema.get('fieldDescriptions', []):
+                        if fdesc.get('field') == setting_name:
+                            if 'knownValueDescriptions' in fdesc:
+                                setting_dict['descriptions'] = fdesc['knownValueDescriptions']
+                            elif 'description' in fdesc:
+                                setting_dict['descriptions'] = [fdesc['description']]
+                schema_dict['settings'][setting_name.lower()] = setting_dict
+        schema_objects[schema_name.lower()] = schema_dict
+    return schema_objects
+
+
+def printshow_schemas():
+    svc = build()
+    sfilter = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'filter':
+            sfilter = sys.argv[i+1]
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromeschema"'
+            controlflow.system_error_exit(3, msg)
+    schemas = build_schemas(svc, sfilter)
+    for _, value in sorted(iter(schemas.items())):
+        print(f'{value.get("name")}: {value.get("description")}')
+        for val in value['settings'].values():
+            vtype = val.get('type')
+            print(f'  {val.get("name")}: {vtype}')
+            if vtype == 'TYPE_ENUM':
+                enums = val.get('enums', [])
+                descriptions = val.get('descriptions', [])
+                for i in range(len(val.get('enums', []))):
+                    print(f'    {enums[i]}: {descriptions[i]}')
+            elif vtype == 'TYPE_BOOL':
+                pvs = val.get('descriptions')
+                for pvi in pvs:
+                    if isinstance(pvi, dict):
+                        pvalue = pvi.get('value')
+                        pdescription = pvi.get('description')
+                        print(f'    {pvalue}: {pdescription}')
+                    elif isinstance(pvi, list):
+                        print(f'    {pvi[0]}')
+            else:
+                description = val.get('descriptions')
+                if len(description) > 0:
+                    print(f'    {description[0]}')
+        print()
+
+
+def delete_policy():
+    svc = build()
+    customer = _get_customerid()
+    schemas = build_schemas(svc)
+    orgunit = None
+    printer_id = None
+    app_id = None
+    i = 3
+    body = {'requests': []}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg == 'printerid':
+            printer_id = sys.argv[i+1]
+            i += 2
+        elif myarg == 'appid':
+            app_id = sys.argv[i+1]
+            i += 2
+        elif myarg in schemas:
+            body['requests'].append({'policySchema': schemas[myarg]['name']})
+            i += 1
+        else:
+            msg = f'{myarg} is not a valid argument to "gam delete chromepolicy"'
+            controlflow.system_error_exit(3, msg)
+    if not orgunit:
+        controlflow.system_error_exit(3, 'You must specify an orgunit')
+    for request in body['requests']:
+        request['policyTargetKey'] = {'targetResource': orgunit}
+        if printer_id:
+            request['policyTargetKey']['additionalTargetKeys'] = {'printer_id': printer_id}
+        elif app_id:
+            request['policyTargetKey']['additionalTargetKeys'] = {'app_id': app_id}
+    gapi.call(svc.customers().policies().orgunits(), 'batchInherit', customer=customer, body=body)
+
+
+CHROME_SCHEMA_TYPE_MESSAGE = {
+  'chrome.users.SessionLength':
+    {'field': 'sessiondurationlimit', 'casedField': 'sessionDurationLimit',
+     'type': 'duration', 'minVal': 1, 'maxVal': 1440, 'scale': 60},
+  'chrome.users.BrowserSwitcherDelayDuration':
+    {'field': 'browserswitcherdelayduration', 'casedField': 'browserSwitcherDelayDuration',
+     'type': 'duration', 'minVal': 0, 'maxVal': 30, 'scale': 1},
+  'chrome.users.MaxInvalidationFetchDelay':
+    {'field': 'maxinvalidationfetchdelay', 'casedField': 'maxInvalidationFetchDelay',
+     'type': 'duration', 'minVal': 1, 'maxVal': 30, 'scale': 1},
+  'chrome.users.SecurityTokenSessionSettings':
+    {'field': 'securitytokensessionnotificationseconds', 'casedField': 'securityTokenSessionNotificationSeconds',
+     'type': 'duration', 'minVal': 0, 'maxVal': 9999, 'scale': 1},
+  'chrome.users.PrintingMaxSheetsAllowed':
+    {'field': 'printingmaxsheetsallowednullable', 'casedField': 'printingMaxSheetsAllowedNullable',
+     'type': 'value', 'minVal': 1, 'maxVal': None, 'scale': 1},
+  }
+
+
+def update_policy():
+    svc = build()
+    customer = _get_customerid()
+    schemas = build_schemas(svc)
+    orgunit = None
+    printer_id = None
+    app_id = None
+    i = 3
+    body = {'requests': []}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg == 'printerid':
+            printer_id = sys.argv[i+1]
+            i += 2
+        elif myarg == 'appid':
+            app_id = sys.argv[i+1]
+            i += 2
+        elif myarg in schemas:
+            schemaName = schemas[myarg]['name']
+            body['requests'].append({'policyValue': {'policySchema': schemaName,
+                                                     'value': {}},
+                                     'updateMask': ''})
+            i += 1
+            while i < len(sys.argv):
+                field = sys.argv[i].lower()
+                if field in ['ou', 'org', 'orgunit', 'printerid', 'appid'] or '.' in field:
+                    break # field is actually a new policy, orgunit or app/printer id
+                # Handle TYPE_MESSAGE fields with durations or counts as a special case
+                schema = CHROME_SCHEMA_TYPE_MESSAGE.get(schemaName)
+                if schema and field == schema['field']:
+                  casedField = schema['casedField']
+                  value = gam.getInteger(sys.argv[i+1], casedField,
+                                         minVal=schema['minVal'], maxVal=schema['maxVal'])*schema['scale']
+                  if schema['type'] == 'duration':
+                    body['requests'][-1]['policyValue']['value'][casedField] = {schema['type']: f'{value}s'}
+                  else:
+                    body['requests'][-1]['policyValue']['value'][casedField] = {schema['type']: value}
+                  body['requests'][-1]['updateMask'] += f'{casedField},'
+                  i += 2
+                  continue
+                expected_fields = ', '.join(schemas[myarg]['settings'])
+                if field not in expected_fields:
+                    msg = f'Expected {myarg} field of {expected_fields}. Got {field}.'
+                    controlflow.system_error_exit(4, msg)
+                cased_field = schemas[myarg]['settings'][field]['name']
+                value = sys.argv[i+1]
+                vtype = schemas[myarg]['settings'][field]['type']
+                if vtype in ['TYPE_INT64', 'TYPE_INT32', 'TYPE_UINT64']:
+                    if not value.isnumeric():
+                        msg = f'Value for {myarg} {field} must be a number, got {value}'
+                        controlflow.system_error_exit(7, msg)
+                    value = int(value)
+                elif vtype in ['TYPE_BOOL']:
+                    value = gam.getBoolean(value, field)
+                elif vtype in ['TYPE_ENUM']:
+                    value = value.upper()
+                    prefix = schemas[myarg]['settings'][field]['enum_prefix']
+                    enum_values = schemas[myarg]['settings'][field]['enums']
+                    if value in enum_values:
+                        value = f'{prefix}{value}'
+                    elif value.replace(prefix, '') in enum_values:
+                        pass
+                    else:
+                        expected_enums = ', '.join(enum_values)
+                        msg = f'Expected {myarg} {field} value to be one of ' \
+                              f'{expected_enums}, got {value}'
+                        controlflow.system_error_exit(8, msg)
+                elif vtype in ['TYPE_LIST']:
+                    value = value.split(',')
+                if myarg == 'chrome.users.chromebrowserupdates' and \
+                      cased_field == 'targetVersionPrefixSetting':
+                    mg = re.compile(r'^([a-z]+)-(\d+)$').match(value)
+                    if mg:
+                        channel = mg.group(1).lower().replace('_', '')
+                        minus = mg.group(2)
+                        channel_map = gapi_chromehistory.get_channel_map(None)
+                        if channel not in channel_map:
+                            expected_channels = ', '.join(channel_map)
+                            msg = f'Expected {myarg} {cased_field} channel to be one of ' \
+                                f'{expected_channels}, got {channel}'
+                            controlflow.system_error_exit(8, msg)
+                        milestone = gapi_chromehistory.get_relative_milestone(
+                            channel_map[channel], int(minus))
+                        if not milestone:
+                            msg = f'{myarg} {cased_field} channel {channel} offset {minus} does not exist'
+                            controlflow.system_error_exit(8, msg)
+                        value = f'{milestone}.'
+                body['requests'][-1]['policyValue']['value'][cased_field] = value
+                body['requests'][-1]['updateMask'] += f'{cased_field},'
+                i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam update chromepolicy"'
+            controlflow.system_error_exit(4, msg)
+    if not orgunit:
+        controlflow.system_error_exit(3, 'You must specify an orgunit')
+    for request in body['requests']:
+        request['policyTargetKey'] = {'targetResource': orgunit}
+        if printer_id:
+            request['policyTargetKey']['additionalTargetKeys'] = {'printer_id': printer_id}
+        elif app_id:
+            request['policyTargetKey']['additionalTargetKeys'] = {'app_id': app_id}
+    gapi.call(svc.customers().policies().orgunits(),
+              'batchModify',
+              customer=customer,
+              body=body)

src/gam/gapi/cloudidentity/__init__.py

@@ -0,0 +1,9 @@
+import gam
+
+
+def build(api='cloudidentity'):
+    return gam.buildGAPIObject(api)
+
+def build_dwd(api='cloudidentity'):
+    admin = gam._get_admin_email()
+    return gam.buildGAPIServiceObject(api, admin, True)

src/gam/gapi/cloudidentity/devices.py

@@ -0,0 +1,462 @@
+import csv
+import sys
+
+import googleapiclient
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
+from gam.gapi import errors as gapi_errors
+from gam.gapi import cloudidentity as gapi_cloudidentity
+from gam.gapi.directory import customer as gapi_directory_customer
+
+def _get_device_customerid():
+  customer = GC_Values[GC_CUSTOMER_ID]
+  if customer.startswith('C'):
+    customer = customer[1:]
+  return f'customers/{customer}'
+
+def create():
+    ci = gapi_cloudidentity.build_dwd()
+    customer = _get_device_customerid()
+    device_types = gapi.get_enum_values_minus_unspecified(
+        ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1Device']['properties']['deviceType']['enum'])
+    body = {'deviceType': '', 'serialNumber': ''}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'serialnumber':
+            body['serialNumber'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'devicetype':
+            body['deviceType'] = sys.argv[i+1].upper()
+            if body['deviceType'] not in device_types:
+                controlflow.expected_argument_exit('device_type',
+                                                   ', '.join(device_types),
+                                                   sys.argv[i+1])
+            i += 2
+        elif myarg in {'assettag', 'assetid'}:
+            body['assetTag'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create device')
+    if not body['serialNumber'] or not body['deviceType']:
+        controlflow.system_error_exit(
+            3, 'serial_number and device_type are required arguments for "gam create device".')
+    result = gapi.call(ci.devices(), 'create', customer=customer, body=body)
+    print(f'Created device {result["response"]["name"]}')
+
+
+def _get_device_name():
+    name = sys.argv[3]
+    if name == 'id':
+        name = sys.argv[4]
+    if not name.startswith('devices/'):
+        name = f'devices/{name}'
+    return name
+
+
+def info():
+    ci = gapi_cloudidentity.build_dwd()
+    customer = _get_device_customerid()
+    name = _get_device_name()
+    device = gapi.call(ci.devices(), 'get', name=name, customer=customer)
+    device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
+        'deviceUsers', parent=name, customer=customer)
+    for device_user in device_users:
+        parent = device_user['name']
+        device_user['client_states'] = gapi.get_all_pages(
+                  ci.devices().deviceUsers().clientStates(),
+                  'list', 'clientStates', parent=parent, customer=customer)
+    display.print_json(device)
+    print('Device Users:')
+    display.print_json(device_users)
+
+
+def _generic_action(action, device_user=False):
+    ci = gapi_cloudidentity.build_dwd()
+    customer = _get_device_customerid()
+    name = _get_device_name()
+
+    # bah, inconsistencies in API
+    if action == 'delete':
+        kwargs = {'customer': customer}
+    else:
+        kwargs = {'body': {'customer': customer}}
+
+    if device_user:
+        endpoint = ci.devices().deviceUsers()
+    else:
+        endpoint = ci.devices()
+    op = gapi.call(endpoint, action, name=name, **kwargs)
+    print(op)
+
+
+def delete():
+    _generic_action('delete')
+
+
+def cancel_wipe():
+    _generic_action('cancelWipe')
+
+
+def wipe():
+    _generic_action('wipe')
+
+
+def approve_user():
+    _generic_action('approve', True)
+
+
+def block_user():
+    _generic_action('block', True)
+
+
+def cancel_wipe_user():
+    _generic_action('cancelWipe', True)
+
+
+def delete_user():
+    _generic_action('delete', True)
+
+
+def wipe_user():
+    _generic_action('wipe', True)
+
+
+def _get_deviceuser_name():
+    i = 3
+    name = sys.argv[i]
+    if name == 'id':
+        i += 1
+        name = sys.argv[i]
+    if not name.startswith('devices/'):
+        name = f'devices/{name}'
+    return (i+1, name)
+
+def info_state():
+    ci = gapi_cloudidentity.build_dwd()
+    gapi_directory_customer.setTrueCustomerId()
+    customer = _get_device_customerid()
+    customer_id = customer[10:]
+    client_id = f'{customer_id}-gam'
+    i, deviceuser = _get_deviceuser_name()
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'clientid':
+            client_id = f'{customer_id}-{sys.argv[i+1]}'
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam info deviceuserstate')
+    name = f'{deviceuser}/clientStates/{client_id}'
+    result = gapi.call(ci.devices().deviceUsers().clientStates(), 'get',
+                       name=name, customer=customer)
+    display.print_json(result)
+
+
+def update_state():
+    ci = gapi_cloudidentity.build_dwd()
+    gapi_directory_customer.setTrueCustomerId()
+    customer = _get_device_customerid()
+    customer_id = customer[10:]
+    client_id = f'{customer_id}-gam'
+    body = {}
+    i, deviceuser = _get_deviceuser_name()
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'clientid':
+            client_id = f'{customer_id}-{sys.argv[i+1]}'
+            i += 2
+        elif myarg in ['assettag', 'assettags']:
+            body['assetTags'] = gam.shlexSplitList(sys.argv[i+1])
+            if body['assetTags'] == ['clear']:
+                # TODO: this doesn't work to clear
+                # existing values. Figure out why.
+                body['assetTags'] = [None]
+            i += 2
+        elif myarg in ['compliantstate', 'compliancestate']:
+            comp_states = gapi.get_enum_values_minus_unspecified(
+              ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1ClientState']['properties']['complianceState']['enum'])
+            body['complianceState'] = sys.argv[i+1].upper()
+            if body['complianceState'] not in comp_states:
+                controlflow.expected_argument_exit('compliant_state',
+                                                   ', '.join(comp_states),
+                                                   sys.argv[i+1])
+            i += 2
+        elif myarg == 'customid':
+            body['customId'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'healthscore':
+            health_scores = gapi.get_enum_values_minus_unspecified(
+            ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1ClientState']['properties']['healthScore']['enum'])
+            body['healthScore'] = sys.argv[i+1].upper()
+            if body['healthScore'] == 'CLEAR':
+                body['healthScore'] = None
+            if body['healthScore'] and body['healthScore'] not in health_scores:
+                controlflow.expected_argument_exit('health_score',
+                                                   ', '.join(health_scores),
+                                                   sys.argv[i+1])
+            i += 2
+        elif myarg == 'customvalue':
+            allowed_types = ['bool', 'number', 'string']
+            value_type = sys.argv[i+1].lower()
+            if value_type not in allowed_types:
+                controlflow.expected_argument_exit('custom_value',
+                                                   ', '.join(allowed_types),
+                                                   sys.argv[i+1])
+            key = sys.argv[i+2]
+            value = sys.argv[i+3]
+            if value_type == 'bool':
+                value = gam.getBoolean(value, key)
+            elif value_type == 'number':
+                value = int(value)
+            body.setdefault('keyValuePairs', {})
+            body['keyValuePairs'][key] = {f'{value_type}Value': value}
+            i += 4
+        elif myarg in ['managedstate']:
+            managed_states = gapi.get_enum_values_minus_unspecified(
+              ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1ClientState']['properties']['managed']['enum'])
+            body['managed'] = sys.argv[i+1].upper()
+            if body['managed'] == 'CLEAR':
+                body['managed'] = None
+            if body['managed'] and body['managed'] not in managed_states:
+                controlflow.expected_argument_exit('managed_state',
+                                                   ', '.join(managed_states),
+                                                   sys.argv[i+1])
+            i += 2
+        elif myarg in ['scorereason']:
+            body['scoreReason'] = sys.argv[i+1]
+            if body['scoreReason'] == 'clear':
+                body['scoreReason'] = None
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update deviceuserstate')
+    name = f'{deviceuser}/clientStates/{client_id}'
+    updateMask = ','.join(body.keys())
+    result = gapi.call(ci.devices().deviceUsers().clientStates(), 'patch',
+                       name=name, customer=customer, updateMask=updateMask, body=body)
+    display.print_json(result)
+
+
+def print_():
+    ci = gapi_cloudidentity.build_dwd()
+    customer = _get_device_customerid()
+    parent = 'devices/-'
+    device_filter = None
+    get_device_users = True
+    view = None
+    orderByList = []
+    titles = []
+    csvRows = []
+    todrive = False
+    sortHeaders = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['filter', 'query']:
+            device_filter = sys.argv[i+1]
+            i += 2
+        elif myarg == 'company':
+            view = 'COMPANY_INVENTORY'
+            i += 1
+        elif myarg == 'personal':
+            view = 'USER_ASSIGNED_DEVICES'
+            i += 1
+        elif myarg == 'nocompanydevices':
+            view = 'USER_ASSIGNED_DEVICES'
+            i += 1
+        elif myarg == 'nopersonaldevices':
+            view = 'COMPANY_INVENTORY'
+            i += 1
+        elif myarg == 'nodeviceusers':
+            get_device_users = False
+            i += 1
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'orderby':
+            fieldName = sys.argv[i + 1].lower()
+            i += 2
+            if fieldName in DEVICE_ORDERBY_CHOICES_MAP:
+                fieldName = DEVICE_ORDERBY_CHOICES_MAP[fieldName]
+                orderBy = ''
+                if i < len(sys.argv):
+                    orderBy = sys.argv[i].lower()
+                    if orderBy in SORTORDER_CHOICES_MAP:
+                        orderBy = SORTORDER_CHOICES_MAP[orderBy]
+                        i += 1
+                if orderBy != 'DESCENDING':
+                    orderByList.append(fieldName)
+                else:
+                    orderByList.append(f'{fieldName} desc')
+            else:
+                controlflow.expected_argument_exit(
+                    'orderby', ', '.join(sorted(DEVICE_ORDERBY_CHOICES_MAP)),
+                    fieldName)
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print devices')
+    view_name_map = {
+      None: 'Devices',
+      'COMPANY_INVENTORY': 'Company Devices',
+      'USER_ASSIGNED_DEVICES': 'Personal Devices',
+      }
+    if orderByList:
+        orderBy = ','.join(orderByList)
+    else:
+        orderBy = None
+    devices = []
+    page_message = gapi.got_total_items_msg(view_name_map[view], '...\n')
+    devices += gapi.get_all_pages(ci.devices(), 'list', 'devices',
+        customer=customer, page_message=page_message,
+        pageSize=100, filter=device_filter, view=view, orderBy=orderBy)
+    if get_device_users:
+        page_message = gapi.got_total_items_msg('Device Users', '...\n')
+        device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
+            'deviceUsers', customer=customer, parent=parent,
+            page_message=page_message, pageSize=20, filter=device_filter)
+        for device_user in device_users:
+            for device in devices:
+                if device_user.get('name').startswith(device.get('name')):
+                    if 'users' not in device:
+                        device['users'] = []
+                    device['users'].append(device_user)
+                    break
+    for device in devices:
+        device = utils.flatten_json(device)
+        for a_key in device:
+            if a_key not in titles:
+                titles.append(a_key)
+        csvRows.append(device)
+    if sortHeaders:
+        display.sort_csv_titles(['name',], titles)
+    display.write_csv_file(csvRows, titles, 'Devices', todrive)
+
+
+def sync():
+    ci = gapi_cloudidentity.build_dwd()
+    device_types = gapi.get_enum_values_minus_unspecified(
+        ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1Device']['properties']['deviceType']['enum'])
+    customer = _get_device_customerid()
+    device_filter = None
+    csv_file = None
+    serialnumber_column = 'serialNumber'
+    devicetype_column = 'deviceType'
+    static_devicetype = None
+    assettag_column = None
+    unassigned_missing_action = 'delete'
+    assigned_missing_action = 'donothing'
+    missing_actions = ['delete', 'wipe', 'donothing']
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['filter', 'query']:
+          device_filter = sys.argv[i+1]
+          i += 2
+        elif myarg == 'csvfile':
+          csv_file = sys.argv[i+1]
+          i += 2
+        elif myarg == 'serialnumbercolumn':
+          serialnumber_column = sys.argv[i+1]
+          i += 2
+        elif myarg == 'devicetypecolumn':
+          devicetype_column = sys.argv[i+1]
+          i += 2
+        elif myarg == 'staticdevicetype':
+          static_devicetype = sys.argv[i+1].upper()
+          if static_devicetype not in device_types:
+                controlflow.expected_argument_exit('device_type',
+                                                   ', '.join(device_types),
+                                                   sys.argv[i+1])
+          i += 2
+        elif myarg in {'assettagcolumn', 'assetidcolumn'}:
+          assettag_column = sys.argv[i+1]
+          i += 2
+        elif myarg == 'unassignedmissingaction':
+          unassigned_missing_action = sys.argv[i+1].lower().replace('_', '')
+          if unassigned_missing_action not in missing_actions:
+              controlflow.expected_argument_exit('unassigned_missing_action',
+                                                   ', '.join(missing_actions),
+                                                   sys.argv[i+1])
+          i += 2
+        elif myarg == 'assignedmissingaction':
+          assigned_missing_action = sys.argv[i+1].lower().replace('_', '')
+          if assigned_missing_action not in missing_actions:
+              controlflow.expected_argument_exit('assigned_missing_action',
+                                                   ', '.join(missing_actions),
+                                                   sys.argv[i+1])
+          i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam sync devices')
+    if not csv_file:
+        controlflow.system_error_exit(
+            3, 'csvfile is a required argument for "gam sync devices".')
+    f = fileutils.open_file(csv_file)
+    input_file = csv.DictReader(f, restval='')
+    if serialnumber_column not in input_file.fieldnames:
+        controlflow.csv_field_error_exit(serialnumber_column, input_file.fieldnames)
+    if not static_devicetype and devicetype_column not in input_file.fieldnames:
+        controlflow.csv_field_error_exit(devicetype_column, input_file.fieldnames)
+    if assettag_column and assettag_column not in input_file.fieldnames:
+        controlflow.csv_field_error_exit(assettag_column, input_file.fieldnames)
+    local_devices = []
+    for row in input_file:
+        # upper() is very important to comparison since Google
+        # always return uppercase serials
+        local_device = {'serialNumber': row[serialnumber_column].strip().upper()}
+        if static_devicetype:
+            local_device['deviceType'] = static_devicetype
+        else:
+            local_device['deviceType'] = row[devicetype_column].strip()
+        if assettag_column:
+            local_device['assetTag'] = row[assettag_column].strip()
+        local_devices.append(local_device)
+    fileutils.close_file(f)
+    page_message = gapi.got_total_items_msg('Company Devices', '...\n')
+    device_fields = ['serialNumber', 'deviceType', 'lastSyncTime', 'name']
+    if assettag_column:
+       device_fields.append('assetTag')
+    fields = f'nextPageToken,devices({",".join(device_fields)})'
+    remote_devices = gapi.get_all_pages(ci.devices(), 'list', 'devices',
+            customer=customer, page_message=page_message,
+            pageSize=100, filter=device_filter, view='COMPANY_INVENTORY', fields=fields)
+    remote_device_map = {}
+    for remote_device in remote_devices:
+        sn = remote_device['serialNumber']
+        last_sync = remote_device.pop('lastSyncTime', NEVER_TIME_NOMS)
+        name = remote_device.pop('name')
+        remote_device_map[sn] = {'name': name}
+        if last_sync == NEVER_TIME_NOMS:
+            remote_device_map[sn]['unassigned'] = True
+    devices_to_add = [device for device in local_devices if device not in remote_devices]
+    missing_devices = [device for device in remote_devices if device not in local_devices]
+    print(f'Need to add {len(devices_to_add)} and remove {len(missing_devices)} devices...')
+    for add_device in devices_to_add:
+        print(f'Creating {add_device["serialNumber"]}')
+        try:
+            result = gapi.call(ci.devices(), 'create', customer=customer,
+              throw_reasons=[gapi_errors.ErrorReason.FOUR_O_NINE], body=add_device)
+            print(f' created {result["response"]["deviceType"]} device {result["response"]["name"]} with serial {result["response"]["serialNumber"]}')
+        except googleapiclient.errors.HttpError:
+            print(f' {add_device["serialNumber"]} already exists')
+    for missing_device in missing_devices:
+        sn = missing_device['serialNumber']
+        name = remote_device_map[sn]['name']
+        unassigned = remote_device_map[sn].get('unassigned')
+        action = unassigned_missing_action if unassigned else assigned_missing_action
+        if action == 'donothing':
+            pass
+        else:
+            if action == 'delete':
+                kwargs = {'customer': customer}
+            else:
+                kwargs = {'body': {'customer': customer}}
+            gapi.call(ci.devices(), action,
+                name=name, **kwargs)
+            print(f'{action}d {sn}')

src/gam/gapi/cloudidentity/groups.py

@@ -0,0 +1,902 @@
+import sys
+
+import googleapiclient
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam import utils
+from gam.gapi import errors as gapi_errors
+from gam.gapi import cloudidentity as gapi_cloudidentity
+from gam.gapi.directory import customer as gapi_directory_customer
+
+
+def build():
+    return gapi_cloudidentity.build('cloudidentity')
+
+
+def create():
+    ci = build()
+    initialGroupConfig = 'EMPTY'
+    gapi_directory_customer.setTrueCustomerId()
+    parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    body = {
+        'groupKey': {
+            'id': gam.normalizeEmailAddressOrUID(sys.argv[3], noUid=True)
+        },
+        'parent': parent,
+        'labels': {
+            'cloudidentity.googleapis.com/groups.discussion_forum': ''
+        },
+    }
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['displayName'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['alias', 'aliases']:
+            # As of 2020/06/25 this doesn't work (yet?)
+            aliases = sys.argv[i + 1].split(' ')
+            body['additionalGroupKeys'] = []
+            for alias in aliases:
+                body['additionalGroupKeys'].append({'id': alias})
+            i += 2
+        elif myarg in ['dynamic']:
+            # As of 2020/06/25 this doesn't work (yet?)
+            body['dynamicGroupMetadata'] = {
+                'queries': [{
+                    'query': sys.argv[i + 1],
+                    'resourceType': 'USER'
+                }]
+            }
+            i += 2
+        elif myarg in ['makeowner']:
+            initialGroupConfig = 'WITH_INITIAL_OWNER'
+            i += 1
+        else:
+            print('should not get here')
+            sys.exit(5)
+    print(f'Creating group {body["groupKey"]["id"]}')
+    gapi.call(ci.groups(),
+              'create',
+              initialGroupConfig=initialGroupConfig,
+              body=body)
+
+
+def delete():
+    ci = build()
+    group = sys.argv[3]
+    name = group_email_to_id(ci, group)
+    print(f'Deleting group {group}')
+    gapi.call(ci.groups(), 'delete', name=name)
+
+
+def info():
+    ci = build()
+    group = gam.normalizeEmailAddressOrUID(sys.argv[3])
+    getUsers = True
+    showJoinDate = True
+    showUpdateDate = False
+    showMemberTree = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nousers':
+            getUsers = False
+            i += 1
+        elif myarg == 'nojoindate':
+            showJoinDate = False
+            i += 1
+        elif myarg == 'showupdatedate':
+            showUpdateDate = True
+            i += 1
+        elif myarg == 'membertree':
+            showMemberTree = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam info cigroup')
+    name = group_email_to_id(ci, group)
+    basic_info = gapi.call(ci.groups(), 'get', name=name)
+    display.print_json(basic_info)
+    if getUsers and not showMemberTree:
+        if not showJoinDate and not showUpdateDate:
+            view = 'BASIC'
+            pageSize = 1000
+        else:
+            view = 'FULL'
+            pageSize = 500
+        members = gapi.get_all_pages(ci.groups().memberships(),
+                                     'list',
+                                     'memberships',
+                                     parent=name,
+                                     fields='*',
+                                     pageSize=pageSize,
+                                     view=view)
+        print('Members:')
+        for member in members:
+            role = get_single_role(member.get('roles', [])).lower()
+            email = member.get('memberKey', {}).get('id')
+            jc_string = ''
+            if showJoinDate:
+                joined = member.get('createTime', 'Unknown')
+                jc_string += f'  joined {joined}'
+            if showUpdateDate:
+                updated = member.get('updateTime', 'Unknown')
+                jc_string += f'  updated {updated}'
+            print(
+                f'{role}: {email}{jc_string}'
+                # f' {member.get("role", ROLE_MEMBER).lower()}: {member.get("email", member["id"])} ({member["type"].lower()})'
+            )
+        print(f'Total {len(members)} users in group')
+    elif showMemberTree:
+        print(' Member tree:')
+        global cached_group_members
+        cached_group_members = {}
+        print_member_tree(ci, name)
+
+
+def print_member_tree(ci, group_id, spaces=2):
+    if not group_id in cached_group_members:
+        cached_group_members[group_id] = gapi.get_all_pages(ci.groups().memberships(),
+                                                            'list',
+                                                            'memberships',
+                                                            parent=group_id,
+                                                            fields='*',
+                                                            pageSize=1000)
+    for member in cached_group_members[group_id]:
+        member_id = member.get('name', '')
+        member_id = member_id.split('/')[-1]
+        if member_id.isdigit():
+            member_type = 'user'
+        else:
+            member_type = 'group'
+        member_email = member.get('preferredMemberKey', {}).get('id')
+        relation_type = member.get('relationType', '').lower()
+        if member_type == 'user':
+            print(f'{" " * spaces}{member_email} - user')
+        elif member_type == 'group':
+            print(f'{" " * spaces}{member_email} - group')
+            group_id = group_email_to_id(ci, member_email)
+            print_member_tree(ci, group_id, spaces + 2)
+        else:
+            print(f'unknown member type: {member_type} for {member_email}')
+
+
+def info_member():
+    ci = build()
+    member = gam.normalizeEmailAddressOrUID(sys.argv[3])
+    group = gam.normalizeEmailAddressOrUID(sys.argv[4])
+    group_name = gapi.call(ci.groups(),
+                           'lookup',
+                           groupKey_id=group,
+                           fields='name').get('name')
+    member_name = gapi.call(ci.groups().memberships(),
+                            'lookup',
+                            parent=group_name,
+                            memberKey_id=member,
+                            fields='name').get('name')
+    member_details = gapi.call(ci.groups().memberships(),
+                               'get',
+                               name=member_name)
+    display.print_json(member_details)
+
+
+UPDATE_GROUP_SUBCMDS = ['add', 'clear', 'delete', 'remove', 'sync', 'update']
+GROUP_ROLES_MAP = {
+    'owner': ROLE_OWNER,
+    'owners': ROLE_OWNER,
+    'manager': ROLE_MANAGER,
+    'managers': ROLE_MANAGER,
+    'member': ROLE_MEMBER,
+    'members': ROLE_MEMBER,
+}
+
+
+def print_():
+    ci = build()
+    i = 3
+    members = membersCountOnly = managers = managersCountOnly = owners = ownersCountOnly = False
+    gapi_directory_customer.setTrueCustomerId()
+    parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    usemember = None
+    memberDelimiter = '\n'
+    todrive = False
+    titles = []
+    csvRows = []
+    roles = []
+    sortHeaders = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'enterprisemember':
+            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
+            i += 2
+        elif myarg == 'delimiter':
+            memberDelimiter = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in ['members', 'memberscount']:
+            roles.append(ROLE_MEMBER)
+            members = True
+            if myarg == 'memberscount':
+                membersCountOnly = True
+            i += 1
+        elif myarg in ['owners', 'ownerscount']:
+            roles.append(ROLE_OWNER)
+            owners = True
+            if myarg == 'ownerscount':
+                ownersCountOnly = True
+            i += 1
+        elif myarg in ['managers', 'managerscount']:
+            roles.append(ROLE_MANAGER)
+            managers = True
+            if myarg == 'managerscount':
+                managersCountOnly = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print cigroups')
+    if roles:
+        if members:
+            display.add_titles_to_csv_file([
+                'MembersCount',
+            ], titles)
+            if not membersCountOnly:
+                display.add_titles_to_csv_file([
+                    'Members',
+                ], titles)
+        if managers:
+            display.add_titles_to_csv_file([
+                'ManagersCount',
+            ], titles)
+            if not managersCountOnly:
+                display.add_titles_to_csv_file([
+                    'Managers',
+                ], titles)
+        if owners:
+            display.add_titles_to_csv_file([
+                'OwnersCount',
+            ], titles)
+            if not ownersCountOnly:
+                display.add_titles_to_csv_file([
+                    'Owners',
+                ], titles)
+    gam.printGettingAllItems('Groups', usemember)
+    page_message = gapi.got_total_items_first_last_msg('Groups')
+    if usemember:
+        try:
+            result = gapi.get_all_pages(ci.groups().memberships(),
+                                        'searchTransitiveGroups',
+                                        'memberships',
+                                        throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
+                                        page_message=page_message,
+                                        message_attribute=['groupKey', 'id'],
+                                        parent='groups/-', query=usemember,
+                                        fields='nextPageToken,memberships(group,groupKey(id),relationType)',
+                                        pageSize=1000)
+        except googleapiclient.errors.HttpError:
+            controlflow.system_error_exit(
+                2,
+                f'enterprisemember requires Enterprise license')
+        entityList = []
+        for entity in result:
+            if entity['relationType'] == 'DIRECT':
+                entityList.append(gapi.call(ci.groups(), 'get', name=entity['group']))
+    else:
+        entityList = gapi.get_all_pages(ci.groups(),
+                                        'list',
+                                        'groups',
+                                        page_message=page_message,
+                                        message_attribute=['groupKey', 'id'],
+                                        parent=parent,
+                                        view='FULL',
+                                        pageSize=500)
+    i = 0
+    count = len(entityList)
+    for groupEntity in entityList:
+        i += 1
+        groupEmail = groupEntity['groupKey']['id']
+        for k, v in iter(groupEntity.pop('labels', {}).items()):
+            if v == '':
+                groupEntity[f'labels.{k}'] = True
+            else:
+                groupEntity[f'labels.{k}'] = v
+        group = utils.flatten_json(groupEntity)
+        for a_key in group:
+            if a_key not in titles:
+                titles.append(a_key)
+        groupKey_id = groupEntity['name']
+        if roles:
+            sys.stderr.write(
+                f' Getting {roles} for {groupEmail}{gam.currentCountNL(i, count)}'
+            )
+            page_message = gapi.got_total_items_first_last_msg('Members')
+            validRoles, _, _ = gam._getRoleVerification(
+                '.'.join(roles), 'nextPageToken,members(email,id,role)')
+            groupMembers = gapi.get_all_pages(ci.groups().memberships(),
+                                              'list',
+                                              'memberships',
+                                              page_message=page_message,
+                                              message_attribute=['memberKey', 'id'],
+                                              soft_errors=True,
+                                              parent=groupKey_id,
+                                              view='BASIC')
+            if members:
+                membersList = []
+                membersCount = 0
+            if managers:
+                managersList = []
+                managersCount = 0
+            if owners:
+                ownersList = []
+                ownersCount = 0
+            for member in groupMembers:
+                member_email = member['memberKey']['id']
+                role = get_single_role(member.get('roles'))
+                if not validRoles or role in validRoles:
+                    if role == ROLE_MEMBER:
+                        if members:
+                            membersCount += 1
+                            if not membersCountOnly:
+                                membersList.append(member_email)
+                    elif role == ROLE_MANAGER:
+                        if managers:
+                            managersCount += 1
+                            if not managersCountOnly:
+                                managersList.append(member_email)
+                    elif role == ROLE_OWNER:
+                        if owners:
+                            ownersCount += 1
+                            if not ownersCountOnly:
+                                ownersList.append(member_email)
+                    elif members:
+                        membersCount += 1
+                        if not membersCountOnly:
+                            membersList.append(member_email)
+            if members:
+                group['MembersCount'] = membersCount
+                if not membersCountOnly:
+                    group['Members'] = memberDelimiter.join(membersList)
+            if managers:
+                group['ManagersCount'] = managersCount
+                if not managersCountOnly:
+                    group['Managers'] = memberDelimiter.join(managersList)
+            if owners:
+                group['OwnersCount'] = ownersCount
+                if not ownersCountOnly:
+                    group['Owners'] = memberDelimiter.join(ownersList)
+        csvRows.append(group)
+    if sortHeaders:
+        display.sort_csv_titles([
+            'name', 'groupKey.id'
+        ], titles)
+    display.write_csv_file(csvRows, titles, 'Groups', todrive)
+
+
+def _get_groups_list(ci=None, member=None, parent=None):
+    if not ci:
+        ci = build()
+    if not parent:
+            gapi_directory_customer.setTrueCustomerId()
+            parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    gam.printGettingAllItems('Groups', member)
+    page_message = gapi.got_total_items_first_last_msg('Groups')
+    if member:
+        fields = 'nextPageToken,memberships(groupKey(id),relationType)'
+        try:
+            groups_to_get = gapi.get_all_pages(ci.groups().memberships(),
+                                               'searchTransitiveGroups',
+                                               'memberships',
+                                               throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
+                                               message_attribute=['groupKey', 'id'],
+                                               page_message=page_message,
+                                               parent='groups/-',
+                                               query=member,
+                                               pageSize=1000,
+                                               fields=fields)
+        except googleapiclient.errors.HttpError:
+            controlflow.system_error_exit(
+                    2,
+                    f'enterprisemember requires Enterprise license')
+        return [group['groupKey']['id'] for group in groups_to_get if group['relationType'] == 'DIRECT']
+    else:
+        groups_to_get = gapi.get_all_pages(
+            ci.groups(),
+            'list',
+            'groups',
+            message_attribute=['groupKey', 'id'],
+            page_message=page_message,
+            parent=parent,
+            view='BASIC',
+            pageSize=1000,
+            fields='nextPageToken,groups(groupKey(id))')
+        return [group['groupKey']['id'] for group in groups_to_get]
+
+
+def get_membership_graph(member):
+    ci = build()
+    query = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
+    result = gapi.call(ci.groups().memberships(),
+                     'getMembershipGraph',
+                     parent='groups/-',
+                     query=query)
+    return result.get('response')
+
+
+def print_members():
+    ci = build()
+    todrive = False
+    gapi_directory_customer.setTrueCustomerId()
+    parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    usemember = None
+    roles = []
+    titles = ['group']
+    csvRows = []
+    groups_to_get = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['role', 'roles']:
+            for role in sys.argv[i + 1].lower().replace(',', ' ').split():
+                if role in GROUP_ROLES_MAP:
+                    roles.append(GROUP_ROLES_MAP[role])
+                else:
+                    controlflow.system_error_exit(
+                        2,
+                        f'{role} is not a valid role for "gam print group-members {myarg}"'
+                    )
+            i += 2
+        elif myarg == 'enterprisemember':
+            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
+            i += 2
+        elif myarg in ['cigroup', 'cigroups']:
+            group_email = gam.normalizeEmailAddressOrUID(sys.argv[i + 1])
+            groups_to_get = [group_email]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print cigroup-members')
+    if not groups_to_get:
+        groups_to_get = _get_groups_list(ci, usemember, parent)
+    i = 0
+    count = len(groups_to_get)
+    for group_email in groups_to_get:
+        i += 1
+
+        sys.stderr.write(
+            f'Getting members for {group_email}{gam.currentCountNL(i, count)}')
+        group_id = group_email_to_id(ci, group_email)
+        print(f'Getting members of cigroup {group_email}...')
+        page_message = f' {gapi.got_total_items_first_last_msg("Members")}'
+        group_members = gapi.get_all_pages(
+            ci.groups().memberships(),
+            'list',
+            'memberships',
+            soft_errors=True,
+            parent=group_id,
+            view='FULL',
+            pageSize=500,
+            page_message=page_message,
+            message_attribute=['memberKey', 'id'])
+        #fields='nextPageToken,memberships(memberKey,roles,createTime,updateTime)')
+        if roles:
+            group_members = filter_members_to_roles(group_members, roles)
+        for member in group_members:
+            # reduce role to a single value
+            member['role'] = get_single_role(member.pop('roles'))
+            member = utils.flatten_json(member)
+            for title in member:
+                if title not in titles:
+                    titles.append(title)
+            member['group'] = group_email
+            csvRows.append(member)
+    display.write_csv_file(csvRows, titles, 'Group Members', todrive)
+
+
+def update():
+
+    # Convert foo@googlemail.com to foo@gmail.com; eliminate periods in name for foo.bar@gmail.com
+    def _cleanConsumerAddress(emailAddress, mapCleanToOriginal):
+        atLoc = emailAddress.find('@')
+        if atLoc > 0:
+            if emailAddress[atLoc + 1:] in ['gmail.com', 'googlemail.com']:
+                cleanEmailAddress = emailAddress[:atLoc].replace(
+                    '.', '') + '@gmail.com'
+                if cleanEmailAddress != emailAddress:
+                    mapCleanToOriginal[cleanEmailAddress] = emailAddress
+                    return cleanEmailAddress
+        return emailAddress
+
+    def _getRoleAndUsers():
+        checkSuspended = None
+        role = ROLE_MEMBER
+        expireTime = None
+        i = 5
+        if sys.argv[i].lower() in GROUP_ROLES_MAP:
+            role = GROUP_ROLES_MAP[sys.argv[i].lower()]
+            i += 1
+        if sys.argv[i].lower() in ['suspended', 'notsuspended']:
+            checkSuspended = sys.argv[i].lower() == 'suspended'
+            i += 1
+        if sys.argv[i].lower() in ['expire', 'expires']:
+            if role != ROLE_MEMBER:
+                controlflow.invalid_argument_exit(
+                    sys.argv[i], f'role {role}')
+            expireTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        if sys.argv[i].lower() in usergroup_types:
+            users_email = gam.getUsersToModify(entity_type=sys.argv[i].lower(),
+                                               entity=sys.argv[i + 1],
+                                               checkSuspended=checkSuspended,
+                                               groupUserMembersOnly=False)
+        else:
+            users_email = [
+                gam.normalizeEmailAddressOrUID(sys.argv[i],
+                                               checkForCustomerId=True)
+            ]
+        return (role, expireTime, users_email)
+
+    ci = build()
+    group = sys.argv[3]
+    myarg = sys.argv[4].lower()
+    items = []
+    if myarg in UPDATE_GROUP_SUBCMDS:
+        group = gam.normalizeEmailAddressOrUID(group)
+        if group.startswith('groups/'):
+            parent = group
+        else:
+            parent = group_email_to_id(ci, group)
+        if not parent:
+            return
+        if myarg == 'add':
+            role, expireTime, users_email = _getRoleAndUsers()
+            if len(users_email) > 1:
+                sys.stderr.write(
+                    f'Group: {group}, Will add {len(users_email)} {role}s.\n')
+                for user_email in users_email:
+                    item = [
+                        'gam', 'update', 'cigroup', f'id:{parent}', 'add', role,
+                    ]
+                    if expireTime:
+                        item.extend(['expires', expireTime])
+                    item.append(user_email)
+                    items.append(item)
+            elif len(users_email) > 0:
+                body = {
+                    'memberKey': {
+                        'id': users_email[0]
+                    },
+                    'roles': [{
+                        'name': ROLE_MEMBER
+                    }]
+                }
+                if role != ROLE_MEMBER:
+                    body['roles'].append({'name': role})
+                elif expireTime not in {None, NEVER_TIME}:
+                    for role in body['roles']:
+                        if role['name'] == ROLE_MEMBER:
+                            role['expiryDetail'] = {'expireTime': expireTime}
+                add_text = [f'as {role}']
+                for i in range(2):
+                    try:
+                        gapi.call(
+                            ci.groups().memberships(),
+                            'create',
+                            throw_reasons=[
+                                gapi_errors.ErrorReason.FOUR_O_NINE,
+                                gapi_errors.ErrorReason.MEMBER_NOT_FOUND,
+                                gapi_errors.ErrorReason.RESOURCE_NOT_FOUND,
+                                gapi_errors.ErrorReason.INVALID_MEMBER,
+                                gapi_errors.ErrorReason.
+                                CYCLIC_MEMBERSHIPS_NOT_ALLOWED
+                            ],
+                            parent=parent,
+                            body=body)
+                        print(
+                            f' Group: {group}, {users_email[0]} Added {" ".join(add_text)}'
+                        )
+                        break
+                    except (gapi_errors.GapiMemberNotFoundError,
+                            gapi_errors.GapiResourceNotFoundError,
+                            gapi_errors.GapiInvalidMemberError,
+                            gapi_errors.GapiCyclicMembershipsNotAllowedError
+                           ) as e:
+                        print(
+                            f' Group: {group}, {users_email[0]} Add {" ".join(add_text)} Failed: {str(e)}'
+                        )
+                        break
+        elif myarg == 'sync':
+            syncMembersSet = set()
+            syncMembersMap = {}
+            role, expireTime, users_email = _getRoleAndUsers()
+            for user_email in users_email:
+                if user_email in ('*', GC_Values[GC_CUSTOMER_ID]):
+                    syncMembersSet.add(GC_Values[GC_CUSTOMER_ID])
+                else:
+                    syncMembersSet.add(
+                        _cleanConsumerAddress(user_email.lower(),
+                                              syncMembersMap))
+            currentMembersSet = set()
+            currentMembersMap = {}
+            for current_email in gam.getUsersToModify(
+                    entity_type='cigroup',
+                    entity=group,
+                    member_type=role,
+                    groupUserMembersOnly=False):
+                if current_email == GC_Values[GC_CUSTOMER_ID]:
+                    currentMembersSet.add(current_email)
+                else:
+                    currentMembersSet.add(
+                        _cleanConsumerAddress(current_email.lower(),
+                                              currentMembersMap))
+            to_add = [
+                syncMembersMap.get(emailAddress, emailAddress)
+                for emailAddress in syncMembersSet - currentMembersSet
+            ]
+            to_remove = [
+                currentMembersMap.get(emailAddress, emailAddress)
+                for emailAddress in currentMembersSet - syncMembersSet
+            ]
+            sys.stderr.write(
+                f'Group: {group}, Will add {len(to_add)} and remove {len(to_remove)} {role}s.\n'
+            )
+            for user in to_add:
+                item = ['gam', 'update', 'cigroup', f'id:{parent}', 'add',
+                        role,]
+                if role == ROLE_MEMBER and expireTime not in {None, NEVER_TIME}:
+                    item.extend(['expires', expireTime])
+                item.append(user)
+                items.append(item)
+            for user in to_remove:
+                items.append([
+                    'gam', 'update', 'cigroup', f'id:{parent}', 'remove', user
+                ])
+        elif myarg in ['delete', 'remove']:
+            _, _, users_email = _getRoleAndUsers()
+            if len(users_email) > 1:
+                sys.stderr.write(
+                    f'Group: {group}, Will remove {len(users_email)} emails.\n')
+                for user_email in users_email:
+                    items.append([
+                        'gam', 'update', 'cigroup', f'id:{parent}', 'remove',
+                        user_email
+                    ])
+            elif len(users_email) == 1:
+                name = membership_email_to_id(ci, parent, users_email[0])
+                try:
+                    gapi.call(ci.groups().memberships(),
+                              'delete',
+                              throw_reasons=[
+                                  gapi_errors.ErrorReason.MEMBER_NOT_FOUND,
+                                  gapi_errors.ErrorReason.INVALID_MEMBER
+                              ],
+                              name=name)
+                    print(f' Group: {group}, {users_email[0]} Removed')
+                except (gapi_errors.GapiMemberNotFoundError,
+                        gapi_errors.GapiInvalidMemberError) as e:
+                    print(
+                        f' Group: {group}, {users_email[0]} Remove Failed: {str(e)}'
+                    )
+        elif myarg == 'update':
+            role, expireTime, users_email = _getRoleAndUsers()
+            if len(users_email) > 1:
+                sys.stderr.write(
+                    f'Group: {group}, Will update {len(users_email)} {role}s.\n'
+                )
+                for user_email in users_email:
+                    item = [
+                        'gam', 'update', 'cigroup', f'id:{parent}', 'update',
+                        role,]
+                    if expireTime:
+                        item.extend(['expires', expireTime])
+                    item.append(user_email)
+                    items.append(item)
+            elif len(users_email) > 0:
+                name = membership_email_to_id(ci, parent, users_email[0])
+                preUpdateRoles = []
+                addRoles = []
+                removeRoles = []
+                postUpdateRoles = []
+                member_roles = gapi.call(ci.groups().memberships(),
+                                         'get',
+                                         name=name,
+                                         fields='roles').get('roles', [{'name': ROLE_MEMBER}])
+                current_roles = [crole['name'] for crole in member_roles]
+                # When upgrading role, strip any expiryDetail from member before role changes
+                if role != ROLE_MEMBER:
+                    for crole in member_roles:
+                        if 'expiryDetail' in crole:
+                            preUpdateRoles.append(
+                                {'fieldMask': 'expiryDetail.expireTime',
+                                 'membershipRole': {'name': ROLE_MEMBER,
+                                                    'expiryDetail': {'expireTime': None}}})
+                            break
+                # When downgrading role or simply updating member expireTime, update expiryDetail after role changes
+                elif expireTime:
+                    postUpdateRoles.append(
+                        {'fieldMask': 'expiryDetail.expireTime',
+                         'membershipRole': {'name': role,
+                                            'expiryDetail': {'expireTime':  expireTime if expireTime != NEVER_TIME else None}}})
+                for crole in current_roles:
+                    if crole not in {ROLE_MEMBER, role}:
+                        removeRoles.append(crole)
+                if role not in current_roles:
+                    new_role = {'name': role}
+                    if role == ROLE_MEMBER and expireTime not in {None, NEVER_TIME}:
+                        new_role['expiryDetail'] = {'expireTime': expireTime}
+                        postUpdateRoles = []
+                    addRoles.append(new_role)
+                bodys = []
+                if preUpdateRoles:
+                    bodys.append({'updateRolesParams': preUpdateRoles})
+                if addRoles:
+                    bodys.append({'addRoles': addRoles})
+                if removeRoles:
+                    bodys.append({'removeRoles': removeRoles})
+                if postUpdateRoles:
+                    bodys.append({'updateRolesParams': postUpdateRoles})
+                for body in bodys:
+                    try:
+                        gapi.call(ci.groups().memberships(),
+                                  'modifyMembershipRoles',
+                                  throw_reasons=[
+                                      gapi_errors.ErrorReason.MEMBER_NOT_FOUND,
+                                      gapi_errors.ErrorReason.INVALID_MEMBER
+                                  ],
+                                  name=name,
+                                  body=body)
+                    except (gapi_errors.GapiMemberNotFoundError,
+                            gapi_errors.GapiInvalidMemberError) as e:
+                        print(
+                            f' Group: {group}, {users_email[0]} Update to {role} Failed: {str(e)}'
+                        )
+                        break
+                print(
+                  f' Group: {group}, {users_email[0]} Updated to {role}'
+                )
+
+        else:  # clear
+            roles = []
+            i = 5
+            while i < len(sys.argv):
+                myarg = sys.argv[i].lower()
+                if myarg.upper() in [ROLE_OWNER, ROLE_MANAGER, ROLE_MEMBER]:
+                    roles.append(myarg.upper())
+                    i += 1
+                else:
+                    controlflow.invalid_argument_exit(
+                        sys.argv[i], 'gam update cigroup clear')
+            if not roles:
+                roles = [ROLE_MEMBER]
+            group = gam.normalizeEmailAddressOrUID(group)
+            member_type_message = f'{",".join(roles).lower()}s'
+            sys.stderr.write(
+                f'Getting {member_type_message} of {group} (may take some time for large groups)...\n'
+            )
+            page_message = gapi.got_total_items_msg(f'{member_type_message}',
+                                                    '...')
+            try:
+                result = gapi.get_all_pages(
+                    ci.groups().memberships(),
+                    'list',
+                    'memberships',
+                    page_message=page_message,
+                    throw_reasons=gapi_errors.MEMBERS_THROW_REASONS,
+                    parent=parent,
+                    fields='nextPageToken,memberships(memberKey,roles)')
+                result = filter_members_to_roles(result, roles)
+                if not result:
+                    print('Group already has 0 members')
+                    return
+                users_email = [member['memberKey']['id'] for member in result]
+                sys.stderr.write(
+                    f'Group: {group}, Will remove {len(users_email)} {", ".join(roles).lower()}s.\n'
+                )
+                for user_email in users_email:
+                    items.append([
+                        'gam', 'update', 'cigroup', group, 'remove', user_email
+                    ])
+            except (gapi_errors.GapiGroupNotFoundError,
+                    gapi_errors.GapiDomainNotFoundError,
+                    gapi_errors.GapiInvalidError,
+                    gapi_errors.GapiForbiddenError):
+                gam.entityUnknownWarning('Group', group, 0, 0)
+        if items:
+            gam.run_batch(items)
+    else:
+        i = 4
+        body = {}
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower().replace('_', '')
+            if myarg == 'name':
+                body['displayName'] = sys.argv[i + 1]
+                i += 2
+            elif myarg == 'description':
+                body['description'] = sys.argv[i + 1]
+                i += 2
+            elif myarg == 'security':
+                body['labels'] = {
+                    'cloudidentity.googleapis.com/groups.security': '',
+                    'cloudidentity.googleapis.com/groups.discussion_forum': ''
+                }
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i],
+                                                  'gam update cigroup')
+        updateMask = ','.join(body.keys())
+        name = group_email_to_id(ci, group)
+        print(f'Updating group {group}')
+        gapi.call(ci.groups(),
+                  'patch',
+                  updateMask=updateMask,
+                  name=name,
+                  body=body)
+
+
+def group_email_to_id(ci, group, i=0, count=0):
+    group = gam.normalizeEmailAddressOrUID(group)
+    try:
+        return gapi.call(ci.groups(),
+                         'lookup',
+                         throw_reasons=gapi_errors.GROUP_GET_THROW_REASONS,
+                         retry_reasons=gapi_errors.GROUP_GET_RETRY_REASONS,
+                         groupKey_id=group,
+                         fields='name').get('name')
+    except (gapi_errors.GapiGroupNotFoundError,
+            gapi_errors.GapiDomainNotFoundError,
+            gapi_errors.GapiDomainCannotUseApisError,
+            gapi_errors.GapiForbiddenError, gapi_errors.GapiBadRequestError):
+        gam.entityUnknownWarning('Group', group, i, count)
+        return None
+
+
+def membership_email_to_id(ci, parent, membership, i=0, count=0):
+    membership = gam.normalizeEmailAddressOrUID(membership)
+    try:
+        return gapi.call(ci.groups().memberships(),
+                         'lookup',
+                         throw_reasons=gapi_errors.GROUP_GET_THROW_REASONS,
+                         retry_reasons=gapi_errors.GROUP_GET_RETRY_REASONS,
+                         parent=parent,
+                         memberKey_id=membership,
+                         fields='name').get('name')
+    except (gapi_errors.GapiGroupNotFoundError,
+            gapi_errors.GapiDomainNotFoundError,
+            gapi_errors.GapiDomainCannotUseApisError,
+            gapi_errors.GapiForbiddenError, gapi_errors.GapiBadRequestError):
+        gam.entityUnknownWarning('Membership', membership, i, count)
+        return None
+
+
+def get_single_role(roles):
+    ''' returns the highest role of member '''
+    roles = [role.get('name') for role in roles]
+    if not roles:
+        return ROLE_MEMBER
+    for a_role in [ROLE_OWNER, ROLE_MANAGER, ROLE_MEMBER]:
+        if a_role in roles:
+            return a_role
+    return roles[0]
+
+
+def filter_members_to_roles(members, roles):
+    filtered_members = []
+    for member in members:
+        role = get_single_role(member.get('roles', []))
+        if role in roles:
+            filtered_members.append(member)
+    return filtered_members

src/gam/gapi/cloudidentity/userinvitations.py

@@ -0,0 +1,207 @@
+"""Methods related to Cloud Identity User Invitation API"""
+import sys
+from urllib.parse import quote_plus
+
+import googleapiclient
+
+import gam
+from gam.var import GC_CUSTOMER_ID, GC_Values, MY_CUSTOMER, SORTORDER_CHOICES_MAP
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import errors as gapi_errors
+from gam.gapi import cloudidentity as gapi_cloudidentity
+
+def _get_customerid():
+    ''' returns customer in "customers/(C){customer_id}' format needed for this API'''
+    customer = GC_Values[GC_CUSTOMER_ID]
+    if customer != MY_CUSTOMER and customer[0] != 'C':
+        customer = 'C' + customer
+    return f'customers/{customer}'
+
+def _reduce_name(name):
+    ''' converts long name into email address'''
+    return name.split('/')[-1]
+
+def is_invitable_user(email):
+    '''return email isInvitableUser'''
+    svc = gapi_cloudidentity.build_dwd('cloudidentity_beta')
+    customer = _get_customerid()
+    encoded_email = quote_plus(email)
+    name = f'{customer}/userinvitations/{encoded_email}'
+    return gapi.call(svc.customers().userinvitations(), 'isInvitableUser',
+                     name=name)['isInvitableUser']
+
+
+def _generic_action(action):
+    '''generic function to call actionable APIs'''
+    svc = gapi_cloudidentity.build_dwd('cloudidentity_beta')
+    customer = _get_customerid()
+    email = sys.argv[3].lower()
+    encoded_email = quote_plus(email)
+    name = f'{customer}/userinvitations/{encoded_email}'
+    action_map = {
+            'cancel': 'Cancelling',
+            'send': 'Sending'
+            }
+    print_action = action_map[action]
+    print(f'{print_action} user invitation...')
+    result = gapi.call(svc.customers().userinvitations(), action,
+            name=name)
+    name = result.get('response', {}).get('name')
+    if name:
+        result['response']['name'] = _reduce_name(name)
+    display.print_json(result)
+
+def _generic_get(get_type):
+    '''generic function to call read data APIs'''
+    svc = gapi_cloudidentity.build_dwd('cloudidentity_beta')
+    customer = _get_customerid()
+    email = sys.argv[3].lower()
+    encoded_email = quote_plus(email)
+    name = f'{customer}/userinvitations/{encoded_email}'
+    result = gapi.call(svc.customers().userinvitations(), get_type,
+            name=name)
+    if 'name' in result:
+        result['name'] = _reduce_name(result['name'])
+    display.print_json(result)
+
+
+# /batch is broken for Cloud Identity. Once fixed move this to using batch.
+# Current serial implementation will be SLOW...
+def bulk_is_invitable(emails):
+    '''gam <users> check isinvitable'''
+    def _invitation_result(request_id, response, _):
+        if response.get('isInvitableUser'):
+            rows.append({'invitableUsers': request_id})
+
+    svc = gapi_cloudidentity.build_dwd('cloudidentity_beta')
+    customer = _get_customerid()
+    todrive = False
+    #batch_size = 1000
+    #ebatch = svc.new_batch_http_request(callback=_invitation_result)
+    rows = []
+    throw_reasons = [gapi_errors.ErrorReason.FOUR_O_THREE]
+    for email in emails:
+        encoded_email = quote_plus(email)
+        name = f'{customer}/userinvitations/{encoded_email}'
+        endpoint = svc.customers().userinvitations()
+        #if len(ebatch._order) == batch_size:
+        #    ebatch.execute()
+        #    ebatch = svc.new_batch_http_request(callback=_invitation_result)
+        #req = endpoint.isInvitableUser(name=name)
+        #ebatch.add(req, request_id=email)
+        try:
+            result = gapi.call(endpoint,
+                               'isInvitableUser',
+                               throw_reasons=throw_reasons,
+                               name=name)
+        except googleapiclient.errors.HttpError:
+            continue
+        if result.get('isInvitableUser'):
+            rows.append({'invitableUsers': email})
+    #ebatch.execute()
+    titles = ['invitableUsers']
+    display.write_csv_file(rows, titles, 'Invitable Users', todrive)
+
+
+def cancel():
+    '''gam cancel userinvitation <email>'''
+    _generic_action('cancel')
+
+
+def get():
+    '''gam info userinvitation <email>'''
+    _generic_get('get')
+
+
+def check():
+    '''gam check userinvitation <email>'''
+    _generic_get('isInvitableUser')
+
+
+def send():
+    '''gam send userinvitation <email>'''
+    _generic_action('send')
+
+
+USERINVITATION_ORDERBY_CHOICES_MAP = {
+    'email': 'email',
+    'updatetime': 'update_time',
+    }
+
+USERINVITATION_STATE_CHOICES_MAP = {
+    'accepted': 'ACCEPTED',
+    'declined': 'DECLINED',
+    'invited': 'INVITED',
+    'notyetsent': 'NOT_YET_SENT',
+    }
+
+def print_():
+    '''gam print userinvitations'''
+    svc = gapi_cloudidentity.build_dwd('cloudidentity_beta')
+    customer = _get_customerid()
+    todrive = False
+    titles = ['name', 'state', 'updateTime']
+    rows = []
+    filter_ = None
+    orderByList = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'state':
+            state = sys.argv[i + 1].lower().replace('_', '')
+            if state in USERINVITATION_STATE_CHOICES_MAP:
+                filter_ = f"state=='{USERINVITATION_STATE_CHOICES_MAP[state]}'"
+            else:
+                controlflow.expected_argument_exit('state',
+                                                   ', '.join(USERINVITATION_STATE_CHOICES_MAP),
+                                                   state)
+            i += 2
+        elif myarg == 'orderby':
+            fieldName = sys.argv[i + 1].lower()
+            i += 2
+            if fieldName in USERINVITATION_ORDERBY_CHOICES_MAP:
+                fieldName = USERINVITATION_ORDERBY_CHOICES_MAP[fieldName]
+                orderBy = ''
+                if i < len(sys.argv):
+                    orderBy = sys.argv[i].lower()
+                    if orderBy in SORTORDER_CHOICES_MAP:
+                        orderBy = SORTORDER_CHOICES_MAP[orderBy]
+                        i += 1
+                if orderBy != 'DESCENDING':
+                    orderByList.append(fieldName)
+                else:
+                    orderByList.append(f'{fieldName} desc')
+            else:
+                controlflow.expected_argument_exit(
+                    'orderby', ', '.join(sorted(USERINVITATION_ORDERBY_CHOICES_MAP)),
+                    fieldName)
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print userinvitations')
+    if orderByList:
+        orderBy = ' '.join(orderByList)
+    else:
+        orderBy = None
+    gam.printGettingAllItems('User Invitations', filter_)
+    page_message = gapi.got_total_items_msg('User Invitations', '...\n')
+    invitations = gapi.get_all_pages(svc.customers().userinvitations(),
+                                'list',
+                                'userInvitations',
+                                page_message=page_message,
+                                parent=customer,
+                                filter=filter_,
+                                orderBy=orderBy)
+    for invitation in invitations:
+        invitation['name'] = _reduce_name(invitation['name'])
+        row = {}
+        for key, val in invitation.items():
+            if key not in titles:
+                titles.append(key)
+            row[key] = val
+        rows.append(row)
+    display.write_csv_file(rows, titles, 'User Invitations', todrive)

src/gam/gapi/contactdelegation.py

@@ -0,0 +1,96 @@
+"""Contact Delegation API calls"""
+
+import sys
+
+import gam
+from gam.gapi.directory import users as gapi_directory_users
+from gam import controlflow
+from gam import display
+from gam import gapi
+
+
+def build():
+    return gam.buildGAPIObject('contactdelegation')
+
+
+def create(users):
+    condel = build()
+    delegate = gam.normalizeEmailAddressOrUID(sys.argv[5])
+    delegate = gapi_directory_users.get_primary(delegate)
+    if not delegate:
+        controlflow.system_error_exit(5,
+                                      f'{sys.argv[5]} is not the primary address of a user.')
+    body = {'email': delegate}
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        print(
+            f'Granting {delegate} contact delegate access to {user}{gam.currentCount(i, count)}'
+        )
+        gapi.call(condel.delegates(),
+                  'create',
+                  soft_errors=True,
+                  user=user,
+                  body=body)
+
+
+def delete(users):
+    condel = build()
+    delegate = gam.normalizeEmailAddressOrUID(sys.argv[5])
+    delegate = gapi_directory_users.get_primary(delegate)
+    if not delegate:
+        controlflow.system_error_exit(5,
+                                      f'{sys.argv[5]} is not the primary address of a user.')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        print(
+            f'Deleting {delegate} contact delegate access to {user}{gam.currentCount(i, count)}'
+        )
+        gapi.call(condel.delegates(),
+                  'delete',
+                  soft_errors=True,
+                  user=user,
+                  delegate=delegate)
+
+
+def print_(users, csvFormat):
+    condel = build()
+    if csvFormat:
+        todrive = False
+        csv_rows = []
+        titles = ['User', 'delegateAddress']
+    else:
+        csvStyle = False
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if not csvFormat and myarg == 'csv':
+            csvStyle = True
+            i += 1
+        elif csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print contactdelegation')
+    page_message = gapi.got_total_items_msg('Contact Delegates', '...\n')
+    for user in users:
+        delegates = gapi.get_all_pages(condel.delegates(), 'list',
+                                       'delegates',
+                                       page_message=page_message,
+                                       user=user)
+        for delegate in delegates:
+            if csvFormat:
+                csv_rows.append({'User': user, 'delegateAddress': delegate['email']})
+            else:
+                if csvStyle:
+                    print(f'{user},{delegate["email"]}')
+                else:
+                    print(
+                        f'Delegator: {user}\n  Delegate Email: {delegate["email"]}\n'
+                    )
+    if csvFormat:
+        display.write_csv_file(csv_rows, titles, 'Contact Delegates', todrive)

src/gam/gapi/directory/__init__.py

@@ -0,0 +1,5 @@
+import gam
+
+
+def build():
+    return gam.buildGAPIObject('directory')

src/gam/gapi/directory/asps.py

@@ -0,0 +1,57 @@
+import sys
+
+from gam.var import *
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def info(users):
+    cd = gapi_directory.build()
+    for user in users:
+        asps = gapi.get_items(cd.asps(), 'list', 'items', userKey=user)
+        if asps:
+            print(f'Application-Specific Passwords for {user}')
+            for asp in asps:
+                if asp['creationTime'] == '0':
+                    created_date = 'Unknown'
+                else:
+                    created_date = utils.formatTimestampYMDHMS(
+                        asp['creationTime'])
+                if asp['lastTimeUsed'] == '0':
+                    used_date = 'Never'
+                else:
+                    last_used = asp['lastTimeUsed']
+                    used_date = utils.formatTimestampYMDHMS(last_used)
+                print(f' ID: {asp["codeId"]}\n' \
+                      f'  Name: {asp["name"]}\n' \
+                      f'  Created: {created_date}\n' \
+                      f'  Last Used: {used_date}\n')
+        else:
+            print(f' no ASPs for {user}\n')
+
+
+def delete(users, cd=None, codeIdList=None):
+    if not cd:
+        cd = gapi_directory.build()
+    if not codeIdList:
+        codeIdList = sys.argv[5].lower()
+    if codeIdList == 'all':
+        allCodeIds = True
+    else:
+        allCodeIds = False
+        codeIds = codeIdList.replace(',', ' ').split()
+    for user in users:
+        if allCodeIds:
+            print(f'Getting Application Specific Passwords for {user}')
+            asps = gapi.get_items(cd.asps(),
+                                  'list',
+                                  'items',
+                                  userKey=user,
+                                  fields='items/codeId')
+            codeIds = [asp['codeId'] for asp in asps]
+        if not codeIds:
+            print('No ASPs')
+        for codeId in codeIds:
+            gapi.call(cd.asps(), 'delete', userKey=user, codeId=codeId)
+            print(f'deleted ASP {codeId} for {user}')

src/gam/gapi/directory/cros.py

@@ -0,0 +1,928 @@
+import datetime
+import json
+import os
+import sys
+import time
+
+import googleapiclient
+
+from gam.var import *
+import gam
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import errors as gapi_errors
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+from gam import utils
+
+
+def _display_cros_command_result(cd, device_id, command_id, times_to_check_status):
+    print(f'deviceId: {device_id}, commandId: {command_id}')
+    final_states = {'EXPIRED', 'CANCELLED', 'EXECUTED_BY_CLIENT'}
+    for _ in range(0, times_to_check_status):
+        time.sleep(2)
+        result = gapi.call(cd.customer().devices().chromeos().commands(), 'get',
+            customerId=GC_Values[GC_CUSTOMER_ID], deviceId=device_id,
+            commandId=command_id)
+        display.print_json(result)
+        if result.get('state') in final_states:
+            return
+
+def issue_command():
+    cd = gapi_directory.build()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    body = {}
+    valid_commands = gapi.get_enum_values_minus_unspecified(
+            cd._rootDesc['schemas']
+            ['DirectoryChromeosdevicesIssueCommandRequest']
+            ['properties']['commandType']['enum'])
+    command_map = {}
+    for valid_command in valid_commands:
+        v = valid_command.lower().replace('_', '')
+        command_map[v] = valid_command
+    times_to_check_status = 1
+    doit = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'command':
+             command = sys.argv[i+1].lower().replace('_', '')
+             if command not in command_map:
+                 controlflow.system_error_exit(2, f'expected command of ' \
+                     f'{", ".join(valid_commands)} got {command}')
+             body['commandType'] = command_map[command]
+             i += 2
+             if command == 'setvolume':
+                 body['payload'] = json.dumps({'volume': sys.argv[i]})
+                 i += 1
+        elif myarg == 'timestocheckstatus':
+            times_to_check_status = int(sys.argv[i+1])
+            i += 2
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam issuecommand cros')
+    if 'commandType' not in body:
+        controlflow.missing_argument_exit('command <CrOSCommand>', 'gam issuecommand cros')
+    if body['commandType'] == 'WIPE_USERS' and not doit:
+        controlflow.system_error_exit(2, 'wipe_users command requires admin ' \
+            'acknowledge user data will be destroyed with the ' \
+            'doit argument')
+    if body['commandType'] == 'REMOTE_POWERWASH' and not doit:
+        controlflow.system_error_exit(2, 'remote_powerwash command requires ' \
+            'admin acknowledge user data will be destroyed, device will need' \
+            ' to be reconnected to WiFi and re-enrolled with the doit argument')
+    for device_id in devices:
+        try:
+            result = gapi.call(cd.customer().devices().chromeos(), 'issueCommand',
+                  customerId=GC_Values[GC_CUSTOMER_ID], deviceId=device_id,
+                  throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
+                  body=body)
+        except googleapiclient.errors.HttpError:
+            controlflow.system_error_exit(4, '400 response from Google. This ' \
+              'usually indicates the devices was not in a state where it will' \
+              ' accept the command. For example, reboot, set_volume and take_a_screenshot' \
+              ' require the device to be in auto-start kiosk app mode.')
+        command_id = result.get('commandId')
+        _display_cros_command_result(cd, device_id, command_id, times_to_check_status)
+
+def get_command():
+    cd = gapi_directory.build()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    command_id = None
+    times_to_check_status = 1
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'commandid':
+            command_id = sys.argv[i+1]
+            i += 2
+        elif myarg == 'timestocheckstatus':
+            times_to_check_status = int(sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam getcommand cros')
+    for device_id in devices:
+        _display_cros_command_result(cd, device_id, command_id, times_to_check_status)
+
+def doUpdateCros():
+    cd = gapi_directory.build()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    update_body = {}
+    action_body = {}
+    orgUnitPath = None
+    ack_wipe = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'user':
+            update_body['annotatedUser'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'location':
+            update_body['annotatedLocation'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'notes':
+            update_body['notes'] = sys.argv[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg in ['tag', 'asset', 'assetid']:
+            update_body['annotatedAssetId'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['ou', 'org']:
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'action':
+            action = sys.argv[i + 1].lower().replace('_', '').replace('-', '')
+            deprovisionReason = None
+            if action in [
+                    'deprovisionsamemodelreplace',
+                    'deprovisionsamemodelreplacement'
+            ]:
+                action = 'deprovision'
+                deprovisionReason = 'same_model_replacement'
+            elif action in [
+                    'deprovisiondifferentmodelreplace',
+                    'deprovisiondifferentmodelreplacement'
+            ]:
+                action = 'deprovision'
+                deprovisionReason = 'different_model_replacement'
+            elif action in ['deprovisionretiringdevice']:
+                action = 'deprovision'
+                deprovisionReason = 'retiring_device'
+            elif action == 'deprovisionupgradetransfer':
+                action = 'deprovision'
+                deprovisionReason = 'upgrade_transfer'
+            elif action not in ['disable', 'reenable']:
+                controlflow.system_error_exit(2, f'expected action of ' \
+                    f'deprovision_same_model_replace, ' \
+                    f'deprovision_different_model_replace, ' \
+                    f'deprovision_retiring_device, ' \
+                    f'deprovision_upgrade_transfer, disable or reenable,'
+                    f' got {action}')
+            action_body = {'action': action}
+            if deprovisionReason:
+                action_body['deprovisionReason'] = deprovisionReason
+            i += 2
+        elif myarg == 'acknowledgedevicetouchrequirement':
+            ack_wipe = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update cros')
+    i = 0
+    count = len(devices)
+    if action_body:
+        if action_body['action'] == 'deprovision' and not ack_wipe:
+            print(f'WARNING: Refusing to deprovision {count} devices because '
+                  'acknowledge_device_touch_requirement not specified. ' \
+                  'Deprovisioning a device means the device will have to ' \
+                  'be physically wiped and re-enrolled to be managed by ' \
+                  'your domain again. This requires physical access to ' \
+                  'the device and is very time consuming to perform for ' \
+                  'each device. Please add ' \
+                  '"acknowledge_device_touch_requirement" to the GAM ' \
+                  'command if you understand this and wish to proceed ' \
+                  'with the deprovision. Please also be aware that ' \
+                  'deprovisioning can have an effect on your device ' \
+                  'license count. See ' \
+                  'https://support.google.com/chrome/a/answer/3523633 '\
+                  'for full details.')
+            sys.exit(3)
+        for deviceId in devices:
+            i += 1
+            cur_count = gam.currentCount(i, count)
+            print(f' performing action {action} for {deviceId}{cur_count}')
+            gapi.call(cd.chromeosdevices(),
+                      function='action',
+                      customerId=GC_Values[GC_CUSTOMER_ID],
+                      resourceId=deviceId,
+                      body=action_body)
+    else:
+        if update_body:
+            for deviceId in devices:
+                i += 1
+                current_count = gam.currentCount(i, count)
+                print(f' updating {deviceId}{current_count}')
+                gapi.call(cd.chromeosdevices(),
+                          'update',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          deviceId=deviceId,
+                          body=update_body)
+        if orgUnitPath:
+            # split moves into max 50 devices per batch
+            for l in range(0, len(devices), 50):
+                move_body = {'deviceIds': devices[l:l + 50]}
+                print(f' moving {len(move_body["deviceIds"])} devices to ' \
+                      f'{orgUnitPath}')
+                gapi.call(cd.chromeosdevices(),
+                          'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath,
+                          body=move_body)
+
+
+def doGetCrosInfo():
+    cd = gapi_directory.build()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    downloadfile = None
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    projection = None
+    fieldsList = []
+    noLists = False
+    startDate = endDate = None
+    listLimit = 0
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nolists':
+            noLists = True
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            fieldsList = []
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg])
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i + 1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field])
+                    if field in CROS_ACTIVE_TIME_RANGES_ARGUMENTS + \
+                                CROS_DEVICE_FILES_ARGUMENTS + \
+                                CROS_RECENT_USERS_ARGUMENTS:
+                        projection = 'FULL'
+                        noLists = False
+                else:
+                    controlflow.invalid_argument_exit(field,
+                                                      'gam info cros fields')
+            i += 2
+        elif myarg == 'downloadfile':
+            downloadfile = sys.argv[i + 1]
+            if downloadfile.lower() == 'latest':
+                downloadfile = downloadfile.lower()
+            i += 2
+        elif myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i + 1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam info cros')
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = ','.join(set(fieldsList)).replace('.', '/')
+    else:
+        fields = None
+    i = 0
+    device_count = len(devices)
+    for deviceId in devices:
+        i += 1
+        cros = gapi.call(cd.chromeosdevices(),
+                         'get',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         deviceId=deviceId,
+                         projection=projection,
+                         fields=fields)
+        print(f'CrOS Device: {deviceId} ({i} of {device_count})')
+        if 'notes' in cros:
+            cros['notes'] = cros['notes'].replace('\n', '\\n')
+        if 'autoUpdateExpiration' in cros:
+            cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                cros['autoUpdateExpiration'])
+        _checkTPMVulnerability(cros)
+        for up in CROS_SCALAR_PROPERTY_PRINT_ORDER:
+            if up in cros:
+                if isinstance(cros[up], str):
+                    print(f'  {up}: {cros[up]}')
+                else:
+                    sys.stdout.write(f'  {up}:')
+                    display.print_json(cros[up], '  ')
+        if not noLists:
+            activeTimeRanges = _filterTimeRanges(
+                cros.get('activeTimeRanges', []), startDate, endDate)
+            lenATR = len(activeTimeRanges)
+            if lenATR:
+                print('  activeTimeRanges')
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    active_date = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    duration = utils.formatMilliSeconds(active_time)
+                    minutes = active_time // 60000
+                    print(f'    date: {active_date}')
+                    print(f'      activeTime: {active_time}')
+                    print(f'      duration: {duration}')
+                    print(f'      minutes: {minutes}')
+            recentUsers = cros.get('recentUsers', [])
+            lenRU = len(recentUsers)
+            if lenRU:
+                print('  recentUsers')
+                num_ranges = min(lenRU, listLimit or lenRU)
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get('email')
+                    if not useremail:
+                        if recentUser['type'] == 'USER_TYPE_UNMANAGED':
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    print(f'    type: {recentUser["type"]}')
+                    print(f'      email: {useremail}')
+            deviceFiles = _filterCreateReportTime(cros.get('deviceFiles',
+                                                           []), 'createTime',
+                                                  startDate, endDate)
+            lenDF = len(deviceFiles)
+            if lenDF:
+                num_ranges = min(lenDF, listLimit or lenDF)
+                print('  deviceFiles')
+                for deviceFile in deviceFiles[:num_ranges]:
+                    device_type = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    print(f'    {device_type}: {create_time}')
+            if downloadfile:
+                deviceFiles = cros.get('deviceFiles', [])
+                lenDF = len(deviceFiles)
+                if lenDF:
+                    if downloadfile == 'latest':
+                        deviceFile = deviceFiles[-1]
+                    else:
+                        for deviceFile in deviceFiles:
+                            if deviceFile['createTime'] == downloadfile:
+                                break
+                        else:
+                            print(f'ERROR: file {downloadfile} not ' \
+                                  f'available to download.')
+                            deviceFile = None
+                    if deviceFile:
+                        created = deviceFile['createTime']
+                        downloadfile = f'cros-logs-{deviceId}-{created}.zip'
+                        downloadfilename = os.path.join(targetFolder,
+                                                        downloadfile)
+                        dl_url = deviceFile['downloadUrl']
+                        _, content = cd._http.request(dl_url)
+                        fileutils.write_file(downloadfilename,
+                                             content,
+                                             mode='wb',
+                                             continue_on_error=True)
+                        print(f'Downloaded: {downloadfilename}')
+                elif downloadfile:
+                    print('ERROR: no files to download.')
+            cpuStatusReports = _filterCreateReportTime(
+                cros.get('cpuStatusReports', []), 'reportTime', startDate,
+                endDate)
+            lenCSR = len(cpuStatusReports)
+            if lenCSR:
+                print('  cpuStatusReports')
+                num_ranges = min(lenCSR, listLimit or lenCSR)
+                for cpuStatusReport in cpuStatusReports[:num_ranges]:
+                    print(f'    reportTime: {cpuStatusReport["reportTime"]}')
+                    print('      cpuTemperatureInfo')
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        temp_label = tempInfo['label'].strip()
+                        temperature = tempInfo['temperature']
+                        print(f'        {temp_label}: {temperature}')
+                    if 'cpuUtilizationPercentageInfo' in cpuStatusReport:
+                        pct_info = cpuStatusReport['cpuUtilizationPercentageInfo']
+                        util = ','.join([str(x) for x in pct_info])
+                        print(f'      cpuUtilizationPercentageInfo: {util}')
+            diskVolumeReports = cros.get('diskVolumeReports', [])
+            lenDVR = len(diskVolumeReports)
+            if lenDVR:
+                print('  diskVolumeReports')
+                print('    volumeInfo')
+                num_ranges = min(lenDVR, listLimit or lenDVR)
+                for diskVolumeReport in diskVolumeReports[:num_ranges]:
+                    volumeInfo = diskVolumeReport['volumeInfo']
+                    for volume in volumeInfo:
+                        vid = volume['volumeId']
+                        vstorage_free = volume['storageFree']
+                        vstorage_total = volume['storageTotal']
+                        print(f'      volumeId: {vid}')
+                        print(f'        storageFree: {vstorage_free}')
+                        print(f'        storageTotal: {vstorage_total}')
+            systemRamFreeReports = _filterCreateReportTime(
+                cros.get('systemRamFreeReports', []), 'reportTime', startDate,
+                endDate)
+            lenSRFR = len(systemRamFreeReports)
+            if lenSRFR:
+                print('  systemRamFreeReports')
+                num_ranges = min(lenSRFR, listLimit or lenSRFR)
+                for systemRamFreeReport in systemRamFreeReports[:num_ranges]:
+                    report_time = systemRamFreeReport['reportTime']
+                    free_info = systemRamFreeReport['systemRamFreeInfo']
+                    free_ram = ','.join(free_info)
+                    print(f'    reportTime: {report_time}')
+                    print(f'      systemRamFreeInfo: {free_ram}')
+
+
+def doPrintCrosActivity():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber',
+        'orgUnitPath'
+    ]
+    csvRows = []
+    fieldsList = [
+        'deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber',
+        'orgUnitPath'
+    ]
+    startDate = endDate = None
+    selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False
+    listLimit = 0
+    delimiter = ','
+    orgUnitPath = None
+    queries = [None]
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectActiveTimeRanges = True
+            i += 1
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectDeviceFiles = True
+            i += 1
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectRecentUsers = True
+            i += 1
+        elif myarg == 'both':
+            selectActiveTimeRanges = selectRecentUsers = True
+            i += 1
+        elif myarg == 'all':
+            selectActiveTimeRanges = selectDeviceFiles = True
+            selectRecentUsers = True
+            i += 1
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=0)
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print crosactivity')
+    if not selectActiveTimeRanges and \
+       not selectDeviceFiles and \
+       not selectRecentUsers:
+        selectActiveTimeRanges = selectRecentUsers = True
+    if selectRecentUsers:
+        fieldsList.append('recentUsers')
+        display.add_titles_to_csv_file([
+            'recentUsers.email',
+        ], titles)
+    if selectActiveTimeRanges:
+        fieldsList.append('activeTimeRanges')
+        titles_to_add = [
+            'activeTimeRanges.date', 'activeTimeRanges.duration',
+            'activeTimeRanges.minutes'
+        ]
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    if selectDeviceFiles:
+        fieldsList.append('deviceFiles')
+        titles_to_add = ['deviceFiles.type', 'deviceFiles.createTime']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    fields = f'nextPageToken,chromeosdevices({",".join(fieldsList)})'
+    for query in queries:
+        gam.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(),
+                                      'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection='FULL',
+                                      fields=fields,
+                                      orgUnitPath=orgUnitPath)
+        for cros in all_cros:
+            row = {}
+            skip_attribs = ['recentUsers', 'activeTimeRanges', 'deviceFiles']
+            for attrib in cros:
+                if attrib not in skip_attribs:
+                    row[attrib] = cros[attrib]
+            if selectActiveTimeRanges:
+                activeTimeRanges = _filterTimeRanges(
+                    cros.get('activeTimeRanges', []), startDate, endDate)
+                lenATR = len(activeTimeRanges)
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['activeTimeRanges.date'] = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    newrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    newrow['activeTimeRanges.minutes'] = \
+                        activeTimeRange['activeTime']//60000
+                    csvRows.append(newrow)
+            if selectRecentUsers:
+                recentUsers = cros.get('recentUsers', [])
+                lenRU = len(recentUsers)
+                num_ranges = min(lenRU, listLimit or lenRU)
+                recent_users = []
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get('email')
+                    if not useremail:
+                        if recentUser['type'] == 'USER_TYPE_UNMANAGED':
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    recent_users.append(useremail)
+                row['recentUsers.email'] = delimiter.join(recent_users)
+                csvRows.append(row)
+            if selectDeviceFiles:
+                deviceFiles = _filterCreateReportTime(
+                    cros.get('deviceFiles', []), 'createTime', startDate,
+                    endDate)
+                lenDF = len(deviceFiles)
+                num_ranges = min(lenDF, listLimit or lenDF)
+                for deviceFile in deviceFiles[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['deviceFiles.type'] = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    newrow['deviceFiles.createTime'] = create_time
+                    csvRows.append(newrow)
+    display.write_csv_file(csvRows, titles, 'CrOS Activity', todrive)
+
+
+def _checkTPMVulnerability(cros):
+    if 'tpmVersionInfo' in cros and \
+       'firmwareVersion' in cros['tpmVersionInfo']:
+        firmware_version = cros['tpmVersionInfo']['firmwareVersion']
+        if firmware_version in CROS_TPM_VULN_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'VULNERABLE'
+        elif firmware_version in CROS_TPM_FIXED_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'UPDATED'
+        else:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'NOT IMPACTED'
+
+
+def doPrintCrosDevices():
+
+    def _getSelectedLists(myarg):
+        if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectedLists['activeTimeRanges'] = True
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectedLists['recentUsers'] = True
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectedLists['deviceFiles'] = True
+        elif myarg in CROS_CPU_STATUS_REPORTS_ARGUMENTS:
+            selectedLists['cpuStatusReports'] = True
+        elif myarg in CROS_DISK_VOLUME_REPORTS_ARGUMENTS:
+            selectedLists['diskVolumeReports'] = True
+        elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS:
+            selectedLists['systemRamFreeReports'] = True
+
+    cd = gapi_directory.build()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    display.add_field_to_csv_file('deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                  fieldsList, fieldsTitles, titles)
+    projection = orderBy = sortOrder = orgUnitPath = None
+    queries = [None]
+    noLists = sortHeaders = False
+    selectedLists = {}
+    startDate = endDate = None
+    listLimit = 0
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'nolists':
+            noLists = True
+            selectedLists = {}
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=0)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i + 1].lower().replace('_', '')
+            validOrderBy = [
+                'location', 'user', 'lastsync', 'notes', 'serialnumber',
+                'status', 'supportenddate'
+            ]
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(validOrderBy),
+                                                   orderBy)
+            if orderBy == 'location':
+                orderBy = 'annotatedLocation'
+            elif orderBy == 'user':
+                orderBy = 'annotatedUser'
+            elif orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'serialnumber':
+                orderBy = 'serialNumber'
+            elif orderBy == 'supportenddate':
+                orderBy = 'supportEndDate'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            sortHeaders = True
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            sortHeaders = True
+            fieldsList = []
+            i += 1
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in CROS_LISTS_ARGUMENTS:
+            _getSelectedLists(myarg)
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_fields_list(myarg,
+                                             CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                             fieldsList)
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i + 1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_LISTS_ARGUMENTS:
+                    _getSelectedLists(field)
+                elif field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    display.add_field_to_fields_list(
+                        field, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+                else:
+                    controlflow.invalid_argument_exit(field,
+                                                      'gam print cros fields')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print cros')
+    if selectedLists:
+        noLists = False
+        projection = 'FULL'
+        for selectList in selectedLists:
+            display.add_field_to_fields_list(selectList,
+                                             CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                             fieldsList)
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = f'nextPageToken,chromeosdevices({",".join(set(fieldsList))})'.replace(
+            '.', '/')
+    else:
+        fields = None
+    for query in queries:
+        gam.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(),
+                                      'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection=projection,
+                                      orgUnitPath=orgUnitPath,
+                                      orderBy=orderBy,
+                                      sortOrder=sortOrder,
+                                      fields=fields)
+        for cros in all_cros:
+            _checkTPMVulnerability(cros)
+        if not noLists and not selectedLists:
+            for cros in all_cros:
+                if 'notes' in cros:
+                    cros['notes'] = cros['notes'].replace('\n', '\\n')
+                if 'autoUpdateExpiration' in cros:
+                    cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                        cros['autoUpdateExpiration'])
+                for cpuStatusReport in cros.get('cpuStatusReports', []):
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        tempInfo['label'] = tempInfo['label'].strip()
+                display.add_row_titles_to_csv_file(
+                    utils.flatten_json(cros, listLimit=listLimit), csvRows,
+                    titles)
+            continue
+        for cros in all_cros:
+            if 'notes' in cros:
+                cros['notes'] = cros['notes'].replace('\n', '\\n')
+            if 'autoUpdateExpiration' in cros:
+                cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                    cros['autoUpdateExpiration'])
+            row = {}
+            for attrib in cros:
+                if attrib not in set([
+                        'kind', 'etag', 'tpmVersionInfo', 'recentUsers',
+                        'activeTimeRanges', 'deviceFiles', 'cpuStatusReports',
+                        'diskVolumeReports', 'systemRamFreeReports'
+                ]):
+                    row[attrib] = cros[attrib]
+            if selectedLists.get('activeTimeRanges'):
+                timergs = cros.get('activeTimeRanges', [])
+            else:
+                timergs = []
+            activeTimeRanges = _filterTimeRanges(timergs, startDate, endDate)
+            if selectedLists.get('recentUsers'):
+                recentUsers = cros.get('recentUsers', [])
+            else:
+                recentUsers = []
+            if selectedLists.get('deviceFiles'):
+                device_files = cros.get('deviceFiles', [])
+            else:
+                device_files = []
+            deviceFiles = _filterCreateReportTime(device_files, 'createTime',
+                                                  startDate, endDate)
+            if selectedLists.get('cpuStatusReports'):
+                cpu_reports = cros.get('cpuStatusReports', [])
+            else:
+                cpu_reports = []
+            cpuStatusReports = _filterCreateReportTime(cpu_reports,
+                                                       'reportTime', startDate,
+                                                       endDate)
+            if selectedLists.get('diskVolumeReports'):
+                diskVolumeReports = cros.get('diskVolumeReports', [])
+            else:
+                diskVolumeReports = []
+            if selectedLists.get('systemRamFreeReports'):
+                ram_reports = cros.get('systemRamFreeReports', [])
+            else:
+                ram_reports = []
+            systemRamFreeReports = _filterCreateReportTime(
+                ram_reports, 'reportTime', startDate, endDate)
+            if noLists or (not activeTimeRanges and \
+                           not recentUsers and \
+                           not deviceFiles and \
+                           not cpuStatusReports and \
+                           not diskVolumeReports and \
+                           not systemRamFreeReports):
+                display.add_row_titles_to_csv_file(row, csvRows, titles)
+                continue
+            lenATR = len(activeTimeRanges)
+            lenRU = len(recentUsers)
+            lenDF = len(deviceFiles)
+            lenCSR = len(cpuStatusReports)
+            lenDVR = len(diskVolumeReports)
+            lenSRFR = len(systemRamFreeReports)
+            max_len = max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR)
+            for i in range(min(max_len, listLimit or max_len)):
+                nrow = row.copy()
+                if i < lenATR:
+                    nrow['activeTimeRanges.date'] = \
+                        activeTimeRanges[i]['date']
+                    nrow['activeTimeRanges.activeTime'] = \
+                        str(activeTimeRanges[i]['activeTime'])
+                    active_time = activeTimeRanges[i]['activeTime']
+                    nrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    nrow['activeTimeRanges.minutes'] = active_time // 60000
+                if i < lenRU:
+                    nrow['recentUsers.type'] = recentUsers[i]['type']
+                    nrow['recentUsers.email'] = recentUsers[i].get('email')
+                    if not nrow['recentUsers.email']:
+                        if nrow['recentUsers.type'] == 'USER_TYPE_UNMANAGED':
+                            nrow['recentUsers.email'] = 'UnmanagedUser'
+                        else:
+                            nrow['recentUsers.email'] = 'Unknown'
+                if i < lenDF:
+                    nrow['deviceFiles.type'] = deviceFiles[i]['type']
+                    nrow['deviceFiles.createTime'] = \
+                        deviceFiles[i]['createTime']
+                if i < lenCSR:
+                    nrow['cpuStatusReports.reportTime'] = \
+                        cpuStatusReports[i]['reportTime']
+                    tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        label = tempInfo['label'].strip()
+                        base = 'cpuStatusReports.cpuTemperatureInfo.'
+                        nrow[f'{base}{label}'] = tempInfo['temperature']
+                    cpu_field = 'cpuUtilizationPercentageInfo'
+                    if cpu_field in cpuStatusReports[i]:
+                        cpu_reports = cpuStatusReports[i][cpu_field]
+                        cpu_pcts = [str(x) for x in cpu_reports]
+                        nrow[f'cpuStatusReports.{cpu_field}'] = ','.join(cpu_pcts)
+                if i < lenDVR:
+                    volumeInfo = diskVolumeReports[i]['volumeInfo']
+                    j = 0
+                    vfield = 'diskVolumeReports.volumeInfo.'
+                    for volume in volumeInfo:
+                        nrow[f'{vfield}{j}.volumeId'] = \
+                            volume['volumeId']
+                        nrow[f'{vfield}{j}.storageFree'] = \
+                            volume['storageFree']
+                        nrow[f'{vfield}{j}.storageTotal'] = \
+                            volume['storageTotal']
+                        j += 1
+                if i < lenSRFR:
+                    nrow['systemRamFreeReports.reportTime'] = \
+                        systemRamFreeReports[i]['reportTime']
+                    ram_reports = systemRamFreeReports[i]['systemRamFreeInfo']
+                    ram_info = [str(x) for x in ram_reports]
+                    nrow['systenRamFreeReports.systemRamFreeInfo'] = \
+                        ','.join(ram_info)
+                display.add_row_titles_to_csv_file(nrow, csvRows, titles)
+    if sortHeaders:
+        display.sort_csv_titles([
+            'deviceId',
+        ], titles)
+    display.write_csv_file(csvRows, titles, 'CrOS', todrive)
+
+
+def getCrOSDeviceEntity(i, cd):
+    myarg = sys.argv[i].lower()
+    if myarg == 'cros_sn':
+        return i + 2, gam.getUsersToModify('cros_sn', sys.argv[i + 1])
+    if myarg == 'query':
+        return i + 2, gam.getUsersToModify('crosquery', sys.argv[i + 1])
+    if myarg[:6] == 'query:':
+        query = sys.argv[i][6:]
+        if query[:12].lower() == 'orgunitpath:':
+            kwargs = {'orgUnitPath': query[12:]}
+        else:
+            kwargs = {'query': query}
+        fields = 'nextPageToken,chromeosdevices(deviceId)'
+        devices = gapi.get_all_pages(cd.chromeosdevices(),
+                                     'list',
+                                     'chromeosdevices',
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     fields=fields,
+                                     **kwargs)
+        return i + 1, [device['deviceId'] for device in devices]
+    return i + 1, sys.argv[i].replace(',', ' ').split()
+
+
+def _getFilterDate(dateStr):
+    return datetime.datetime.strptime(dateStr, YYYYMMDD_FORMAT)
+
+
+def _filterTimeRanges(activeTimeRanges, startDate, endDate):
+    if startDate is None and endDate is None:
+        return activeTimeRanges
+    filteredTimeRanges = []
+    for timeRange in activeTimeRanges:
+        activityDate = datetime.datetime.strptime(timeRange['date'],
+                                                  YYYYMMDD_FORMAT)
+        if ((startDate is None) or \
+            (activityDate >= startDate)) and \
+           ((endDate is None) or \
+            (activityDate <= endDate)):
+            filteredTimeRanges.append(timeRange)
+    return filteredTimeRanges
+
+
+def _filterCreateReportTime(items, timeField, startTime, endTime):
+    if startTime is None and endTime is None:
+        return items
+    filteredItems = []
+    time_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    for item in items:
+        timeValue = datetime.datetime.strptime(item[timeField], time_format)
+        if ((startTime is None) or \
+            (timeValue >= startTime)) and \
+           ((endTime is None) or \
+            (timeValue <= endTime)):
+            filteredItems.append(item)
+    return filteredItems

src/gam/gapi/directory/customer.py

@@ -0,0 +1,162 @@
+import datetime
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import reports as gapi_reports
+
+
+def _get_customerid():
+    customer = GC_Values[GC_CUSTOMER_ID]
+    if customer != MY_CUSTOMER and customer[0] != 'C':
+        customer = 'C' + customer
+    return customer
+
+def doGetCustomerInfo():
+    cd = gapi_directory.build()
+    customer_id = _get_customerid()
+    customer_info = gapi.call(cd.customers(),
+                              'get',
+                              customerKey=customer_id)
+    print(f'Customer ID: {customer_info["id"]}')
+    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    try:
+        result = gapi.call(
+            cd.domains(),
+            'get',
+            customer=customer_id,
+            domainName=customer_info['customerDomain'],
+            fields='verified',
+            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND])
+    except gapi.errors.GapiDomainNotFoundError:
+        result = {'verified': False}
+    print(f'Primary Domain Verified: {result["verified"]}')
+    # If customer has changed primary domain customerCreationTime is date
+    # of current primary being added, not customer create date.
+    # We should also get all domains and use oldest date
+    customer_creation = customer_info['customerCreationTime']
+    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    oldest = datetime.datetime.strptime(customer_creation, date_format)
+    domains = gapi.get_items(cd.domains(),
+                             'list',
+                             'domains',
+                             customer=customer_id,
+                             fields='domains(creationTime)')
+    for domain in domains:
+        creation_timestamp = int(domain['creationTime']) / 1000
+        domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
+        if domain_creation < oldest:
+            oldest = domain_creation
+    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
+    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    print(f'Default Language: {customer_language}')
+    if 'postalAddress' in customer_info:
+        print('Address:')
+        for field in ADDRESS_FIELDS_PRINT_ORDER:
+            if field in customer_info['postalAddress']:
+                print(f' {field}: {customer_info["postalAddress"][field]}')
+    if 'phoneNumber' in customer_info:
+        print(f'Phone: {customer_info["phoneNumber"]}')
+    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    user_counts_map = {
+        'accounts:num_users': 'Total Users',
+        'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',
+        'accounts:gsuite_basic_used_licenses': 'G Suite Basic Users',
+        'accounts:gsuite_enterprise_total_licenses': 'Workspace Enterprise Plus ' \
+        'Licenses',
+        'accounts:gsuite_enterprise_used_licenses': 'Workspace Enterprise Plus ' \
+        'Users',
+        'accounts:gsuite_unlimited_total_licenses': 'G Suite Business ' \
+        'Licenses',
+        'accounts:gsuite_unlimited_used_licenses': 'G Suite Business Users'
+    }
+    parameters = ','.join(list(user_counts_map))
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    reports_customer_id = customer_id
+    if reports_customer_id == MY_CUSTOMER:
+        reports_customer_id = None
+    rep = gapi_reports.build()
+    usage = None
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.FORBIDDEN
+    ]
+    while True:
+        try:
+            result = gapi.call(rep.customerUsageReports(),
+                               'get',
+                               throw_reasons=throw_reasons,
+                               customerId=reports_customer_id,
+                               date=tryDate,
+                               parameters=parameters)
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = gapi_reports._adjust_date(str(e))
+            continue
+        except gapi.errors.GapiForbiddenError:
+            return
+        warnings = result.get('warnings', [])
+        fullDataRequired = ['accounts']
+        usage = result.get('usageReports')
+        has_reports = bool(usage)
+        fullData, tryDate = gapi_reports._check_full_data_available(
+            warnings, tryDate, fullDataRequired, has_reports)
+        if fullData < 0:
+            print('No user report available.')
+            sys.exit(1)
+        if fullData == 0:
+            continue
+        break
+    print(f'User counts as of {tryDate}:')
+    for item in usage[0]['parameters']:
+        api_name = user_counts_map.get(item['name'])
+        api_value = int(item.get('intValue', 0))
+        if api_name and api_value:
+            print(f'  {api_name}: {api_value:,}')
+
+
+def doUpdateCustomer():
+    cd = gapi_directory.build()
+    body = {}
+    customer_id = _get_customerid()
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ADDRESS_FIELDS_ARGUMENT_MAP:
+            body.setdefault('postalAddress', {})
+            arg = ADDRESS_FIELDS_ARGUMENT_MAP[myarg]
+            body['postalAddress'][arg] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['adminsecondaryemail', 'alternateemail']:
+            body['alternateEmail'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['phone', 'phonenumber']:
+            body['phoneNumber'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'language':
+            body['language'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam update customer')
+    if not body:
+        controlflow.system_error_exit(
+            2, 'no arguments specified for "gam '
+            'update customer"')
+    gapi.call(cd.customers(),
+              'patch',
+              customerKey=customer_id,
+              body=body)
+    print('Updated customer')
+
+
+def setTrueCustomerId(cd=None):
+    customer_id = GC_Values[GC_CUSTOMER_ID]
+    if customer_id == MY_CUSTOMER:
+        if not cd:
+            cd = gapi_directory.build()
+        result = gapi.call(cd.customers(),
+                           'get',
+                            customerKey=customer_id,
+                            fields='id')
+        GC_Values[GC_CUSTOMER_ID] = result.get('id',
+                                               customer_id)

src/gam/gapi/directory/domainaliases.py

@@ -0,0 +1,76 @@
+import sys
+
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def create():
+    cd = gapi_directory.build()
+    body = {'domainAliasName': sys.argv[3], 'parentDomainName': sys.argv[4]}
+    print(f'Adding {body["domainAliasName"]} alias for ' \
+       f'{body["parentDomainName"]}')
+    gapi.call(cd.domainAliases(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def delete():
+    cd = gapi_directory.build()
+    domainAliasName = sys.argv[3]
+    print(f'Deleting domain alias {domainAliasName}')
+    gapi.call(cd.domainAliases(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              domainAliasName=domainAliasName)
+
+
+def info():
+    cd = gapi_directory.build()
+    alias = sys.argv[3]
+    result = gapi.call(cd.domainAliases(),
+                       'get',
+                       customer=GC_Values[GC_CUSTOMER_ID],
+                       domainAliasName=alias)
+    if 'creationTime' in result:
+        result['creationTime'] = utils.formatTimestampYMDHMSF(
+            result['creationTime'])
+    display.print_json(result)
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'domainAliasName',
+    ]
+    csvRows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print domainaliases')
+    results = gapi.call(cd.domainAliases(),
+                        'list',
+                        customer=GC_Values[GC_CUSTOMER_ID])
+    for domainAlias in results['domainAliases']:
+        domainAlias_attributes = {}
+        for attr in domainAlias:
+            if attr in ['kind', 'etag']:
+                continue
+            if attr == 'creationTime':
+                domainAlias[attr] = utils.formatTimestampYMDHMSF(
+                    domainAlias[attr])
+            if attr not in titles:
+                titles.append(attr)
+            domainAlias_attributes[attr] = domainAlias[attr]
+        csvRows.append(domainAlias_attributes)
+    display.write_csv_file(csvRows, titles, 'Domains', todrive)

src/gam/gapi/directory/domains.py

@@ -0,0 +1,124 @@
+import sys
+
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi.directory import customer as gapi_directory_customer
+from gam import utils
+
+
+def create():
+    cd = gapi_directory.build()
+    domain_name = sys.argv[3]
+    body = {'domainName': domain_name}
+    gapi.call(cd.domains(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print(f'Added domain {domain_name}')
+
+
+def info():
+    if (len(sys.argv) < 4) or (sys.argv[3] == 'logo'):
+        gapi_directory_customer.doGetCustomerInfo()
+        return
+    cd = gapi_directory.build()
+    domainName = sys.argv[3]
+    result = gapi.call(cd.domains(),
+                       'get',
+                       customer=GC_Values[GC_CUSTOMER_ID],
+                       domainName=domainName)
+    if 'creationTime' in result:
+        result['creationTime'] = utils.formatTimestampYMDHMSF(
+            result['creationTime'])
+    if 'domainAliases' in result:
+        for i in range(0, len(result['domainAliases'])):
+            if 'creationTime' in result['domainAliases'][i]:
+                result['domainAliases'][i][
+                    'creationTime'] = utils.formatTimestampYMDHMSF(
+                        result['domainAliases'][i]['creationTime'])
+    display.print_json(result)
+
+
+def update():
+    cd = gapi_directory.build()
+    domain_name = sys.argv[3]
+    i = 4
+    body = {}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'primary':
+            body['customerDomain'] = domain_name
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update domain')
+    gapi.call(cd.customers(),
+              'update',
+              customerKey=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print(f'{domain_name} is now the primary domain.')
+
+
+def delete():
+    cd = gapi_directory.build()
+    domainName = sys.argv[3]
+    print(f'Deleting domain {domainName}')
+    gapi.call(cd.domains(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              domainName=domainName)
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'domainName',
+    ]
+    csvRows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print domains')
+    results = gapi.call(cd.domains(),
+                        'list',
+                        customer=GC_Values[GC_CUSTOMER_ID])
+    for domain in results.get('domains', []):
+        domain_attributes = {}
+        domain['type'] = ['secondary', 'primary'][domain['isPrimary']]
+        for attr in domain:
+            if attr in ['kind', 'etag', 'domainAliases', 'isPrimary']:
+                continue
+            if attr in [
+                    'creationTime',
+            ]:
+                domain[attr] = utils.formatTimestampYMDHMSF(domain[attr])
+            if attr not in titles:
+                titles.append(attr)
+            domain_attributes[attr] = domain[attr]
+        csvRows.append(domain_attributes)
+        if 'domainAliases' in domain:
+            for aliasdomain in domain['domainAliases']:
+                aliasdomain['domainName'] = aliasdomain['domainAliasName']
+                del aliasdomain['domainAliasName']
+                aliasdomain['type'] = 'alias'
+                aliasdomain_attributes = {}
+                for attr in aliasdomain:
+                    if attr in ['kind', 'etag']:
+                        continue
+                    if attr in [
+                            'creationTime',
+                    ]:
+                        aliasdomain[attr] = utils.formatTimestampYMDHMSF(
+                            aliasdomain[attr])
+                    if attr not in titles:
+                        titles.append(attr)
+                    aliasdomain_attributes[attr] = aliasdomain[attr]
+                csvRows.append(aliasdomain_attributes)
+    display.write_csv_file(csvRows, titles, 'Domains', todrive)

src/gam/gapi/directory/mobiledevices.py

@@ -0,0 +1,237 @@
+import sys
+import uuid
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def delete():
+    cd = gapi_directory.build()
+    resourceId = sys.argv[3]
+    gapi.call(cd.mobiledevices(),
+              'delete',
+              resourceId=resourceId,
+              customerId=GC_Values[GC_CUSTOMER_ID])
+
+
+
+def info():
+    cd = gapi_directory.build()
+    resourceId = sys.argv[3]
+    device_info = gapi.call(cd.mobiledevices(),
+                            'get',
+                            customerId=GC_Values[GC_CUSTOMER_ID],
+                            resourceId=resourceId)
+    if 'deviceId' in device_info:
+        device_info['deviceId'] = device_info['deviceId'].encode('unicode-escape').decode(
+            UTF8)
+    attrib = 'securityPatchLevel'
+    if attrib in device_info and int(device_info[attrib]):
+        device_info[attrib] = utils.formatTimestampYMDHMS(device_info[attrib])
+    display.print_json(device_info)
+
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = []
+    csvRows = []
+    fields = None
+    projection = orderBy = sortOrder = None
+    queries = [None]
+    delimiter = ' '
+    listLimit = 1
+    appsLimit = -1
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['query', 'queries']:
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
+            i += 2
+        elif myarg == 'appslimit':
+            appsLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
+            i += 2
+        elif myarg == 'fields':
+            fields = f'nextPageToken,mobiledevices({sys.argv[i+1]})'
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i + 1].lower()
+            validOrderBy = [
+                'deviceid', 'email', 'lastsync', 'model', 'name', 'os',
+                'status', 'type'
+            ]
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(validOrderBy),
+                                                   orderBy)
+            if orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'deviceid':
+                orderBy = 'deviceId'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print mobile')
+    for query in queries:
+        gam.printGettingAllItems('Mobile Devices', query)
+        page_message = gapi.got_total_items_msg('Mobile Devices', '...\n')
+        all_mobile = gapi.get_all_pages(cd.mobiledevices(),
+                                        'list',
+                                        'mobiledevices',
+                                        page_message=page_message,
+                                        customerId=GC_Values[GC_CUSTOMER_ID],
+                                        query=query,
+                                        projection=projection,
+                                        fields=fields,
+                                        orderBy=orderBy,
+                                        sortOrder=sortOrder)
+        for mobile in all_mobile:
+            row = {}
+            for attrib in mobile:
+                if attrib in ['kind', 'etag']:
+                    continue
+                if attrib in ['name', 'email', 'otherAccountsInfo']:
+                    if attrib not in titles:
+                        titles.append(attrib)
+                    if listLimit > 0:
+                        row[attrib] = delimiter.join(
+                            mobile[attrib][0:listLimit])
+                    elif listLimit == 0:
+                        row[attrib] = delimiter.join(mobile[attrib])
+                elif attrib == 'applications':
+                    if appsLimit >= 0:
+                        if attrib not in titles:
+                            titles.append(attrib)
+                        applications = []
+                        j = 0
+                        for app in mobile[attrib]:
+                            j += 1
+                            if appsLimit and (j > appsLimit):
+                                break
+                            appDetails = []
+                            for field in [
+                                    'displayName', 'packageName', 'versionName'
+                            ]:
+                                appDetails.append(app.get(field, '<None>'))
+                            appDetails.append(
+                                str(app.get('versionCode', '<None>')))
+                            permissions = app.get('permission', [])
+                            if permissions:
+                                appDetails.append('/'.join(permissions))
+                            else:
+                                appDetails.append('<None>')
+                            applications.append('-'.join(appDetails))
+                        row[attrib] = delimiter.join(applications)
+                else:
+                    if attrib not in titles:
+                        titles.append(attrib)
+                    if attrib == 'deviceId':
+                        row[attrib] = mobile[attrib].encode(
+                            'unicode-escape').decode(UTF8)
+                    elif attrib == 'securityPatchLevel' and int(mobile[attrib]):
+                        row[attrib] = utils.formatTimestampYMDHMS(
+                            mobile[attrib])
+                    else:
+                        row[attrib] = mobile[attrib]
+            csvRows.append(row)
+    display.sort_csv_titles(
+        ['resourceId', 'deviceId', 'serialNumber', 'name', 'email', 'status'],
+        titles)
+    display.write_csv_file(csvRows, titles, 'Mobile', todrive)
+
+
+def update():
+    cd = gapi_directory.build()
+    resourceIds = sys.argv[3]
+    match_users = None
+    doit = False
+    if resourceIds[:6] == 'query:':
+        query = resourceIds[6:]
+        fields = 'nextPageToken,mobiledevices(resourceId,email)'
+        page_message = gapi.got_total_items_msg('Mobile Devices', '...\n')
+        devices = gapi.get_all_pages(cd.mobiledevices(),
+                                     'list',
+                                     page_message=page_message,
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     items='mobiledevices',
+                                     query=query,
+                                     fields=fields)
+    else:
+        devices = [{'resourceId': resourceIds, 'email': ['not set']}]
+        doit = True
+    i = 4
+    body = {}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            body['action'] = sys.argv[i + 1].lower()
+            validActions = [
+                'wipe', 'wipeaccount', 'accountwipe', 'wipe_account',
+                'account_wipe', 'approve', 'block',
+                'cancel_remote_wipe_then_activate',
+                'cancel_remote_wipe_then_block'
+            ]
+            if body['action'] not in validActions:
+                controlflow.expected_argument_exit('action',
+                                                   ', '.join(validActions),
+                                                   body['action'])
+            if body['action'] == 'wipe':
+                body['action'] = 'admin_remote_wipe'
+            elif body['action'].replace('_',
+                                        '') in ['accountwipe', 'wipeaccount']:
+                body['action'] = 'admin_account_wipe'
+            i += 2
+        elif myarg in ['ifusers', 'matchusers']:
+            match_users = gam.getUsersToModify(entity_type=sys.argv[i + 1].lower(),
+                                           entity=sys.argv[i + 2])
+            i += 3
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update mobile')
+    if body:
+        if doit:
+            print(f'Updating {len(devices)} devices')
+            describe_as = 'Performing'
+        else:
+            print(
+                f'Showing {len(devices)} changes that would be made, not actually making changes because doit argument not specified'
+            )
+            describe_as = 'Would perform'
+        for device in devices:
+            device_user = device.get('email', [''])[0]
+            if match_users and device_user not in match_users:
+                print(
+                    f'Skipping device for user {device_user} that did not match match_users argument'
+                )
+            else:
+                print(
+                    f'{describe_as} {body["action"]} on user {device_user} device {device["resourceId"]}'
+                )
+                if doit:
+                    gapi.call(cd.mobiledevices(),
+                              'action',
+                              resourceId=device['resourceId'],
+                              body=body,
+                              customerId=GC_Values[GC_CUSTOMER_ID])

src/gam/gapi/directory/orgunits.py

@@ -0,0 +1,419 @@
+import sys
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import errors as gapi_errors
+
+
+def create():
+    cd = gapi_directory.build()
+    name = getOrgUnitItem(sys.argv[3], pathOnly=True, absolutePath=False)
+    parent = ''
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'description':
+            body['description'] = sys.argv[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'parent':
+            parent = getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'noinherit':
+            body['blockInheritance'] = True
+            i += 1
+        elif myarg == 'inherit':
+            body['blockInheritance'] = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create org')
+    if parent.startswith('id:'):
+        parent = gapi.call(cd.orgunits(),
+                           'get',
+                           customerId=GC_Values[GC_CUSTOMER_ID],
+                           orgUnitPath=parent,
+                           fields='orgUnitPath')['orgUnitPath']
+    if parent == '/':
+        orgUnitPath = parent + name
+    else:
+        orgUnitPath = parent + '/' + name
+    if orgUnitPath.count('/') > 1:
+        body['parentOrgUnitPath'], body['name'] = orgUnitPath.rsplit('/', 1)
+    else:
+        body['parentOrgUnitPath'] = '/'
+        body['name'] = orgUnitPath[1:]
+    parent = body['parentOrgUnitPath']
+    gapi.call(cd.orgunits(),
+              'insert',
+              customerId=GC_Values[GC_CUSTOMER_ID],
+              body=body,
+              retry_reasons=[gapi_errors.ErrorReason.DAILY_LIMIT_EXCEEDED])
+    print(f'Created OrgUnit {body["name"]}')
+
+
+def delete():
+    cd = gapi_directory.build()
+    name = getOrgUnitItem(sys.argv[3])
+    print(f'Deleting organization {name}')
+    gapi.call(cd.orgunits(),
+              'delete',
+              customerId=GC_Values[GC_CUSTOMER_ID],
+              orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(name)))
+
+
+def info(name=None, return_attrib=None):
+    cd = gapi_directory.build()
+    checkSuspended = None
+    if not name:
+        name = getOrgUnitItem(sys.argv[3])
+        get_users = True
+        show_children = False
+        i = 4
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower()
+            if myarg == 'nousers':
+                get_users = False
+                i += 1
+            elif myarg in ['children', 'child']:
+                show_children = True
+                i += 1
+            elif myarg in ['suspended', 'notsuspended']:
+                checkSuspended = myarg == 'suspended'
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i], 'gam info org')
+    if name == '/':
+        orgs = gapi.call(cd.orgunits(),
+                         'list',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         type='children',
+                         fields='organizationUnits/parentOrgUnitId')
+        if 'organizationUnits' in orgs and orgs['organizationUnits']:
+            name = orgs['organizationUnits'][0]['parentOrgUnitId']
+        else:
+            topLevelOrgId = getTopLevelOrgId(cd, '/')
+            if topLevelOrgId:
+                name = topLevelOrgId
+    else:
+        name = makeOrgUnitPathRelative(name)
+    result = gapi.call(cd.orgunits(),
+                       'get',
+                       customerId=GC_Values[GC_CUSTOMER_ID],
+                       orgUnitPath=encodeOrgUnitPath(name))
+    if return_attrib:
+        return result[return_attrib]
+    display.print_json(result)
+    if get_users:
+        name = result['orgUnitPath']
+        page_message = gapi.got_total_items_first_last_msg('Users')
+        users = gapi.get_all_pages(
+            cd.users(),
+            'list',
+            'users',
+            page_message=page_message,
+            message_attribute='primaryEmail',
+            customer=GC_Values[GC_CUSTOMER_ID],
+            query=orgUnitPathQuery(name, checkSuspended),
+            fields='users(primaryEmail,orgUnitPath),nextPageToken')
+        if checkSuspended is None:
+            print('Users:')
+        elif not checkSuspended:
+            print('Users (Not suspended):')
+        else:
+            print('Users (Suspended):')
+        for user in users:
+            if show_children or (name.lower() == user['orgUnitPath'].lower()):
+                sys.stdout.write(f' {user["primaryEmail"]}')
+                if name.lower() != user['orgUnitPath'].lower():
+                    print(' (child)')
+                else:
+                    print('')
+
+
+def print_():
+    print_order = [
+        'orgUnitPath', 'orgUnitId', 'name', 'description', 'parentOrgUnitPath',
+        'parentOrgUnitId', 'blockInheritance'
+    ]
+    cd = gapi_directory.build()
+    listType = 'all'
+    orgUnitPath = '/'
+    todrive = False
+    fields = ['orgUnitPath', 'name', 'orgUnitId', 'parentOrgUnitId']
+    titles = []
+    csvRows = []
+    parentOrgIds = []
+    retrievedOrgIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'toplevelonly':
+            listType = 'children'
+            i += 1
+        elif myarg == 'fromparent':
+            orgUnitPath = getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg == 'fields':
+            fields += sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print orgs')
+    gam.printGettingAllItems('Organizational Units', None)
+    if fields:
+        get_fields = ','.join(fields)
+        list_fields = f'organizationUnits({get_fields})'
+    else:
+        list_fields = None
+        get_fields = None
+    orgs = gapi.call(cd.orgunits(),
+                     'list',
+                     customerId=GC_Values[GC_CUSTOMER_ID],
+                     type=listType,
+                     orgUnitPath=orgUnitPath,
+                     fields=list_fields)
+    if not 'organizationUnits' in orgs:
+        topLevelOrgId = getTopLevelOrgId(cd, orgUnitPath)
+        if topLevelOrgId:
+            parentOrgIds.append(topLevelOrgId)
+        orgunits = []
+    else:
+        orgunits = orgs['organizationUnits']
+    for row in orgunits:
+        retrievedOrgIds.append(row['orgUnitId'])
+        if row['parentOrgUnitId'] not in parentOrgIds:
+            parentOrgIds.append(row['parentOrgUnitId'])
+    missing_parents = set(parentOrgIds) - set(retrievedOrgIds)
+    for missing_parent in missing_parents:
+        try:
+            result = gapi.call(cd.orgunits(),
+                               'get',
+                               throw_reasons=['required'],
+                               customerId=GC_Values[GC_CUSTOMER_ID],
+                               orgUnitPath=missing_parent,
+                               fields=get_fields)
+            orgunits.append(result)
+        except:
+            pass
+    for row in orgunits:
+        orgEntity = {}
+        for key, value in list(row.items()):
+            if key in ['kind', 'etag', 'etags']:
+                continue
+            if key not in titles:
+                titles.append(key)
+            orgEntity[key] = value
+        csvRows.append(orgEntity)
+    for title in titles:
+        if title not in print_order:
+            print_order.append(title)
+    titles = sorted(titles, key=print_order.index)
+    # sort results similar to how they list in admin console
+    csvRows.sort(key=lambda x: x['orgUnitPath'].lower(), reverse=False)
+    display.write_csv_file(csvRows, titles, 'Orgs', todrive)
+
+
+def update():
+    cd = gapi_directory.build()
+    orgUnitPath = getOrgUnitItem(sys.argv[3])
+    if sys.argv[4].lower() in ['move', 'add']:
+        entity_type = sys.argv[5].lower()
+        if entity_type in usergroup_types:
+            users = gam.getUsersToModify(entity_type=entity_type,
+                                         entity=sys.argv[6])
+        else:
+            entity_type = 'users'
+            users = gam.getUsersToModify(entity_type=entity_type,
+                                         entity=sys.argv[5])
+        if (entity_type.startswith('cros')) or (
+            (entity_type == 'all') and (sys.argv[6].lower() == 'cros')):
+            for l in range(0, len(users), 50):
+                move_body = {'deviceIds': users[l:l + 50]}
+                print(
+                    f' moving {len(move_body["deviceIds"])} devices to {orgUnitPath}'
+                )
+                gapi.call(cd.chromeosdevices(),
+                          'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath,
+                          body=move_body)
+        else:
+            i = 0
+            count = len(users)
+            for user in users:
+                i += 1
+                sys.stderr.write(
+                    f' moving {user} to {orgUnitPath}{gam.currentCountNL(i, count)}'
+                )
+                try:
+                    gapi.call(cd.users(),
+                              'update',
+                              throw_reasons=[
+                                  gapi_errors.ErrorReason.CONDITION_NOT_MET
+                              ],
+                              userKey=user,
+                              body={'orgUnitPath': orgUnitPath})
+                except gapi_errors.GapiConditionNotMetError:
+                    pass
+    else:
+        body = {}
+        i = 4
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower()
+            if myarg == 'name':
+                body['name'] = sys.argv[i + 1]
+                i += 2
+            elif myarg == 'description':
+                body['description'] = sys.argv[i + 1].replace('\\n', '\n')
+                i += 2
+            elif myarg == 'parent':
+                parent = getOrgUnitItem(sys.argv[i + 1])
+                if parent.startswith('id:'):
+                    body['parentOrgUnitId'] = parent
+                else:
+                    body['parentOrgUnitPath'] = parent
+                i += 2
+            elif myarg == 'noinherit':
+                body['blockInheritance'] = True
+                i += 1
+            elif myarg == 'inherit':
+                body['blockInheritance'] = False
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i], 'gam update org')
+        gapi.call(cd.orgunits(),
+                  'update',
+                  customerId=GC_Values[GC_CUSTOMER_ID],
+                  orgUnitPath=encodeOrgUnitPath(
+                      makeOrgUnitPathRelative(orgUnitPath)),
+                  body=body)
+
+
+def orgUnitPathQuery(path, checkSuspended):
+    query = "orgUnitPath='{0}'".format(path.replace(
+        "'", "\\'")) if path != '/' else ''
+    if checkSuspended is not None:
+        query += f' isSuspended={checkSuspended}'
+    return query
+
+
+def makeOrgUnitPathAbsolute(path):
+    if path == '/':
+        return path
+    if path.startswith('/'):
+        return path.rstrip('/')
+    if path.startswith('id:'):
+        return path
+    if path.startswith('uid:'):
+        return path[1:]
+    return '/' + path.rstrip('/')
+
+
+def makeOrgUnitPathRelative(path):
+    if path == '/':
+        return path
+    if path.startswith('/'):
+        return path[1:].rstrip('/')
+    if path.startswith('id:'):
+        return path
+    if path.startswith('uid:'):
+        return path[1:]
+    return path.rstrip('/')
+
+
+def encodeOrgUnitPath(path):
+    if path.find('+') == -1 and path.find('%') == -1:
+        return path
+    encpath = ''
+    for c in path:
+        if c == '+':
+            encpath += '%2B'
+        elif c == '%':
+            encpath += '%25'
+        else:
+            encpath += c
+    return encpath
+
+
+def getOrgUnitItem(orgUnit, pathOnly=False, absolutePath=True):
+    if pathOnly and (orgUnit.startswith('id:') or orgUnit.startswith('uid:')):
+        controlflow.system_error_exit(
+            2, f'{orgUnit} is not valid in this context')
+    if absolutePath:
+        return makeOrgUnitPathAbsolute(orgUnit)
+    return makeOrgUnitPathRelative(orgUnit)
+
+
+def getTopLevelOrgId(cd, orgUnitPath):
+    try:
+        # create a temp org so we can learn what the top level org ID is (sigh)
+        temp_org = gapi.call(cd.orgunits(),
+                             'insert',
+                             customerId=GC_Values[GC_CUSTOMER_ID],
+                             body={
+                                 'name': 'temp-delete-me',
+                                 'parentOrgUnitPath': orgUnitPath
+                             },
+                             fields='parentOrgUnitId,orgUnitId')
+        gapi.call(cd.orgunits(),
+                  'delete',
+                  customerId=GC_Values[GC_CUSTOMER_ID],
+                  orgUnitPath=temp_org['orgUnitId'])
+        return temp_org['parentOrgUnitId']
+    except:
+        pass
+    return None
+
+
+def getOrgUnitId(orgUnit, cd=None):
+    if cd is None:
+        cd = gapi_directory.build()
+    orgUnit = getOrgUnitItem(orgUnit)
+    if orgUnit[:3] == 'id:':
+        return (orgUnit, orgUnit)
+    if orgUnit == '/':
+        result = gapi.call(cd.orgunits(),
+                           'list',
+                           customerId=GC_Values[GC_CUSTOMER_ID],
+                           orgUnitPath='/',
+                           type='children',
+                           fields='organizationUnits(parentOrgUnitId)')
+        if result.get('organizationUnits', []):
+            return (orgUnit, result['organizationUnits'][0]['parentOrgUnitId'])
+        topLevelOrgId = getTopLevelOrgId(cd, '/')
+        if topLevelOrgId:
+            return (orgUnit, topLevelOrgId)
+        return (orgUnit, '/')  #Bogus but should never happen
+    result = gapi.call(cd.orgunits(),
+                       'get',
+                       customerId=GC_Values[GC_CUSTOMER_ID],
+                       orgUnitPath=encodeOrgUnitPath(
+                           makeOrgUnitPathRelative(orgUnit)),
+                       fields='orgUnitId')
+    return (orgUnit, result['orgUnitId'])
+
+
+def orgunit_from_orgunitid(orgunitid, cd=None):
+    if cd is None:
+        cd = gapi_directory.build()
+    orgunitpath = GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME].get(orgunitid)
+    if not orgunitpath:
+        try:
+            orgunitpath = gapi.call(cd.orgunits(),
+                                    'get',
+                                    customerId=GC_Values[GC_CUSTOMER_ID],
+                                    orgUnitPath=f'id:{orgunitid}' if not orgunitid.startswith('id:') else orgunitid,
+                                    fields='orgUnitPath')['orgUnitPath']
+        except:
+            orgunitpath = orgunitid
+        GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][orgunitid] = orgunitpath
+    return orgunitpath

src/gam/gapi/directory/printers.py

@@ -0,0 +1,187 @@
+'''Commands to manage directory printers.'''
+# pylint: disable=unused-wildcard-import wildcard-import
+
+import sys
+
+import gam
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.var import *
+from gam.gapi import directory as gapi_directory
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+
+
+def _get_customerid():
+    ''' returns customer in "customers/C{customer}" format needed for this API'''
+    customer = GC_Values[GC_CUSTOMER_ID]
+    if customer != MY_CUSTOMER and customer[0] != 'C':
+        customer = 'C' + customer
+    return f'customers/{customer}'
+
+def _get_printer_attributes(i, cdapi=None):
+    '''get printer attributes for create/update commands'''
+    body = {}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'displayname':
+            body['displayName'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'makeandmodel':
+            body['makeAndModel'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['ou', 'org', 'orgunit', 'orgunitid']:
+            _, body['orgUnitId'] = gapi_directory_orgunits.getOrgUnitId(sys.argv[i+1], cdapi)
+            body['orgUnitId'] = body['orgUnitId'][3:]
+            i += 2
+        elif myarg == 'uri':
+            body['uri'] = sys.argv[i+1]
+            i += 2
+        elif myarg in {'driverless', 'usedriverlessconfig'}:
+            body['useDriverlessConfig'] = True
+            i += 1
+    return body
+
+
+def create():
+    '''gam create printer'''
+    cdapi = gapi_directory.build()
+    parent = _get_customerid()
+    body = _get_printer_attributes(3, cdapi)
+    result = gapi.call(cdapi.customers().chrome().printers(),
+                        'create',
+                        parent=parent,
+                        body=body)
+    display.print_json(result)
+
+
+def delete():
+    '''gam delete printer <PrinterIDList>|(file <FileName>)|(csvfile <FileName>:<FieldName>)'''
+    cdapi = gapi_directory.build()
+    customer_id = _get_customerid()
+    printer_id = sys.argv[3]
+    if printer_id.lower() not in {'file', 'csvfile'}:
+        printer_ids = printer_id.replace(',', ' ').split()
+    else:
+        printer_ids = gam.getUsersToModify(f'cros{printer_id.lower()}', sys.argv[4])
+    # max 50 per API call
+    batch_size = 50
+    for chunk in range(0, len(printer_ids), batch_size):
+        body = {
+                 'printerIds': printer_ids[chunk:chunk + batch_size]
+               }
+        result = gapi.call(cdapi.customers().chrome().printers(),
+                           'batchDeletePrinters',
+                           parent=customer_id,
+                           body=body)
+        for printer_id in result.get('printerIds', []):
+            print(f'Deleted printer {printer_id}')
+        for printer_id in result.get('failedPrinters', []):
+            print(f'ERROR: failed to delete {printer_id.get("printerIds")}')
+
+
+def info():
+    '''gam info printer'''
+    cdapi = gapi_directory.build()
+    customer = _get_customerid()
+    printer_id = sys.argv[3]
+    name = f'{customer}/chrome/printers/{printer_id}'
+    printer = gapi.call(cdapi.customers().chrome().printers(),
+                       'get',
+                       name=name)
+    if 'orgUnitId' in printer:
+        printer['orgUnitPath'] = gapi_directory_orgunits.orgunit_from_orgunitid(
+            printer['orgUnitId'], cdapi)
+    display.print_json(printer)
+
+
+def print_():
+    '''gam print printers'''
+    cdapi = gapi_directory.build()
+    parent = _get_customerid()
+    filter_ = None
+    todrive = False
+    titles = []
+    rows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'filter':
+            filter_ = sys.argv[i+1]
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print printermodels')
+    printers = gapi.get_all_pages(cdapi.customers().chrome().printers(),
+                                  'list',
+                                  items='printers',
+                                  parent=parent,
+                                  filter=filter_)
+    for printer in printers:
+        if 'orgUnitId' in printer:
+            printer['orgUnitPath'] = gapi_directory_orgunits.orgunit_from_orgunitid(
+                printer['orgUnitId'], cdapi)
+        row = {}
+        for key, val in printer.items():
+            if key not in titles:
+                titles.append(key)
+            row[key] = val
+        rows.append(row)
+    display.write_csv_file(rows, titles, 'Printers', todrive)
+
+
+def print_models():
+    '''gam print printermodels'''
+    cdapi = gapi_directory.build()
+    parent = _get_customerid()
+    filter_ = None
+    todrive = False
+    titles = []
+    rows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'filter':
+            filter_ = sys.argv[i+1]
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print printermodels')
+    models = gapi.get_all_pages(cdapi.customers().chrome().printers(),
+                                  'listPrinterModels',
+                                  items='printerModels',
+                                  parent=parent,
+                                  pageSize=10000,
+                                  filter=filter_)
+    for model in models:
+        row = {}
+        for key, val in model.items():
+            if key not in titles:
+                titles.append(key)
+            row[key] = val
+        rows.append(row)
+    display.write_csv_file(rows, titles, 'Printer Models', todrive)
+
+
+def update():
+    '''gam update printer'''
+    cdapi = gapi_directory.build()
+    customer = _get_customerid()
+    printer_id = sys.argv[3]
+    name = f'{customer}/chrome/printers/{printer_id}'
+    body = _get_printer_attributes(4, cdapi)
+    update_mask = ','.join(body)
+    # note clearMask seems unnecessary. Updating field to '' clears it.
+    result = gapi.call(cdapi.customers().chrome().printers(),
+                       'patch',
+                       name=name,
+                       updateMask=update_mask,
+                       body=body)
+    display.print_json(result)

src/gam/gapi/directory/privileges.py

@@ -0,0 +1,32 @@
+from gam.var import GC_Values, GC_CUSTOMER_ID
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+
+
+def flatten_privilege_list(privs, parent=None):
+    flat_privs = []
+    for priv in privs:
+        children = []
+        if parent:
+            priv['parent'] = parent
+        if priv.get('childPrivileges'):
+            children = flatten_privilege_list(priv['childPrivileges'],
+                                              parent=priv['privilegeName'])
+            priv['children'] = ' '.join(
+                [child['privilegeName'] for child in children])
+            del priv['childPrivileges']
+        flat_privs = flat_privs + children
+        flat_privs.append(priv)
+    return flat_privs
+
+
+def print_(return_only=False):
+    cd = gapi_directory.build()
+    privs = gapi.call(cd.privileges(),
+                      'list',
+                      customer=GC_Values[GC_CUSTOMER_ID])
+    privs = flatten_privilege_list(privs.get('items', []))
+    if return_only:
+        return privs
+    display.print_json(privs)

src/gam/gapi/directory/resource.py

@@ -0,0 +1,534 @@
+import sys
+import uuid
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def printBuildings():
+    to_drive = False
+    cd = gapi_directory.build()
+    titles = []
+    csvRows = []
+    fieldsList = ['buildingId']
+    # buildings.list() currently doesn't support paging
+    # but should soon, attempt to use it now so we
+    # won't break when it's turned on.
+    fields = 'nextPageToken,buildings(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Building']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        # Allows shorter arguments like "name" instead of "buildingname"
+        elif 'building' + myarg in possible_fields:
+            fieldsList.append(possible_fields['building' + myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print buildings')
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    buildings = gapi.get_all_pages(cd.resources().buildings(),
+                                   'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    for building in buildings:
+        building.pop('etags', None)
+        building.pop('etag', None)
+        building.pop('kind', None)
+        if 'buildingId' in building:
+            building['buildingId'] = f'id:{building["buildingId"]}'
+        if 'floorNames' in building:
+            building['floorNames'] = ','.join(building['floorNames'])
+        building = utils.flatten_json(building)
+        for item in building:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(building)
+    display.sort_csv_titles('buildingId', titles)
+    display.write_csv_file(csvRows, titles, 'Buildings', to_drive)
+
+
+def printResourceCalendars():
+    cd = gapi_directory.build()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    query = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'allfields':
+            fieldsList = []
+            fieldsTitles = {}
+            titles = []
+            for field in RESCAL_ALLFIELDS:
+                display.add_field_to_csv_file(field,
+                                              RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                              fieldsList, fieldsTitles, titles)
+            i += 1
+        elif myarg in RESCAL_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_csv_file(myarg,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print resources')
+    if not fieldsList:
+        for field in RESCAL_DFLTFIELDS:
+            display.add_field_to_csv_file(field,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+    fields = f'nextPageToken,items({",".join(set(fieldsList))})'
+    if 'buildingId' in fieldsList:
+        display.add_field_to_csv_file('buildingName',
+                                      {'buildingName': ['buildingName',]},
+                                      fieldsList, fieldsTitles, titles)
+    gam.printGettingAllItems('Resource Calendars', None)
+    page_message = gapi.got_total_items_first_last_msg('Resource Calendars')
+    resources = gapi.get_all_pages(cd.resources().calendars(),
+                                   'list',
+                                   'items',
+                                   page_message=page_message,
+                                   message_attribute='resourceId',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   query=query,
+                                   fields=fields)
+    for resource in resources:
+        if 'featureInstances' in resource:
+            features = [a_feature['feature']['name'] for \
+                a_feature in resource['featureInstances']]
+            resource['featureInstances'] = ','.join(features)
+        if 'buildingId' in resource:
+            resource['buildingName'] = getBuildingNameById(
+                cd, resource['buildingId'])
+            resource['buildingId'] = f'id:{resource["buildingId"]}'
+        resUnit = {}
+        for field in fieldsList:
+            resUnit[fieldsTitles[field]] = resource.get(field, '')
+        csvRows.append(resUnit)
+    display.sort_csv_titles(['resourceId', 'resourceName', 'resourceEmail'],
+                            titles)
+    display.write_csv_file(csvRows, titles, 'Resources', todrive)
+
+
+RESCAL_DFLTFIELDS = [
+    'id',
+    'name',
+    'email',
+]
+RESCAL_ALLFIELDS = [
+    'id',
+    'name',
+    'email',
+    'description',
+    'type',
+    'buildingid',
+    'category',
+    'capacity',
+    'features',
+    'floor',
+    'floorsection',
+    'generatedresourcename',
+    'uservisibledescription',
+]
+
+RESCAL_ARGUMENT_TO_PROPERTY_MAP = {
+    'description': ['resourceDescription'],
+    'building': ['buildingId',],
+    'buildingid': ['buildingId',],
+    'capacity': ['capacity',],
+    'category': ['resourceCategory',],
+    'email': ['resourceEmail'],
+    'feature': ['featureInstances',],
+    'features': ['featureInstances',],
+    'floor': ['floorName',],
+    'floorname': ['floorName',],
+    'floorsection': ['floorSection',],
+    'generatedresourcename': ['generatedResourceName',],
+    'id': ['resourceId'],
+    'name': ['resourceName'],
+    'type': ['resourceType'],
+    'userdescription': ['userVisibleDescription',],
+    'uservisibledescription': ['userVisibleDescription',],
+}
+
+
+def printFeatures():
+    to_drive = False
+    cd = gapi_directory.build()
+    titles = []
+    csvRows = []
+    fieldsList = ['name']
+    fields = 'nextPageToken,features(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Feature']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        elif 'feature' + myarg in possible_fields:
+            fieldsList.append(possible_fields['feature' + myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print features')
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    features = gapi.get_all_pages(cd.resources().features(),
+                                  'list',
+                                  'features',
+                                  customer=GC_Values[GC_CUSTOMER_ID],
+                                  fields=fields)
+    for feature in features:
+        feature.pop('etags', None)
+        feature.pop('etag', None)
+        feature.pop('kind', None)
+        feature = utils.flatten_json(feature)
+        for item in feature:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(feature)
+    display.sort_csv_titles('name', titles)
+    display.write_csv_file(csvRows, titles, 'Features', to_drive)
+
+
+def _getBuildingAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'id':
+            body['buildingId'] = args[i + 1]
+            i += 2
+        elif myarg == 'name':
+            body['buildingName'] = args[i + 1]
+            i += 2
+        elif myarg in ['lat', 'latitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['latitude'] = args[i + 1]
+            i += 2
+        elif myarg in ['long', 'lng', 'longitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['longitude'] = args[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = args[i + 1]
+            i += 2
+        elif myarg == 'floors':
+            body['floorNames'] = args[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg,
+                                              'gam create|update building')
+    return body
+
+
+def createBuilding():
+    cd = gapi_directory.build()
+    body = {
+        'floorNames': ['1'],
+        'buildingId': str(uuid.uuid4()),
+        'buildingName': sys.argv[3]
+    }
+    body = _getBuildingAttributes(sys.argv[4:], body)
+    print(f'Creating building {body["buildingId"]}...')
+    gapi.call(cd.resources().buildings(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def _makeBuildingIdNameMap(cd):
+    fields = 'nextPageToken,buildings(buildingId,buildingName)'
+    buildings = gapi.get_all_pages(cd.resources().buildings(),
+                                   'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {}
+    GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {}
+    for building in buildings:
+        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][
+            building['buildingId']] = building['buildingName']
+        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][
+            building['buildingName']] = building['buildingId']
+
+
+def getBuildingByNameOrId(cd, which_building, minLen=1):
+    if not which_building or \
+       (minLen == 0 and which_building in ['id:', 'uid:']):
+        if minLen == 0:
+            return ''
+        controlflow.system_error_exit(3, 'Building id/name is empty')
+    cg = UID_PATTERN.match(which_building)
+    if cg:
+        return cg.group(1)
+    if GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] is None:
+        _makeBuildingIdNameMap(cd)
+    # Exact name match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID]:
+        return GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][which_building]
+    # No exact name match, check for case insensitive name matches
+    which_building_lower = which_building.lower()
+    ci_matches = []
+    for buildingName, buildingId in GM_Globals[
+            GM_MAP_BUILDING_NAME_TO_ID].items():
+        if buildingName.lower() == which_building_lower:
+            ci_matches.append({
+                'buildingName': buildingName,
+                'buildingId': buildingId
+            })
+    # One match, return ID
+    if len(ci_matches) == 1:
+        return ci_matches[0]['buildingId']
+    # No or multiple name matches, try ID
+    # Exact ID match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        return which_building
+    # No exact ID match, check for case insensitive id match
+    for buildingId in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        # Match, return ID
+        if buildingId.lower() == which_building_lower:
+            return buildingId
+    # Multiple name  matches
+    if len(ci_matches) > 1:
+        message = 'Multiple buildings with same name:\n'
+        for building in ci_matches:
+            message += f'  Name:{building["buildingName"]}  ' \
+                       f'id:{building["buildingId"]}\n'
+        message += '\nPlease specify building name by exact case or by id.'
+        controlflow.system_error_exit(3, message)
+    # No matches
+    else:
+        controlflow.system_error_exit(3, f'No such building {which_building}')
+
+
+def getBuildingNameById(cd, buildingId):
+    if GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] is None:
+        _makeBuildingIdNameMap(cd)
+    return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, 'UNKNOWN')
+
+
+def updateBuilding():
+    cd = gapi_directory.build()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    body = _getBuildingAttributes(sys.argv[4:])
+    print(f'Updating building {buildingId}...')
+    gapi.call(cd.resources().buildings(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              buildingId=buildingId,
+              body=body)
+
+
+def getBuildingInfo():
+    cd = gapi_directory.build()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    building = gapi.call(cd.resources().buildings(),
+                         'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         buildingId=buildingId)
+    if 'buildingId' in building:
+        building['buildingId'] = f'id:{building["buildingId"]}'
+    if 'floorNames' in building:
+        building['floorNames'] = ','.join(building['floorNames'])
+    if 'buildingName' in building:
+        sys.stdout.write(building.pop('buildingName'))
+    display.print_json(building)
+
+
+def deleteBuilding():
+    cd = gapi_directory.build()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    print(f'Deleting building {buildingId}...')
+    gapi.call(cd.resources().buildings(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              buildingId=buildingId)
+
+
+def _getFeatureAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = args[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg,
+                                              'gam create|update feature')
+    return body
+
+
+def createFeature():
+    cd = gapi_directory.build()
+    body = _getFeatureAttributes(sys.argv[3:])
+    print(f'Creating feature {body["name"]}...')
+    gapi.call(cd.resources().features(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def updateFeature():
+    # update does not work for name and name is only field to be updated
+    # if additional writable fields are added to feature in the future
+    # we'll add support for update as well as rename
+    cd = gapi_directory.build()
+    oldName = sys.argv[3]
+    body = {'newName': sys.argv[5:]}
+    print(f'Updating feature {oldName}...')
+    gapi.call(cd.resources().features(),
+              'rename',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              oldName=oldName,
+              body=body)
+
+
+def deleteFeature():
+    cd = gapi_directory.build()
+    featureKey = sys.argv[3]
+    print(f'Deleting feature {featureKey}...')
+    gapi.call(cd.resources().features(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              featureKey=featureKey)
+
+
+def _getResourceCalendarAttributes(cd, args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['resourceName'] = args[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['resourceDescription'] = args[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'type':
+            body['resourceType'] = args[i + 1]
+            i += 2
+        elif myarg in ['building', 'buildingid']:
+            body['buildingId'] = getBuildingByNameOrId(cd,
+                                                       args[i + 1],
+                                                       minLen=0)
+            i += 2
+        elif myarg in ['capacity']:
+            body['capacity'] = gam.getInteger(args[i + 1], myarg, minVal=0)
+            i += 2
+        elif myarg in ['feature', 'features']:
+            features = args[i + 1].split(',')
+            body['featureInstances'] = []
+            for feature in features:
+                instance = {'feature': {'name': feature}}
+                body['featureInstances'].append(instance)
+            i += 2
+        elif myarg in ['floor', 'floorname']:
+            body['floorName'] = args[i + 1]
+            i += 2
+        elif myarg in ['floorsection']:
+            body['floorSection'] = args[i + 1]
+            i += 2
+        elif myarg in ['category']:
+            body['resourceCategory'] = args[i + 1].upper()
+            if body['resourceCategory'] == 'ROOM':
+                body['resourceCategory'] = 'CONFERENCE_ROOM'
+            i += 2
+        elif myarg in ['uservisibledescription', 'userdescription']:
+            body['userVisibleDescription'] = args[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(args[i],
+                                              'gam create|update resource')
+    return body
+
+
+def createResourceCalendar():
+    cd = gapi_directory.build()
+    body = {'resourceId': sys.argv[3], 'resourceName': sys.argv[4]}
+    body = _getResourceCalendarAttributes(cd, sys.argv[5:], body)
+    print(f'Creating resource {body["resourceId"]}...')
+    gapi.call(cd.resources().calendars(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def updateResourceCalendar():
+    cd = gapi_directory.build()
+    resId = sys.argv[3]
+    body = _getResourceCalendarAttributes(cd, sys.argv[4:])
+    # Use patch since it seems to work better.
+    # update requires name to be set.
+    gapi.call(cd.resources().calendars(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              calendarResourceId=resId,
+              body=body,
+              fields='')
+    print(f'updated resource {resId}')
+
+
+def getResourceCalendarInfo():
+    cd = gapi_directory.build()
+    resId = sys.argv[3]
+    resource = gapi.call(cd.resources().calendars(),
+                         'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         calendarResourceId=resId)
+    if 'featureInstances' in resource:
+        features = []
+        for a_feature in resource.pop('featureInstances'):
+            features.append(a_feature['feature']['name'])
+        resource['features'] = ', '.join(features)
+    if 'buildingId' in resource:
+        resource['buildingName'] = getBuildingNameById(cd,
+                                                       resource['buildingId'])
+        resource['buildingId'] = f'id:{resource["buildingId"]}'
+    display.print_json(resource)
+
+
+def deleteResourceCalendar():
+    resId = sys.argv[3]
+    cd = gapi_directory.build()
+    print(f'Deleting resource calendar {resId}')
+    gapi.call(cd.resources().calendars(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              calendarResourceId=resId)

src/gam/gapi/directory/roles.py

@@ -0,0 +1,124 @@
+import sys
+
+from gam.var import GC_Values, GC_CUSTOMER_ID
+import gam
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi.directory import privileges as gapi_directory_privileges
+
+
+def getPrivileges(body, privs, action):
+    all_privileges = gapi_directory_privileges.print_(return_only=True)
+    if privs == 'ALL':
+        body['rolePrivileges'] = [
+            {'privilegeName': p['privilegeName'], 'serviceId': p['serviceId']} for p in all_privileges
+            ]
+    elif privs == 'ALL_OU':
+        body['rolePrivileges'] = [
+            {'privilegeName': p['privilegeName'], 'serviceId': p['serviceId']} for p in all_privileges if p.get('isOuScopable')
+            ]
+    else:
+      body.setdefault('rolePrivileges', [])
+      for priv in privs.split(','):
+          for p in all_privileges:
+              if priv == p['privilegeName']:
+                body['rolePrivileges'].append({'privilegeName': p['privilegeName'], 'serviceId': p['serviceId']})
+                break
+          else:
+              controlflow.invalid_argument_exit(priv,
+                                                f'gam {action} adminrole privileges')
+
+def create():
+    cd = gapi_directory.build()
+    body = {'roleName': sys.argv[3]}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'privileges':
+            getPrivileges(body, sys.argv[i + 1].upper(), 'create')
+            i += 2
+        elif myarg == 'description':
+            body['roleDescription'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam create adminrole')
+
+    if not body.get('rolePrivileges'):
+        controlflow.missing_argument_exit('privileges',
+                                          'gam create adminrole')
+    print(f'Creating role {body["roleName"]}')
+    gapi.call(cd.roles(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+def update():
+    cd = gapi_directory.build()
+    body = {}
+    roleId = gam.getRoleId(sys.argv[3])
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'privileges':
+            getPrivileges(body, sys.argv[i + 1].upper(), 'update')
+            i += 2
+        elif myarg == 'description':
+            body['roleDescription'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'name':
+            body['roleName'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam update adminrole')
+
+    print(f'Updating role {roleId}')
+    gapi.call(cd.roles(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              roleId=roleId,
+              body=body)
+
+
+def delete():
+    cd = gapi_directory.build()
+    roleId = gam.getRoleId(sys.argv[3])
+    print(f'Deleting role {roleId}')
+    gapi.call(cd.roles(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              roleId=roleId)
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'roleId', 'roleName', 'roleDescription', 'isSuperAdminRole',
+        'isSystemRole'
+    ]
+    fields = f'nextPageToken,items({",".join(titles)})'
+    csvRows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print adminroles')
+    roles = gapi.get_all_pages(cd.roles(),
+                               'list',
+                               'items',
+                               customer=GC_Values[GC_CUSTOMER_ID],
+                               fields=fields)
+    for role in roles:
+        role_attrib = {}
+        for key, value in list(role.items()):
+            role_attrib[key] = value
+        csvRows.append(role_attrib)
+    display.write_csv_file(csvRows, titles, 'Admin Roles', todrive)

src/gam/gapi/directory/users.py

@@ -0,0 +1,64 @@
+from time import sleep
+
+import gam
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+
+
+def get_primary(email):
+    '''returns primary email of user or empty if email is not a user primary or
+    alias address.'''
+    cd = gapi_directory.build()
+    result = gapi.call(cd.users(), 'get', userKey=email,
+            projection='basic', fields='primaryEmail',
+            soft_errors=True)
+    if not result:
+        return ''
+    return result.get('primaryEmail', '').lower()
+
+
+def signout(users):
+    cd = gapi_directory.build()
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user = gam.normalizeEmailAddressOrUID(user)
+        print(f'Signing Out {user}{gam.currentCount(i, count)}')
+        gapi.call(cd.users(),
+                  'signOut',
+                  soft_errors=True,
+                  userKey=user)
+
+
+def turn_off_2sv(users):
+    cd = gapi_directory.build()
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user = gam.normalizeEmailAddressOrUID(user)
+        print(f'Turning Off 2-Step Verification for {user}{gam.currentCount(i, count)}')
+        gapi.call(cd.twoStepVerification(),
+                  'turnOff',
+                  soft_errors=True,
+                  userKey=user)
+
+def wait_for_mailbox(users):
+    '''Wait until users mailbox is provisioned.'''
+    cd = gapi_directory.build()
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user = gam.normalizeEmailAddressOrUID(user)
+        while True:
+            result = gapi.call(cd.users(),
+                               'get',
+                               'fields=isMailboxSetup',
+                               userKey=user)
+            mailbox_is_setup = result.get('isMailboxSetup')
+            print(f'{user} mailboxIsSetup: {mailbox_is_setup}')
+            if mailbox_is_setup:
+                break
+            sleep(3)

src/gam/gapi/errors.py

@@ -0,0 +1,382 @@
+"""GAPI and OAuth Token related errors methods."""
+
+from enum import Enum
+import json
+
+from gam import controlflow
+from gam import display
+from gam.var import UTF8
+
+
+class GapiAbortedError(Exception):
+    pass
+
+
+class GapiAuthErrorError(Exception):
+    pass
+
+
+class GapiBadGatewayError(Exception):
+    pass
+
+
+class GapiBadRequestError(Exception):
+    pass
+
+
+class GapiConditionNotMetError(Exception):
+    pass
+
+
+class GapiCyclicMembershipsNotAllowedError(Exception):
+    pass
+
+
+class GapiDomainCannotUseApisError(Exception):
+    pass
+
+
+class GapiDomainNotFoundError(Exception):
+    pass
+
+
+class GapiDuplicateError(Exception):
+    pass
+
+
+class GapiFailedPreconditionError(Exception):
+    pass
+
+
+class GapiForbiddenError(Exception):
+    pass
+
+
+class GapiGatewayTimeoutError(Exception):
+    pass
+
+
+class GapiGroupNotFoundError(Exception):
+    pass
+
+
+class GapiInvalidError(Exception):
+    pass
+
+
+class GapiInvalidArgumentError(Exception):
+    pass
+
+
+class GapiInvalidMemberError(Exception):
+    pass
+
+
+class GapiMemberNotFoundError(Exception):
+    pass
+
+
+class GapiNotFoundError(Exception):
+    pass
+
+
+class GapiNotImplementedError(Exception):
+    pass
+
+
+class GapiPermissionDeniedError(Exception):
+    pass
+
+
+class GapiResourceNotFoundError(Exception):
+    pass
+
+
+class GapiServiceNotAvailableError(Exception):
+    pass
+
+
+class GapiUserNotFoundError(Exception):
+    pass
+
+
+# GAPI Error Reasons
+class ErrorReason(Enum):
+    """The reason why a non-200 HTTP response was returned from a GAPI."""
+    ABORTED = 'aborted'
+    AUTH_ERROR = 'authError'
+    BACKEND_ERROR = 'backendError'
+    BAD_GATEWAY = 'badGateway'
+    BAD_REQUEST = 'badRequest'
+    CONDITION_NOT_MET = 'conditionNotMet'
+    CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
+    DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
+    DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
+    DOMAIN_NOT_FOUND = 'domainNotFound'
+    DUPLICATE = 'duplicate'
+    FAILED_PRECONDITION = 'failedPrecondition'
+    FORBIDDEN = 'forbidden'
+    FIVE_O_THREE = '503'
+    FOUR_O_NINE = '409'
+    FOUR_O_O = '400'
+    FOUR_O_THREE = '403'
+    FOUR_TWO_NINE = '429'
+    GATEWAY_TIMEOUT = 'gatewayTimeout'
+    GROUP_NOT_FOUND = 'groupNotFound'
+    INTERNAL_ERROR = 'internalError'
+    INVALID = 'invalid'
+    INVALID_ARGUMENT = 'invalidArgument'
+    INVALID_MEMBER = 'invalidMember'
+    MEMBER_NOT_FOUND = 'memberNotFound'
+    NOT_FOUND = 'notFound'
+    NOT_IMPLEMENTED = 'notImplemented'
+    PERMISSION_DENIED = 'permissionDenied'
+    QUOTA_EXCEEDED = 'quotaExceeded'
+    RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
+    RESOURCE_NOT_FOUND = 'resourceNotFound'
+    SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+    SERVICE_LIMIT = 'serviceLimit'
+    SYSTEM_ERROR = 'systemError'
+    USER_NOT_FOUND = 'userNotFound'
+    USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+
+    def __str__(self):
+        return str(self.value)
+
+
+# Common sets of GAPI error reasons
+DEFAULT_RETRY_REASONS = [
+    ErrorReason.QUOTA_EXCEEDED,
+    ErrorReason.RATE_LIMIT_EXCEEDED,
+    ErrorReason.USER_RATE_LIMIT_EXCEEDED,
+    ErrorReason.BACKEND_ERROR,
+    ErrorReason.BAD_GATEWAY,
+    ErrorReason.GATEWAY_TIMEOUT,
+    ErrorReason.INTERNAL_ERROR,
+    ErrorReason.FOUR_TWO_NINE,
+    ErrorReason.FIVE_O_THREE,
+]
+GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
+GROUP_GET_THROW_REASONS = [
+    ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
+    ErrorReason.BAD_REQUEST
+]
+GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
+MEMBERS_THROW_REASONS = [
+    ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
+    ErrorReason.FORBIDDEN
+]
+MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
+
+# A map of GAPI error reasons to the corresponding GAM Python Exception
+ERROR_REASON_TO_EXCEPTION = {
+    ErrorReason.ABORTED:
+        GapiAbortedError,
+    ErrorReason.AUTH_ERROR:
+        GapiAuthErrorError,
+    ErrorReason.BAD_GATEWAY:
+        GapiBadGatewayError,
+    ErrorReason.BAD_REQUEST:
+        GapiBadRequestError,
+    ErrorReason.CONDITION_NOT_MET:
+        GapiConditionNotMetError,
+    ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
+        GapiCyclicMembershipsNotAllowedError,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS:
+        GapiDomainCannotUseApisError,
+    ErrorReason.DOMAIN_NOT_FOUND:
+        GapiDomainNotFoundError,
+    ErrorReason.DUPLICATE:
+        GapiDuplicateError,
+    ErrorReason.FAILED_PRECONDITION:
+        GapiFailedPreconditionError,
+    ErrorReason.FORBIDDEN:
+        GapiForbiddenError,
+    ErrorReason.GATEWAY_TIMEOUT:
+        GapiGatewayTimeoutError,
+    ErrorReason.GROUP_NOT_FOUND:
+        GapiGroupNotFoundError,
+    ErrorReason.INVALID:
+        GapiInvalidError,
+    ErrorReason.INVALID_ARGUMENT:
+        GapiInvalidArgumentError,
+    ErrorReason.INVALID_MEMBER:
+        GapiInvalidMemberError,
+    ErrorReason.MEMBER_NOT_FOUND:
+        GapiMemberNotFoundError,
+    ErrorReason.NOT_FOUND:
+        GapiNotFoundError,
+    ErrorReason.NOT_IMPLEMENTED:
+        GapiNotImplementedError,
+    ErrorReason.PERMISSION_DENIED:
+        GapiPermissionDeniedError,
+    ErrorReason.RESOURCE_NOT_FOUND:
+        GapiResourceNotFoundError,
+    ErrorReason.SERVICE_NOT_AVAILABLE:
+        GapiServiceNotAvailableError,
+    ErrorReason.USER_NOT_FOUND:
+        GapiUserNotFoundError,
+}
+
+# OAuth Token Errors
+OAUTH2_TOKEN_ERRORS = [
+    'access_denied',
+    'access_denied: Requested client not authorized',
+    'internal_failure: Backend Error',
+    'internal_failure: None',
+    'invalid_grant',
+    'invalid_grant: Bad Request',
+    'invalid_grant: Invalid email or User ID',
+    'invalid_grant: Not a valid email',
+    'invalid_grant: Invalid JWT: No valid verifier found for issuer',
+    'invalid_grant: The account has been deleted',
+    'invalid_grant: reauth related error (invalid_rapt)',
+    'invalid_request: Invalid impersonation prn email address',
+    'invalid_request: Invalid impersonation "sub" field',
+    'unauthorized_client: Client is unauthorized to retrieve access tokens '
+    'using this method',
+    'unauthorized_client: Client is unauthorized to retrieve access tokens '
+    'using this method, or client not authorized for any of the scopes '
+    'requested',
+    'unauthorized_client: Unauthorized client or scope in request',
+]
+
+
+def _create_http_error_dict(status_code, reason, message):
+    """Creates a basic error dict similar to most Google API Errors.
+
+  Args:
+    status_code: Int, the error's HTTP response status code.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    dict
+  """
+    return {
+        'error': {
+            'code': status_code,
+            'errors': [{
+                'reason': str(reason),
+                'message': message,
+            }]
+        }
+    }
+
+
+def get_gapi_error_detail(e,
+                          soft_errors=False,
+                          silent_errors=False,
+                          retry_on_http_error=False):
+    """Extracts error detail from a non-200 GAPI Response.
+
+  Args:
+    e: googleapiclient.HttpError, The HTTP Error received.
+    soft_errors: Boolean, If true, causes error messages to be surpressed,
+      rather than sending them to stderr.
+    silent_errors: Boolean, If true, suppresses and ignores any errors from
+      being displayed
+    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
+      code, indicating that the request can be retried. TODO: Remove this param,
+        as it seems to be outside the scope of this method.
+
+  Returns:
+    A tuple containing the HTTP Response code, GAPI error reason, and error
+        message.
+  """
+    try:
+        error = json.loads(e.content.decode(UTF8))
+    except ValueError:
+        error_content = e.content.decode(UTF8) if isinstance(
+            e.content, bytes) else e.content
+        if (e.resp['status'] == '503') and (
+                error_content == 'Quota exceeded for the current request'):
+            return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value,
+                    error_content)
+        if (e.resp['status'] == '403') and (error_content.startswith(
+                'Request rate higher than configured')):
+            return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value,
+                    error_content)
+        if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
+            return (e.resp['status'], ErrorReason.BAD_GATEWAY.value,
+                    error_content)
+        if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
+            return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value,
+                    error_content)
+        if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
+            error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
+                                            'Domain not found')
+        elif (e.resp['status'] == '400') and (
+                'InvalidSsoSigningKey' in error_content):
+            error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                            'InvalidSsoSigningKey')
+        elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
+            error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                            'UnknownError')
+        elif retry_on_http_error:
+            return (-1, None, None)
+        elif soft_errors:
+            if not silent_errors:
+                display.print_error(error_content)
+            return (0, None, None)
+        else:
+            controlflow.system_error_exit(5, error_content)
+        # END: ValueError catch
+
+    if 'error' in error:
+        http_status = error['error']['code']
+        try:
+            message = error['error']['errors'][0]['message']
+        except KeyError:
+            message = error['error']['message']
+        if http_status == 404:
+            if 'Requested entity was not found' in message or 'does not exist' in message:
+                error = _create_http_error_dict(404, ErrorReason.NOT_FOUND.value,
+                                                message)
+    else:
+        if 'error_description' in error:
+            if error['error_description'] == 'Invalid Value':
+                message = error['error_description']
+                http_status = 400
+                error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                                message)
+            else:
+                controlflow.system_error_exit(4, str(error))
+        else:
+            controlflow.system_error_exit(4, str(error))
+
+    # Extract the error reason
+    try:
+        reason = error['error']['errors'][0]['reason']
+        if reason == 'notFound':
+            if 'userKey' in message:
+                reason = ErrorReason.USER_NOT_FOUND.value
+            elif 'groupKey' in message:
+                reason = ErrorReason.GROUP_NOT_FOUND.value
+            elif 'memberKey' in message:
+                reason = ErrorReason.MEMBER_NOT_FOUND.value
+            elif 'Domain not found' in message:
+                reason = ErrorReason.DOMAIN_NOT_FOUND.value
+            elif 'Resource Not Found' in message:
+                reason = ErrorReason.RESOURCE_NOT_FOUND.value
+        elif reason == 'invalid':
+            if 'userId' in message:
+                reason = ErrorReason.USER_NOT_FOUND.value
+            elif 'memberKey' in message:
+                reason = ErrorReason.INVALID_MEMBER.value
+        elif reason == 'failedPrecondition':
+            if 'Bad Request' in message:
+                reason = ErrorReason.BAD_REQUEST.value
+            elif 'Mail service not enabled' in message:
+                reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
+        elif reason == 'required':
+            if 'memberKey' in message:
+                reason = ErrorReason.MEMBER_NOT_FOUND.value
+        elif reason == 'conditionNotMet':
+            if 'Cyclic memberships not allowed' in message:
+                reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
+    except KeyError:
+        reason = f'{http_status}'
+    return (http_status, reason, message)

src/gam/gapi/errors_test.py

@@ -0,0 +1,212 @@
+"""Python unit tests for gapi.errors"""
+
+import json
+
+import unittest
+from unittest.mock import patch
+
+import googleapiclient.errors
+from gam.gapi import errors
+import httplib2
+
+
+def create_simple_http_error(status, reason, message):
+    content = errors._create_http_error_dict(status, reason, message)
+    return create_http_error(status, content)
+
+
+def create_http_error(status, content):
+    response = httplib2.Response({
+        'status': status,
+        'content-type': 'application/json',
+    })
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    return googleapiclient.errors.HttpError(response, content_as_bytes)
+
+
+class ErrorsTest(unittest.TestCase):
+
+    def test_get_gapi_error_detail_quota_exceeded(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_invalid_domain(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_invalid_signing_key(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_unknown_error(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_retry_http_error(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_prints_soft_errors(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_exits_on_unrecoverable_errors(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_quota_exceeded_for_current_request(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_quota_exceeded_high_request_rate(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_extracts_user_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: userKey.')
+        print(err)
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: userKey.')
+
+    def test_get_gapi_error_extracts_group_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: groupKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: groupKey.')
+
+    def test_get_gapi_error_extracts_member_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: memberKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: memberKey.')
+
+    def test_get_gapi_error_extracts_domain_not_found(self):
+        err = create_simple_http_error(404, 'notFound', 'Domain not found.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
+        self.assertEqual(message, 'Domain not found.')
+
+    def test_get_gapi_error_extracts_generic_resource_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: unknownResource.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: unknownResource.')
+
+    def test_get_gapi_error_extracts_invalid_userid(self):
+        err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+        self.assertEqual(message, 'Invalid Input: userId')
+
+    def test_get_gapi_error_extracts_invalid_member(self):
+        err = create_simple_http_error(400, 'invalid',
+                                       'Invalid Input: memberKey')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
+        self.assertEqual(message, 'Invalid Input: memberKey')
+
+    def test_get_gapi_error_extracts_bad_request(self):
+        err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
+        self.assertEqual(message, 'Bad Request')
+
+    def test_get_gapi_error_extracts_service_not_available(self):
+        err = create_simple_http_error(400, 'failedPrecondition',
+                                       'Mail service not enabled')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
+        self.assertEqual(message, 'Mail service not enabled')
+
+    def test_get_gapi_error_extracts_required_member_not_found(self):
+        err = create_simple_http_error(400, 'required',
+                                       'Missing required field: memberKey')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+        self.assertEqual(message, 'Missing required field: memberKey')
+
+    def test_get_gapi_error_extracts_cyclic_memberships_error(self):
+        err = create_simple_http_error(400, 'conditionNotMet',
+                                       'Cyclic memberships not allowed')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(
+            reason, errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
+        self.assertEqual(message, 'Cyclic memberships not allowed')
+
+    def test_get_gapi_error_extracts_single_error_with_message(self):
+        status_code = 999
+        response = httplib2.Response({'status': status_code})
+        # This error does not have an "errors" key describing each error.
+        content = {'error': {'code': status_code, 'message': 'unknown error'}}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, status_code)
+        self.assertEqual(reason, str(status_code))
+        self.assertEqual(message, content['error']['message'])
+
+    def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
+        self):
+        status_code = 999
+        response = httplib2.Response({'status': status_code})
+        # This error only has an error_description_field and an unknown description.
+        content = {'error_description': 'something errored'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        with self.assertRaises(SystemExit) as context:
+            errors.get_gapi_error_detail(err)
+        self.assertEqual(4, context.exception.code)
+
+    def test_get_gapi_error_exits_on_invalid_error_description(self):
+        status_code = 400
+        response = httplib2.Response({'status': status_code})
+        content = {'error_description': 'Invalid Value'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, status_code)
+        self.assertEqual(reason, errors.ErrorReason.INVALID.value)
+        self.assertEqual(message, 'Invalid Value')
+
+    def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
+        status_code = 900
+        response = httplib2.Response({'status': status_code})
+        content = {'notErrorContentThatIsExpected': 'foo'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        with self.assertRaises(SystemExit) as context:
+            errors.get_gapi_error_detail(err)
+        self.assertEqual(4, context.exception.code)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/licensing.py

@@ -0,0 +1,309 @@
+import re
+import sys
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import errors as gapi_errors
+from gam.gapi.directory import customer as gapi_directory_customer
+
+
+def _get_customerid():
+    ''' returns customerId with format C{customer_id}'''
+    gapi_directory_customer.setTrueCustomerId()
+    customer_id = GC_Values[GC_CUSTOMER_ID]
+    if customer_id[0] != 'C':
+        customer_id = 'C' + customer_id
+    return customer_id
+
+def build():
+    return gam.buildGAPIObject('licensing')
+
+
+def getProductAndSKU(sku):
+    l_sku = sku.lower().replace('-', '').replace(' ', '')
+    for a_sku, sku_values in list(SKUS.items()):
+        if l_sku == a_sku.lower().replace(
+                '-',
+                '') or l_sku in sku_values['aliases'] or l_sku == sku_values[
+                    'displayName'].lower().replace(' ', ''):
+            return (sku_values['product'], a_sku)
+    try:
+        product = re.search('^([A-Z,a-z]*-[A-Z,a-z]*)', sku).group(1)
+    except AttributeError:
+        product = sku
+    return (product, sku)
+
+
+def user_lic_result(request_id, response, exception):
+    if exception:
+        http_status, reason, message = gapi_errors.get_gapi_error_detail(
+                exception,
+                soft_errors=True)
+        print(f'ERROR: {request_id}: {http_status} - {reason} {message}')
+
+
+def create(users, sku=None):
+    lic = build()
+    if not sku:
+        sku = sys.argv[5]
+    productId, skuId = getProductAndSKU(sku)
+    sku_name = _formatSKUIdDisplayName(skuId)
+    i = 6
+    if len(sys.argv) > 6 and sys.argv[i].lower() in ['product', 'productid']:
+        productId = sys.argv[i+1]
+        i += 2
+    for user in users:
+        print(f'Adding license {sku_name} from to {user}')
+        gapi.call(lic.licenseAssignments(),
+                  'insert',
+                  soft_errors=True,
+                  productId=productId,
+                  skuId=skuId,
+                  body={'userId': user})
+
+
+def delete(users, sku=None):
+    lic = build()
+    if not sku:
+        sku = sys.argv[5]
+    productId, skuId = getProductAndSKU(sku)
+    sku_name = _formatSKUIdDisplayName(skuId)
+    i = 6
+    if len(sys.argv) > 6 and sys.argv[i].lower() in ['product', 'productid']:
+        productId = sys.argv[i+1]
+        i += 2
+    for user in users:
+         print(f'Removing license {sku_name} from user {user}')
+         gapi.call(lic.licenseAssignments(),
+                   'delete',
+                   soft_errors=True,
+                   productId=productId,
+                   skuId=skuId,
+                   userId=user)
+
+
+def sync(users):
+    sku = sys.argv[5]
+    current_licenses = gam.getUsersToModify(entity_type='license',
+        entity=sku)
+    users_to_license = [user for user in users if user not in current_licenses]
+    users_to_unlicense = [user for user in current_licenses if user not in users]
+    print(f'Need to remove license from {len(users_to_unlicense)} and add to ' \
+          f'{len(users_to_license)} users...')
+    # do the remove first to free up seats
+    delete(users_to_unlicense, sku)
+    create(users_to_license, sku)
+
+
+def update(users, sku=None, old_sku=None):
+    lic = build()
+    if not sku:
+        sku = sys.argv[5]
+    productId, skuId = getProductAndSKU(sku)
+    sku_name = _formatSKUIdDisplayName(skuId)
+    i = 6
+    if len(sys.argv) > 6 and sys.argv[i].lower() in ['product', 'productid']:
+        productId = sys.argv[i+1]
+        i += 2
+    if not old_sku:
+        try:
+            old_sku = sys.argv[i]
+            if old_sku.lower() == 'from':
+                old_sku = sys.argv[i + 1]
+        except KeyError:
+            controlflow.system_error_exit(
+                2,
+                'You need to specify the user\'s old SKU as the last argument'
+            )
+        _, old_sku = getProductAndSKU(old_sku)
+    old_sku_name = _formatSKUIdDisplayName(old_sku)
+    for user in users:
+        print(f'Changing user {user} from license {old_sku_name} to {sku_name}')
+        gapi.call(lic.licenseAssignments(),
+                  'patch',
+                  soft_errors=True,
+                  productId=productId,
+                  skuId=old_sku,
+                  userId=user,
+                  body={'skuId': skuId})
+
+
+def print_(returnFields=None,
+                    skus=None,
+                    countsOnly=False,
+                    returnCounts=False):
+    lic = build()
+    customer_id = _get_customerid()
+    products = []
+    licenses = []
+    licenseCounts = []
+    if not returnFields:
+        csvRows = []
+        todrive = False
+        i = 3
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower()
+            if not returnCounts and myarg == 'todrive':
+                todrive = True
+                i += 1
+            elif myarg in ['products', 'product']:
+                products = sys.argv[i + 1].split(',')
+                i += 2
+            elif myarg in ['sku', 'skus']:
+                skus = sys.argv[i + 1].split(',')
+                i += 2
+            elif myarg == 'allskus':
+                skus = sorted(SKUS)
+                products = []
+                i += 1
+            elif myarg == 'gsuite':
+                skus = [
+                    skuId for skuId in SKUS
+                    if SKUS[skuId]['product'] in ['Google-Apps', '101031']
+                ]
+                products = []
+                i += 1
+            elif myarg == 'countsonly':
+                countsOnly = True
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i],
+                                                  'gam print licenses')
+        if not countsOnly:
+            fields = 'nextPageToken,items(productId,skuId,userId)'
+            titles = ['userId', 'productId', 'skuId']
+        else:
+            fields = 'nextPageToken,items(userId)'
+            if not returnCounts:
+                if skus:
+                    titles = ['productId', 'skuId', 'licenses']
+                else:
+                    titles = ['productId', 'licenses']
+    else:
+        fields = f'nextPageToken,items({returnFields})'
+    if skus:
+        for sku in skus:
+            if not products:
+                product, sku = getProductAndSKU(sku)
+            else:
+                product = products[0]
+            page_message = gapi.got_total_items_msg(
+                f'Licenses for {SKUS.get(sku, {"displayName": sku})["displayName"]}',
+                '...\n')
+            try:
+                licenses += gapi.get_all_pages(
+                    lic.licenseAssignments(),
+                    'listForProductAndSku',
+                    'items',
+                    throw_reasons=[
+                        gapi_errors.ErrorReason.INVALID,
+                        gapi_errors.ErrorReason.FORBIDDEN
+                    ],
+                    page_message=page_message,
+                    customerId=customer_id,
+                    productId=product,
+                    skuId=sku,
+                    fields=fields)
+                if countsOnly:
+                    licenseCounts.append([
+                        'Product', product, 'SKU', sku, 'Licenses',
+                        len(licenses)
+                    ])
+                    licenses = []
+            except (gapi_errors.GapiInvalidError,
+                    gapi_errors.GapiForbiddenError):
+                pass
+    else:
+        if not products:
+            products = sorted(PRODUCTID_NAME_MAPPINGS)
+        for productId in products:
+            page_message = gapi.got_total_items_msg(
+                f'Licenses for {PRODUCTID_NAME_MAPPINGS.get(productId, productId)}',
+                '...\n')
+            try:
+                licenses += gapi.get_all_pages(
+                    lic.licenseAssignments(),
+                    'listForProduct',
+                    'items',
+                    throw_reasons=[
+                        gapi_errors.ErrorReason.INVALID,
+                        gapi_errors.ErrorReason.FORBIDDEN
+                    ],
+                    page_message=page_message,
+                    customerId=customer_id,
+                    productId=productId,
+                    fields=fields)
+                if countsOnly:
+                    licenseCounts.append(
+                        ['Product', productId, 'Licenses',
+                         len(licenses)])
+                    licenses = []
+            except (gapi_errors.GapiInvalidError,
+                    gapi_errors.GapiForbiddenError):
+                pass
+    if countsOnly:
+        if returnCounts:
+            return licenseCounts
+        if skus:
+            for u_license in licenseCounts:
+                csvRows.append({
+                    'productId': u_license[1],
+                    'skuId': u_license[3],
+                    'licenses': u_license[5]
+                })
+        else:
+            for u_license in licenseCounts:
+                csvRows.append({
+                    'productId': u_license[1],
+                    'licenses': u_license[3]
+                })
+        display.write_csv_file(csvRows, titles, 'Licenses', todrive)
+        return
+    if returnFields:
+        if returnFields == 'userId':
+            userIds = []
+            for u_license in licenses:
+                userId = u_license.get('userId', '').lower()
+                if userId:
+                    userIds.append(userId)
+            return userIds
+        userSkuIds = {}
+        for u_license in licenses:
+            userId = u_license.get('userId', '').lower()
+            skuId = u_license.get('skuId')
+            if userId and skuId:
+                userSkuIds.setdefault(userId, [])
+                userSkuIds[userId].append(skuId)
+        return userSkuIds
+    for u_license in licenses:
+        userId = u_license.get('userId', '').lower()
+        skuId = u_license.get('skuId', '')
+        csvRows.append({
+            'userId': userId,
+            'productId': u_license.get('productId', ''),
+            'skuId': _skuIdToDisplayName(skuId)
+        })
+    display.write_csv_file(csvRows, titles, 'Licenses', todrive)
+
+
+def show():
+    licenseCounts = print_(countsOnly=True, returnCounts=True)
+    for u_license in licenseCounts:
+        line = ''
+        for i in range(0, len(u_license), 2):
+            line += f'{u_license[i]}: {u_license[i+1]}, '
+        print(line[:-2])
+
+
+def _skuIdToDisplayName(skuId):
+    return SKUS[skuId]['displayName'] if skuId in SKUS else skuId
+
+
+def _formatSKUIdDisplayName(skuId):
+    skuIdDisplay = _skuIdToDisplayName(skuId)
+    if skuId == skuIdDisplay:
+        return skuId
+    return f'{skuId} ({skuIdDisplay})'

src/gam/gapi/reports.py

@@ -0,0 +1,585 @@
+import calendar
+import datetime
+import re
+import sys
+
+from dateutil.relativedelta import relativedelta
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam import utils
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+
+
+def build():
+    return gam.buildGAPIObject('reports')
+
+
+REPORT_CHOICE_MAP = {
+    'access': 'access_transparency',
+    'accesstransparency': 'access_transparency',
+    'calendars': 'calendar',
+    'customers': 'customer',
+    'doc': 'drive',
+    'docs': 'drive',
+    'domain': 'customer',
+    'enterprisegroups': 'groups_enterprise',
+    'google+': 'gplus',
+    'group': 'groups',
+    'groupsenterprise': 'groups_enterprise',
+    'hangoutsmeet': 'meet',
+    'logins': 'login',
+    'oauthtoken': 'token',
+    'tokens': 'token',
+    'usage': 'usage',
+    'usageparameters': 'usageparameters',
+    'users': 'user',
+    'useraccounts': 'user_accounts',
+}
+
+
+def showUsageParameters():
+    rep = build()
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.BAD_REQUEST
+    ]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit('user or customer',
+                                          'report usageparameters')
+    report = sys.argv[3].lower()
+    titles = ['parameter']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = {}
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = {'userKey': gam._get_admin_email()}
+    else:
+        controlflow.expected_argument_exit('usageparameters',
+                                           ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    all_parameters = set()
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam report usageparameters')
+    fullDataRequired = ['all']
+    while True:
+        try:
+            result = gapi.call(endpoint,
+                               'get',
+                               throw_reasons=throw_reasons,
+                               date=tryDate,
+                               customerId=customerId,
+                               fields='warnings,usageReports(parameters(name))',
+                               **kwargs)
+            warnings = result.get('warnings', [])
+            usage = result.get('usageReports')
+            has_reports = bool(usage)
+            fullData, tryDate = _check_full_data_available(
+                warnings, tryDate, fullDataRequired, has_reports)
+            if fullData < 0:
+                print('No usage parameters available.')
+                sys.exit(1)
+            if has_reports:
+                for parameter in usage[0]['parameters']:
+                    name = parameter.get('name')
+                    if name:
+                        all_parameters.add(name)
+            if fullData == 1:
+                break
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = _adjust_date(str(e))
+    csvRows = []
+    for parameter in sorted(all_parameters):
+        csvRows.append({'parameter': parameter})
+    display.write_csv_file(csvRows, titles,
+                           f'{report.capitalize()} Report Usage Parameters',
+                           todrive)
+
+
+REPORTS_PARAMETERS_SIMPLE_TYPES = [
+    'intValue', 'boolValue', 'datetimeValue', 'stringValue'
+]
+
+
+def showUsage():
+    rep = build()
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.BAD_REQUEST
+    ]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit('user or customer', 'report usage')
+    report = sys.argv[3].lower()
+    titles = ['date']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = [{}]
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = [{'userKey': 'all'}]
+        titles.append('user')
+    else:
+        controlflow.expected_argument_exit('usage', ['user', 'customer'],
+                                           report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    parameters = []
+    start_date = end_date = orgUnitId = None
+    skip_day_numbers = []
+    skip_dates = set()
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'startdate':
+            start_date = utils.get_yyyymmdd(sys.argv[i + 1],
+                                            returnDateTime=True)
+            i += 2
+        elif myarg == 'enddate':
+            end_date = utils.get_yyyymmdd(sys.argv[i + 1], returnDateTime=True)
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i + 1].split(',')
+            i += 2
+        elif myarg == 'skipdates':
+            for skip in sys.argv[i + 1].split(','):
+                if skip.find(':') == -1:
+                    skip_dates.add(utils.get_yyyymmdd(skip,
+                                                      returnDateTime=True))
+                else:
+                    skip_start, skip_end = skip.split(':', 1)
+                    skip_start = utils.get_yyyymmdd(skip_start,
+                                                    returnDateTime=True)
+                    skip_end = utils.get_yyyymmdd(skip_end, returnDateTime=True)
+                    while skip_start <= skip_end:
+                        skip_dates.add(skip_start)
+                        skip_start += one_day
+            i += 2
+        elif myarg == 'skipdaysofweek':
+            skipdaynames = sys.argv[i + 1].split(',')
+            dow = [d.lower() for d in calendar.day_abbr]
+            skip_day_numbers = [dow.index(d) for d in skipdaynames if d in dow]
+            i += 2
+        elif report == 'user' and myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])
+            i += 2
+        elif report == 'user' and myarg in usergroup_types:
+            users = gam.getUsersToModify(myarg, sys.argv[i + 1])
+            kwargs = [{'userKey': user} for user in users]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              f'gam report usage {report}')
+    if parameters:
+        titles.extend(parameters)
+        parameters = ','.join(parameters)
+    else:
+        parameters = None
+    if not end_date:
+        end_date = datetime.datetime.now()
+    if not start_date:
+        start_date = end_date + relativedelta(months=-1)
+    if orgUnitId:
+        for kw in kwargs:
+            kw['orgUnitID'] = orgUnitId
+    usage_on_date = start_date
+    start_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+    usage_end_date = end_date
+    end_date = end_date.strftime(YYYYMMDD_FORMAT)
+    start_use_date = end_use_date = None
+    csvRows = []
+    while usage_on_date <= usage_end_date:
+        if usage_on_date.weekday() in skip_day_numbers or \
+           usage_on_date in skip_dates:
+            usage_on_date += one_day
+            continue
+        use_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+        usage_on_date += one_day
+        try:
+            for kwarg in kwargs:
+                try:
+                    usage = gapi.get_all_pages(endpoint,
+                                               'get',
+                                               'usageReports',
+                                               throw_reasons=throw_reasons,
+                                               customerId=customerId,
+                                               date=use_date,
+                                               parameters=parameters,
+                                               **kwarg)
+                except gapi.errors.GapiBadRequestError:
+                    continue
+                for entity in usage:
+                    row = {'date': use_date}
+                    if 'userEmail' in entity['entity']:
+                        row['user'] = entity['entity']['userEmail']
+                    for item in entity.get('parameters', []):
+                        if 'name' not in item:
+                            continue
+                        name = item['name']
+                        if name == 'cros:device_version_distribution':
+                            for cros_ver in item['msgValue']:
+                                v = cros_ver['version_number']
+                                column_name = f'cros:num_devices_chrome_{v}'
+                                if column_name not in titles:
+                                    titles.append(column_name)
+                                row[column_name] = cros_ver['num_devices']
+                        else:
+                            if not name in titles:
+                                titles.append(name)
+                            for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                                if ptype in item:
+                                    row[name] = item[ptype]
+                                    break
+                            else:
+                                row[name] = ''
+                    if not start_use_date:
+                        start_use_date = use_date
+                    end_use_date = use_date
+                    csvRows.append(row)
+        except gapi.errors.GapiInvalidError as e:
+            display.print_warning(str(e))
+            break
+    if start_use_date:
+        report_name = f'{report.capitalize()} Usage Report - {start_use_date}:{end_use_date}'
+    else:
+        report_name = f'{report.capitalize()} Usage Report - {start_date}:{end_date} - No Data'
+    display.write_csv_file(csvRows, titles, report_name, todrive)
+
+
+def showReport():
+    rep = build()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    report = sys.argv[2].lower()
+    report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    if report == 'usage':
+        showUsage()
+        return
+    if report == 'usageparameters':
+        showUsageParameters()
+        return
+    valid_apps = gapi.get_enum_values_minus_unspecified(
+        rep._rootDesc['resources']['activities']['methods']['list']
+        ['parameters']['applicationName']['enum']) + ['customer', 'user']
+    if report not in valid_apps:
+        controlflow.expected_argument_exit('report',
+                                           ', '.join(sorted(valid_apps)),
+                                           report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    filters = parameters = actorIpAddress = startTime = endTime = eventName = orgUnitId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    to_drive = False
+    userKey = 'all'
+    fullDataRequired = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'date':
+            tryDate = utils.get_yyyymmdd(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'fulldatarequired':
+            fullDataRequired = []
+            fdr = sys.argv[i + 1].lower()
+            if fdr and fdr == 'all':
+                fullDataRequired = 'all'
+            else:
+                fullDataRequired = fdr.replace(',', ' ').split()
+            i += 2
+        elif myarg == 'start':
+            startTime = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'end':
+            endTime = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'event':
+            eventName = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'user':
+            userKey = sys.argv[i + 1].lower()
+            if userKey != 'all':
+                userKey = gam.normalizeEmailAddressOrUID(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['filter', 'filters']:
+            filters = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'ip':
+            actorIpAddress = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'todrive':
+            to_drive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam report')
+    if report == 'user':
+        while True:
+            try:
+                one_page = gapi.call(rep.userUsageReport(),
+                                     'get',
+                                     throw_reasons=throw_reasons,
+                                     date=tryDate,
+                                     userKey=userKey,
+                                     customerId=customerId,
+                                     orgUnitID=orgUnitId,
+                                     fields='warnings,usageReports',
+                                     maxResults=1)
+                warnings = one_page.get('warnings', [])
+                has_reports = bool(one_page.get('usageReports'))
+                fullData, tryDate = _check_full_data_available(
+                    warnings, tryDate, fullDataRequired, has_reports)
+                if fullData < 0:
+                    print('No user report available.')
+                    sys.exit(1)
+                if fullData == 0:
+                    continue
+                page_message = gapi.got_total_items_msg('Users', '...\n')
+                usage = gapi.get_all_pages(rep.userUsageReport(),
+                                           'get',
+                                           'usageReports',
+                                           page_message=page_message,
+                                           throw_reasons=throw_reasons,
+                                           date=tryDate,
+                                           userKey=userKey,
+                                           customerId=customerId,
+                                           orgUnitID=orgUnitId,
+                                           filters=filters,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No user report available.')
+            sys.exit(1)
+        titles = ['email', 'date']
+        csvRows = []
+        for user_report in usage:
+            if 'entity' not in user_report:
+                continue
+            row = {'email': user_report['entity']['userEmail'], 'date': tryDate}
+            for item in user_report.get('parameters', []):
+                if 'name' not in item:
+                    continue
+                name = item['name']
+                if not name in titles:
+                    titles.append(name)
+                for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                    if ptype in item:
+                        row[name] = item[ptype]
+                        break
+                else:
+                    row[name] = ''
+            csvRows.append(row)
+        display.write_csv_file(csvRows, titles, f'User Reports - {tryDate}',
+                               to_drive)
+    elif report == 'customer':
+        while True:
+            try:
+                first_page = gapi.call(rep.customerUsageReports(),
+                                       'get',
+                                       throw_reasons=throw_reasons,
+                                       customerId=customerId,
+                                       date=tryDate,
+                                       fields='warnings,usageReports')
+                warnings = first_page.get('warnings', [])
+                has_reports = bool(first_page.get('usageReports'))
+                fullData, tryDate = _check_full_data_available(
+                    warnings, tryDate, fullDataRequired, has_reports)
+                if fullData < 0:
+                    print('No customer report available.')
+                    sys.exit(1)
+                if fullData == 0:
+                    continue
+                usage = gapi.get_all_pages(rep.customerUsageReports(),
+                                           'get',
+                                           'usageReports',
+                                           throw_reasons=throw_reasons,
+                                           customerId=customerId,
+                                           date=tryDate,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No customer report available.')
+            sys.exit(1)
+        titles = ['name', 'value', 'client_id']
+        csvRows = []
+        auth_apps = list()
+        for item in usage[0]['parameters']:
+            if 'name' not in item:
+                continue
+            name = item['name']
+            if 'intValue' in item:
+                value = item['intValue']
+            elif 'msgValue' in item:
+                if name == 'accounts:authorized_apps':
+                    for subitem in item['msgValue']:
+                        app = {}
+                        for an_item in subitem:
+                            if an_item == 'client_name':
+                                app['name'] = 'App: ' + \
+                                    subitem[an_item].replace('\n', '\\n')
+                            elif an_item == 'num_users':
+                                app['value'] = f'{subitem[an_item]} users'
+                            elif an_item == 'client_id':
+                                app['client_id'] = subitem[an_item]
+                        auth_apps.append(app)
+                    continue
+                values = []
+                for subitem in item['msgValue']:
+                    if 'count' in subitem:
+                        mycount = myvalue = None
+                        for key, value in list(subitem.items()):
+                            if key == 'count':
+                                mycount = value
+                            else:
+                                myvalue = value
+                            if mycount and myvalue:
+                                values.append(f'{myvalue}:{mycount}')
+                        value = ' '.join(values)
+                    elif 'version_number' in subitem \
+                         and 'num_devices' in subitem:
+                        values.append(f'{subitem["version_number"]}:'
+                                      f'{subitem["num_devices"]}')
+                    else:
+                        continue
+                    value = ' '.join(sorted(values, reverse=True))
+            csvRows.append({'name': name, 'value': value})
+        for app in auth_apps:  # put apps at bottom
+            csvRows.append(app)
+        display.write_csv_file(csvRows,
+                               titles,
+                               f'Customer Report - {tryDate}',
+                               todrive=to_drive)
+    else:
+        page_message = gapi.got_total_items_msg('Activities', '...\n')
+        activities = gapi.get_all_pages(rep.activities(),
+                                        'list',
+                                        'items',
+                                        page_message=page_message,
+                                        applicationName=report,
+                                        userKey=userKey,
+                                        customerId=customerId,
+                                        actorIpAddress=actorIpAddress,
+                                        startTime=startTime,
+                                        endTime=endTime,
+                                        eventName=eventName,
+                                        filters=filters,
+                                        orgUnitID=orgUnitId)
+        if activities:
+            titles = ['name']
+            csvRows = []
+            for activity in activities:
+                events = activity['events']
+                del activity['events']
+                activity_row = utils.flatten_json(activity)
+                purge_parameters = True
+                for event in events:
+                    for item in event.get('parameters', []):
+                        if set(item) == set(['value', 'name']):
+                            event[item['name']] = item['value']
+                        elif set(item) == set(['intValue', 'name']):
+                            if item['name'] in ['start_time', 'end_time']:
+                                val = item.get('intValue')
+                                if val is not None:
+                                    val = int(val)
+                                    if val >= 62135683200:
+                                        event[item['name']] = \
+                                            datetime.datetime.fromtimestamp(
+                                                val-62135683200).isoformat()
+                            else:
+                                event[item['name']] = item['intValue']
+                        elif set(item) == set(['boolValue', 'name']):
+                            event[item['name']] = item['boolValue']
+                        elif set(item) == set(['multiValue', 'name']):
+                            event[item['name']] = ' '.join(item['multiValue'])
+                        elif item['name'] == 'scope_data':
+                            parts = {}
+                            for message in item['multiMessageValue']:
+                                for mess in message['parameter']:
+                                    value = mess.get(
+                                        'value',
+                                        ' '.join(mess.get('multiValue', [])))
+                                    parts[mess['name']] = parts.get(
+                                        mess['name'], []) + [value]
+                            for part, v in parts.items():
+                                if part == 'scope_name':
+                                    part = 'scope'
+                                event[part] = ' '.join(v)
+                        else:
+                            purge_parameters = False
+                    if purge_parameters:
+                        event.pop('parameters', None)
+                    row = utils.flatten_json(event)
+                    row.update(activity_row)
+                    for item in row:
+                        if item not in titles:
+                            titles.append(item)
+                    csvRows.append(row)
+            display.sort_csv_titles([
+                'name',
+            ], titles)
+            display.write_csv_file(csvRows, titles,
+                                   f'{report.capitalize()} Activity Report',
+                                   to_drive)
+
+
+def _adjust_date(errMsg):
+    match_date = re.match(
+        'Data for dates later than (.*) is not yet '
+        'available. Please check back later', errMsg)
+    if not match_date:
+        match_date = re.match('Start date can not be later than (.*)', errMsg)
+    if not match_date:
+        controlflow.system_error_exit(4, errMsg)
+    return str(match_date.group(1))
+
+
+def _check_full_data_available(warnings, tryDate, fullDataRequired,
+                               has_reports):
+    one_day = datetime.timedelta(days=1)
+    tryDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
+    # move to day before if we don't have at least one usageReport
+    if not has_reports:
+        tryDateTime -= one_day
+        return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+    for warning in warnings:
+        if warning['code'] == 'PARTIAL_DATA_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   fullDataRequired is not None and \
+                   (fullDataRequired == 'all' or app['value'] in fullDataRequired):
+                    tryDateTime -= one_day
+                    return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+        elif warning['code'] == 'DATA_NOT_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    return (-1, tryDate)
+    return (1, tryDate)

src/gam/gapi/siteverification.py

@@ -0,0 +1,188 @@
+import json
+import sys
+from urllib.parse import urlencode
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import errors as gapi_errors
+from gam.gapi.directory import customer as gapi_directory_customer
+from gam import transport
+from gam import utils
+
+import gam
+
+
+def build():
+    return gam.buildGAPIObject('siteVerification')
+
+
+def create():
+    verif = build()
+    a_domain = sys.argv[3]
+    txt_record = gapi.call(verif.webResource(),
+                           'getToken',
+                           body={
+                               'site': {
+                                   'type': 'INET_DOMAIN',
+                                   'identifier': a_domain
+                               },
+                               'verificationMethod': 'DNS_TXT'
+                           })
+    print(f'TXT Record Name:   {a_domain}')
+    print(f'TXT Record Value:  {txt_record["token"]}')
+    print()
+    cname_record = gapi.call(verif.webResource(),
+                             'getToken',
+                             body={
+                                 'site': {
+                                     'type': 'INET_DOMAIN',
+                                     'identifier': a_domain
+                                 },
+                                 'verificationMethod': 'DNS_CNAME'
+                             })
+    cname_token = cname_record['token']
+    cname_list = cname_token.split(' ')
+    cname_subdomain = cname_list[0]
+    cname_value = cname_list[1]
+    print(f'CNAME Record Name:   {cname_subdomain}.{a_domain}')
+    print(f'CNAME Record Value:  {cname_value}')
+    print('')
+    webserver_file_record = gapi.call(
+        verif.webResource(),
+        'getToken',
+        body={
+            'site': {
+                'type': 'SITE',
+                'identifier': f'http://{a_domain}/'
+            },
+            'verificationMethod': 'FILE'
+        })
+    webserver_file_token = webserver_file_record['token']
+    print(f'Saving web server verification file to: {webserver_file_token}')
+    fileutils.write_file(webserver_file_token,
+                         f'google-site-verification: {webserver_file_token}',
+                         continue_on_error=True)
+    print(f'Verification File URL: http://{a_domain}/{webserver_file_token}')
+    print()
+    webserver_meta_record = gapi.call(
+        verif.webResource(),
+        'getToken',
+        body={
+            'site': {
+                'type': 'SITE',
+                'identifier': f'http://{a_domain}/'
+            },
+            'verificationMethod': 'META'
+        })
+    print(f'Meta URL:               http://{a_domain}/')
+    print(f'Meta HTML Header Data:  {webserver_meta_record["token"]}')
+    print()
+
+
+def info():
+    verif = build()
+    sites = gapi.get_items(verif.webResource(), 'list', 'items')
+    if sites:
+        for site in sites:
+            print(f'Site: {site["site"]["identifier"]}')
+            print(f'Type: {site["site"]["type"]}')
+            print('Owners:')
+            for owner in site['owners']:
+                print(f' {owner}')
+            print()
+    else:
+        print('No Sites Verified.')
+
+
+def update():
+    verif = build()
+    a_domain = sys.argv[3]
+    verificationMethod = sys.argv[4].upper()
+    if verificationMethod == 'CNAME':
+        verificationMethod = 'DNS_CNAME'
+    elif verificationMethod in ['TXT', 'TEXT']:
+        verificationMethod = 'DNS_TXT'
+    if verificationMethod in ['DNS_TXT', 'DNS_CNAME']:
+        verify_type = 'INET_DOMAIN'
+        identifier = a_domain
+    else:
+        verify_type = 'SITE'
+        identifier = f'http://{a_domain}/'
+    body = {
+        'site': {
+            'type': verify_type,
+            'identifier': identifier
+        },
+        'verificationMethod': verificationMethod
+    }
+    try:
+        verify_result = gapi.call(
+            verif.webResource(),
+            'insert',
+            throw_reasons=[gapi_errors.ErrorReason.BAD_REQUEST],
+            verificationMethod=verificationMethod,
+            body=body)
+    except gapi_errors.GapiBadRequestError as e:
+        print(f'ERROR: {str(e)}')
+        verify_data = gapi.call(verif.webResource(), 'getToken', body=body)
+        print(f'Method:  {verify_data["method"]}')
+        print(f'Expected Token:      {verify_data["token"]}')
+        if verify_data['method'] in ['DNS_CNAME', 'DNS_TXT']:
+            simplehttp = transport.create_http()
+            base_url = 'https://dns.google/resolve?'
+            query_params = {}
+            if verify_data['method'] == 'DNS_CNAME':
+                cname_token = verify_data['token']
+                cname_list = cname_token.split(' ')
+                cname_subdomain = cname_list[0]
+                query_params['name'] = f'{cname_subdomain}.{a_domain}'
+                query_params['type'] = 'cname'
+            else:
+                query_params['name'] = a_domain
+                query_params['type'] = 'txt'
+            full_url = base_url + urlencode(query_params)
+            (_, c) = simplehttp.request(full_url, 'GET')
+            result = json.loads(c)
+            status = result['Status']
+            if status == 0 and 'Answer' in result:
+                answers = result['Answer']
+                if verify_data['method'] == 'DNS_CNAME':
+                    answer = answers[0]['data']
+                else:
+                    answer = 'no matching record found'
+                    for possible_answer in answers:
+                        possible_answer['data'] = possible_answer['data'].strip(
+                            '"')
+                        if possible_answer['data'].startswith(
+                                'google-site-verification'):
+                            answer = possible_answer['data']
+                            break
+                        print(
+                            f'Unrelated TXT record: {possible_answer["data"]}')
+                print(f'Found DNS Record: {answer}')
+            elif status == 0:
+                controlflow.system_error_exit(1, 'DNS record not found')
+            else:
+                controlflow.system_error_exit(
+                    status,
+                    DNS_ERROR_CODES_MAP.get(status, f'Unknown error {status}'))
+        return
+    print('SUCCESS!')
+    print(f'Verified:  {verify_result["site"]["identifier"]}')
+    print(f'ID:  {verify_result["id"]}')
+    print(f'Type: {verify_result["site"]["type"]}')
+    print('All Owners:')
+    try:
+        for owner in verify_result['owners']:
+            print(f' {owner}')
+    except KeyError:
+        pass
+    print()
+    print(
+        f'You can now add {a_domain} or it\'s subdomains as secondary or domain aliases of the {GC_Values[GC_DOMAIN]} Google Workspace Account.'
+    )

src/gam/gapi/storage.py

@@ -0,0 +1,81 @@
+import base64
+import os
+import re
+import sys
+
+import googleapiclient
+
+import gam
+from gam.var import *
+from gam import fileutils
+from gam import gapi
+from gam import utils
+
+
+def build_gapi():
+    return gam.buildGAPIObject('storage')
+
+
+def get_cloud_storage_object(s,
+                             bucket,
+                             object_,
+                             local_file=None,
+                             expectedMd5=None):
+    if not local_file:
+        local_file = object_
+    if os.path.exists(local_file):
+        sys.stdout.write(' File already exists. ')
+        sys.stdout.flush()
+        if expectedMd5:
+            sys.stdout.write(f'Verifying {expectedMd5} hash...')
+            sys.stdout.flush()
+            if utils.md5_matches_file(local_file, expectedMd5, False):
+                print('VERIFIED')
+                return
+            print('not verified. Downloading again and over-writing...')
+        else:
+            return  # nothing to verify, just assume we're good.
+    print(f'saving to {local_file}')
+    request = s.objects().get_media(bucket=bucket, object=object_)
+    file_path = os.path.dirname(local_file)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    f = fileutils.open_file(local_file, 'wb')
+    downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+    done = False
+    while not done:
+        status, done = downloader.next_chunk()
+        sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
+        sys.stdout.flush()
+    sys.stdout.write('\n Download complete. Flushing to disk...\n')
+    fileutils.close_file(f, True)
+    if expectedMd5:
+        f = fileutils.open_file(local_file, 'rb')
+        sys.stdout.write(f' Verifying file hash is {expectedMd5}...')
+        sys.stdout.flush()
+        utils.md5_matches_file(local_file, expectedMd5, True)
+        print('VERIFIED')
+        fileutils.close_file(f)
+
+
+def download_bucket():
+    bucket = sys.argv[3]
+    s = build_gapi()
+    page_message = gapi.got_total_items_msg('Files', '...')
+    fields = 'nextPageToken,items(name,id,md5Hash)'
+    objects = gapi.get_all_pages(s.objects(),
+                                 'list',
+                                 'items',
+                                 page_message=page_message,
+                                 bucket=bucket,
+                                 projection='noAcl',
+                                 fields=fields)
+    i = 1
+    for object_ in objects:
+        print(f'{i}/{len(objects)}')
+        expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
+        get_cloud_storage_object(s,
+                                 bucket,
+                                 object_['name'],
+                                 expectedMd5=expectedMd5)
+        i += 1

src/gam/gapi/vault.py

@@ -0,0 +1,940 @@
+import datetime
+import json
+import sys
+from time import sleep
+
+import googleapiclient.http
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import storage as gapi_storage
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+from gam import utils
+
+
+def buildGAPIObject():
+    return gam.buildGAPIObject('vault')
+
+
+def validateCollaborators(collaboratorList, cd):
+    collaborators = []
+    for collaborator in collaboratorList.split(','):
+        collaborator_id = gam.convertEmailAddressToUID(collaborator, cd)
+        if not collaborator_id:
+            controlflow.system_error_exit(
+                4, f'failed to get a UID for '
+                f'{collaborator}. Please make '
+                f'sure this is a real user.')
+        collaborators.append({'email': collaborator, 'id': collaborator_id})
+    return collaborators
+
+
+def createMatter():
+    v = buildGAPIObject()
+    matter_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    body = {'name': f'New Matter - {matter_time}'}
+    collaborators = []
+    cd = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['collaborator', 'collaborators']:
+            if not cd:
+                cd = gam.buildGAPIObject('directory')
+            collaborators.extend(validateCollaborators(sys.argv[i + 1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create matter')
+    matterId = gapi.call(v.matters(), 'create', body=body,
+                         fields='matterId')['matterId']
+    print(f'Created matter {matterId}')
+    for collaborator in collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {
+            'matterPermission': {
+                'role': 'COLLABORATOR',
+                'accountId': collaborator['id']
+            }
+        }
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+
+
+VAULT_SEARCH_METHODS_MAP = {
+    'account': 'ACCOUNT',
+    'accounts': 'ACCOUNT',
+    'entireorg': 'ENTIRE_ORG',
+    'everyone': 'ENTIRE_ORG',
+    'orgunit': 'ORG_UNIT',
+    'ou': 'ORG_UNIT',
+    'room': 'ROOM',
+    'rooms': 'ROOM',
+    'shareddrive': 'SHARED_DRIVE',
+    'shareddrives': 'SHARED_DRIVE',
+    'teamdrive': 'SHARED_DRIVE',
+    'teamdrives': 'SHARED_DRIVE',
+}
+VAULT_SEARCH_METHODS_LIST = [
+    'accounts', 'orgunit', 'shareddrives', 'rooms', 'everyone'
+]
+QUERY_ARGS = ['corpus', 'scope', 'terms', 'start', 'starttime',
+              'end', 'endtime', 'timezone', 'excludedrafts',
+              'driveversiondate', 'includeshareddrives', 'includeteamdrives',
+              'includerooms'] + list(VAULT_SEARCH_METHODS_MAP.keys())
+
+def _build_query(query, myarg, i, query_discovery):
+    if not query:
+        query = {'dataScope': 'ALL_DATA'}
+    if myarg == 'corpus':
+        query['corpus'] = sys.argv[i + 1].upper()
+        allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+                  query_discovery['properties']['corpus']['enum'])
+        if query['corpus'] not in allowed_corpuses:
+            controlflow.expected_argument_exit('corpus',
+                                               ', '.join(allowed_corpuses),
+                                               sys.argv[i + 1])
+        i += 2
+    elif myarg in VAULT_SEARCH_METHODS_MAP:
+        if query.get('searchMethod'):
+            message = f'Multiple search methods ' \
+                      f'({", ".join(VAULT_SEARCH_METHODS_LIST)})' \
+                      f'specified, only one is allowed'
+            controlflow.system_error_exit(3, message)
+        searchMethod = VAULT_SEARCH_METHODS_MAP[myarg]
+        query['searchMethod'] = searchMethod
+        if searchMethod == 'ACCOUNT':
+            query['accountInfo'] = {
+                'emails': sys.argv[i + 1].split(',')
+            }
+            i += 2
+        elif searchMethod == 'ORG_UNIT':
+            query['orgUnitInfo'] = {
+                'orgUnitId': gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])[1]
+            }
+            i += 2
+        elif searchMethod == 'SHARED_DRIVE':
+            query['sharedDriveInfo'] = {
+                'sharedDriveIds': sys.argv[i + 1].split(',')
+            }
+            i += 2
+        elif searchMethod == 'ROOM':
+            query['hangoutsChatInfo'] = {
+                'roomId': sys.argv[i + 1].split(',')
+            }
+            i += 2
+        else:
+            i += 1
+    elif myarg == 'scope':
+        query['dataScope'] = sys.argv[i + 1].upper()
+        allowed_scopes = gapi.get_enum_values_minus_unspecified(
+              query_discovery['properties']['dataScope']['enum'])
+        if query['dataScope'] not in allowed_scopes:
+            controlflow.expected_argument_exit('scope',
+                                               ', '.join(allowed_scopes),
+                                               sys.argv[i + 1])
+        i += 2
+    elif myarg in ['terms']:
+        query['terms'] = sys.argv[i + 1]
+        i += 2
+    elif myarg in ['start', 'starttime']:
+        query['startTime'] = utils.get_date_zero_time_or_full_time(
+            sys.argv[i + 1])
+        i += 2
+    elif myarg in ['end', 'endtime']:
+        query['endTime'] = utils.get_date_zero_time_or_full_time(
+            sys.argv[i + 1])
+        i += 2
+    elif myarg in ['timezone']:
+        query['timeZone'] = sys.argv[i + 1]
+        i += 2
+    elif myarg in ['excludedrafts']:
+        query['mailOptions'] = {
+           'excludeDrafts': gam.getBoolean(sys.argv[i + 1], myarg)
+        }
+        i += 2
+    elif myarg in ['driveversiondate']:
+        query.setdefault('driveOptions', {})['versionDate'] = \
+            utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+        i += 2
+    elif myarg in ['includeshareddrives', 'includeteamdrives']:
+        query.setdefault(
+            'driveOptions', {})['includeSharedDrives'] = gam.getBoolean(
+                sys.argv[i + 1], myarg)
+        i += 2
+    elif myarg in ['includerooms']:
+        query['hangoutsChatOptions'] = {
+            'includeRooms': gam.getBoolean(sys.argv[i + 1], myarg)
+        }
+        i += 2
+    return (query, i)
+
+def _validate_query(query, query_discovery):
+    if 'corpus' not in query:
+        allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+            query_discovery['properties']['corpus']['enum'])
+        controlflow.system_error_exit(3, 'you must specify a corpus. ' \
+          f'Choose one of {", ".join(allowed_corpuses)}')
+    if 'searchMethod' not in query:
+        controlflow.system_error_exit(3, f'you must specify a search method. ' \
+          'Choose one of ' \
+          f'{", ".join(VAULT_SEARCH_METHODS_LIST)}')
+
+
+def createExport():
+    v = buildGAPIObject()
+    query_discovery = v._rootDesc['schemas']['Query']
+    allowed_formats = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['MailExportOptions']['properties']
+        ['exportFormat']['enum'])
+    export_format = 'MBOX'
+    showConfidentialModeContent = None  # default to not even set
+    matterId = None
+    query = None
+    body = {'exportOptions': {}}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            body['matterId'] = matterId
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in QUERY_ARGS:
+            query, i = _build_query(query, myarg, i, query_discovery)
+        elif myarg in ['format']:
+            export_format = sys.argv[i + 1].upper()
+            if export_format not in allowed_formats:
+                controlflow.expected_argument_exit('export format',
+                                                   ', '.join(allowed_formats),
+                                                   export_format)
+            i += 2
+        elif myarg in ['showconfidentialmodecontent']:
+            showConfidentialModeContent = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg in ['region']:
+            allowed_regions = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['ExportOptions']['properties']['region']
+                ['enum'])
+            body['exportOptions']['region'] = sys.argv[i + 1].upper()
+            if body['exportOptions']['region'] not in allowed_regions:
+                controlflow.expected_argument_exit(
+                    'region', ', '.join(allowed_regions),
+                    body['exportOptions']['region'])
+            i += 2
+        elif myarg in ['includeaccessinfo']:
+            body['exportOptions'].setdefault(
+                'driveOptions', {})['includeAccessInfo'] = gam.getBoolean(
+                    sys.argv[i + 1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create export')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new export.')
+    _validate_query(query, query_discovery)
+    body['query'] = query
+    if 'name' not in body:
+        corpus_name = body['query']['corpus']
+        corpus_date = datetime.datetime.now()
+        body['name'] = f'GAM {corpus_name} export - {corpus_date}'
+    options_field = None
+    if body['query']['corpus'] == 'MAIL':
+        options_field = 'mailOptions'
+    elif body['query']['corpus'] == 'GROUPS':
+        options_field = 'groupsOptions'
+    elif body['query']['corpus'] == 'HANGOUTS_CHAT':
+        options_field = 'hangoutsChatOptions'
+    if options_field:
+        body['exportOptions'].pop('driveOptions', None)
+        body['exportOptions'][options_field] = {'exportFormat': export_format}
+        if showConfidentialModeContent is not None:
+            body['exportOptions'][options_field][
+                'showConfidentialModeContent'] = showConfidentialModeContent
+    results = gapi.call(v.matters().exports(),
+                        'create',
+                        matterId=matterId,
+                        body=body)
+    print(f'Created export {results["id"]}')
+    display.print_json(results)
+
+
+def deleteExport():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    print(f'Deleting export {sys.argv[4]} / {exportId}')
+    gapi.call(v.matters().exports(),
+              'delete',
+              matterId=matterId,
+              exportId=exportId)
+
+
+def getExportInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
+    display.print_json(export)
+
+
+def print_count():
+    v = buildGAPIObject()
+    query_discovery = v._rootDesc['schemas']['Query']
+    matterId = None
+    operation_wait = 15
+    query = None
+    body = {'view': 'ALL'}
+    name = None
+    todrive = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'operation':
+            name = sys.argv[i+1]
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in QUERY_ARGS:
+            query, i = _build_query(query, myarg, i, query_discovery)
+        elif myarg == 'wait':
+            operation_wait = int(sys.argv[i + 1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create export')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the count.')
+    if name:
+        operation = {'name': name}
+    else:
+        _validate_query(query, query_discovery)
+        body['query'] = query
+        operation = gapi.call(v.matters(), 'count', matterId=matterId, body=body)
+    print(f'Watching operation {operation["name"]}...')
+    while not operation.get('done'):
+        print(f' operation {operation["name"]} is not done yet. Checking again in {operation_wait} seconds')
+        sleep(operation_wait)
+        operation = gapi.call(v.operations(), 'get', name=operation['name'])
+    response = operation.get('response', {})
+    query = operation['metadata']['query']
+    search_method = query.get('searchMethod')
+    # ARGH count results don't include accounts with zero items.
+    # so we keep track of which accounts we searched and can report
+    # zero data for them.
+    if search_method == 'ACCOUNT':
+      query_accounts = query.get('accountInfo', [])
+    elif search_method == 'ENTIRE_ORG':
+      query_accounts = gam.getUsersToModify('all', 'users')
+    elif search_method == 'ORG_UNIT':
+      org_unit = query['orgUnitInfo']['orgUnitId']
+      query_accounts = gam.getUsersToModify('ou', org_unit)
+    mailcounts = response.get('mailCountResult', {})
+    groupcounts = response.get('groupsCountResult', {})
+    csv_rows = []
+    for a_count in [mailcounts, groupcounts]:
+        for errored_account in a_count.get('accountCountErrors', []):
+            account = errored_account.get('account')
+            csv_rows.append({'account': account, 'error': errored_account.get('errorType')})
+            if account in query_accounts: query_accounts.remove(account)
+        for account in a_count.get('nonQueryableAccounts', []):
+            csv_rows.append({'account': account, 'error': 'Not queried because not on hold'})
+            if account in query_accounts: query_accounts.remove(account)
+        for account in a_count.get('accountCounts', []):
+            email = account.get('account', {}).get('email', '')
+            csv_rows.append({'account': email, 'count': account.get('count')})
+            if email in query_accounts: query_accounts.remove(email)
+    for account in query_accounts:
+        csv_rows.append({'account': account, 'count': 0})
+    titles = ['account', 'count', 'error']
+    display.write_csv_file(csv_rows, titles, 'Vault Counts', todrive)
+
+def createHold():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Hold']['properties']['corpus']['enum'])
+    body = {'query': {}}
+    i = 3
+    query = None
+    start_time = None
+    end_time = None
+    matterId = None
+    accounts = []
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'corpus':
+            body['corpus'] = sys.argv[i + 1].upper()
+            if body['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit('corpus',
+                                                   ', '.join(allowed_corpuses),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg in ['accounts', 'users', 'groups']:
+            accounts = sys.argv[i + 1].split(',')
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])[1]
+            }
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new hold.')
+    if not body.get('name'):
+        controlflow.system_error_exit(
+            3, 'you must specify a name for the new hold.')
+    if not body.get('corpus'):
+        controlflow.system_error_exit(3, f'you must specify a corpus for ' \
+          f'the new hold. Choose one of {", ".join(allowed_corpuses)}')
+    if body['corpus'] == 'HANGOUTS_CHAT':
+        query_type = 'hangoutsChatQuery'
+    else:
+        query_type = f'{body["corpus"].lower()}Query'
+    body['query'][query_type] = {}
+    if body['corpus'] == 'DRIVE':
+        if query:
+            try:
+                body['query'][query_type] = json.loads(query)
+            except ValueError as e:
+                controlflow.system_error_exit(3, f'{str(e)}, query: {query}')
+    elif body['corpus'] in ['GROUPS', 'MAIL']:
+        if query:
+            body['query'][query_type] = {'terms': query}
+        if start_time:
+            body['query'][query_type]['startTime'] = start_time
+        if end_time:
+            body['query'][query_type]['endTime'] = end_time
+    if accounts:
+        body['accounts'] = []
+        cd = gam.buildGAPIObject('directory')
+        account_type = 'group' if body['corpus'] == 'GROUPS' else 'user'
+        for account in accounts:
+            body['accounts'].append({
+                'accountId':
+                    gam.convertEmailAddressToUID(account, cd, account_type)
+            })
+    holdId = gapi.call(v.matters().holds(),
+                       'create',
+                       matterId=matterId,
+                       body=body,
+                       fields='holdId')
+    print(f'Created hold {holdId["holdId"]}')
+
+
+def deleteHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam delete hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    print(f'Deleting hold {hold} / {holdId}')
+    gapi.call(v.matters().holds(), 'delete', matterId=matterId, holdId=holdId)
+
+
+def getHoldInfo():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam info hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    results = gapi.call(v.matters().holds(),
+                        'get',
+                        matterId=matterId,
+                        holdId=holdId)
+    cd = gam.buildGAPIObject('directory')
+    if 'accounts' in results:
+        account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
+        for i in range(0, len(results['accounts'])):
+            uid = f'uid:{results["accounts"][i]["accountId"]}'
+            acct_email = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
+            results['accounts'][i]['email'] = acct_email
+    if 'orgUnit' in results:
+        results['orgUnit']['orgUnitPath'] = gapi_directory_orgunits.info(
+            results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
+    display.print_json(results)
+
+
+def convertExportNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'exports(id,name),nextPageToken'
+    exports = gapi.get_all_pages(v.matters().exports(),
+                                 'list',
+                                 'exports',
+                                 matterId=matterId,
+                                 fields=fields)
+    for export in exports:
+        if export['name'].lower() == nameOrID:
+            return export['id']
+    controlflow.system_error_exit(
+        4, f'could not find export name {nameOrID} '
+        f'in matter {matterId}')
+
+
+def convertHoldNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'holds(holdId,name),nextPageToken'
+    holds = gapi.get_all_pages(v.matters().holds(),
+                               'list',
+                               'holds',
+                               matterId=matterId,
+                               fields=fields)
+    for hold in holds:
+        if hold['name'].lower() == nameOrID:
+            return hold['holdId']
+    controlflow.system_error_exit(
+        4, f'could not find hold name {nameOrID} '
+        f'in matter {matterId}')
+
+
+def convertMatterNameToID(v, nameOrID):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'matters(matterId,name),nextPageToken'
+    matters = gapi.get_all_pages(v.matters(),
+                                 'list',
+                                 'matters',
+                                 view='BASIC',
+                                 fields=fields)
+    for matter in matters:
+        if matter['name'].lower() == nameOrID:
+            return matter['matterId']
+    return None
+
+
+def getMatterItem(v, nameOrID):
+    matterId = convertMatterNameToID(v, nameOrID)
+    if not matterId:
+        controlflow.system_error_exit(4, f'could not find matter {nameOrID}')
+    return matterId
+
+
+def updateHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    body = {}
+    query = None
+    add_accounts = []
+    del_accounts = []
+    start_time = None
+    end_time = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])[1]
+            }
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['addusers', 'addaccounts', 'addgroups']:
+            add_accounts = sys.argv[i + 1].split(',')
+            i += 2
+        elif myarg in ['removeusers', 'removeaccounts', 'removegroups']:
+            del_accounts = sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam update hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    if query or start_time or end_time or body.get('orgUnit'):
+        fields = 'corpus,query,orgUnit'
+        old_body = gapi.call(v.matters().holds(),
+                             'get',
+                             matterId=matterId,
+                             holdId=holdId,
+                             fields=fields)
+        body['query'] = old_body['query']
+        body['corpus'] = old_body['corpus']
+        if 'orgUnit' in old_body and 'orgUnit' not in body:
+            # bah, API requires this to be sent
+            # on update even when it's not changing
+            body['orgUnit'] = old_body['orgUnit']
+        query_type = f'{body["corpus"].lower()}Query'
+        if body['corpus'] == 'DRIVE':
+            if query:
+                try:
+                    body['query'][query_type] = json.loads(query)
+                except ValueError as e:
+                    message = f'{str(e)}, query: {query}'
+                    controlflow.system_error_exit(3, message)
+        elif body['corpus'] in ['GROUPS', 'MAIL']:
+            if query:
+                body['query'][query_type]['terms'] = query
+            if start_time:
+                body['query'][query_type]['startTime'] = start_time
+            if end_time:
+                body['query'][query_type]['endTime'] = end_time
+    if body:
+        print(f'Updating hold {hold} / {holdId}')
+        gapi.call(v.matters().holds(),
+                  'update',
+                  matterId=matterId,
+                  holdId=holdId,
+                  body=body)
+    if add_accounts or del_accounts:
+        cd = gam.buildGAPIObject('directory')
+        for account in add_accounts:
+            print(f'adding {account} to hold.')
+            add_body = {'accountId': gam.convertEmailAddressToUID(account, cd)}
+            gapi.call(v.matters().holds().accounts(),
+                      'create',
+                      matterId=matterId,
+                      holdId=holdId,
+                      body=add_body)
+        for account in del_accounts:
+            print(f'removing {account} from hold.')
+            accountId = gam.convertEmailAddressToUID(account, cd)
+            gapi.call(v.matters().holds().accounts(),
+                      'delete',
+                      matterId=matterId,
+                      holdId=holdId,
+                      accountId=accountId)
+
+
+def updateMatter(action=None):
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    body = {}
+    action_kwargs = {'body': {}}
+    add_collaborators = []
+    remove_collaborators = []
+    cd = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            action = sys.argv[i + 1].lower()
+            if action not in VAULT_MATTER_ACTIONS:
+                controlflow.system_error_exit(3, f'allowed actions are ' \
+                    f'{", ".join(VAULT_MATTER_ACTIONS)}, got {action}')
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['addcollaborator', 'addcollaborators']:
+            if not cd:
+                cd = gam.buildGAPIObject('directory')
+            add_collaborators.extend(validateCollaborators(sys.argv[i + 1], cd))
+            i += 2
+        elif myarg in ['removecollaborator', 'removecollaborators']:
+            if not cd:
+                cd = gam.buildGAPIObject('directory')
+            remove_collaborators.extend(
+                validateCollaborators(sys.argv[i + 1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update matter')
+    if action == 'delete':
+        action_kwargs = {}
+    if body:
+        print(f'Updating matter {sys.argv[3]}...')
+        if 'name' not in body or 'description' not in body:
+            # bah, API requires name/description to be sent
+            # on update even when it's not changing
+            result = gapi.call(v.matters(),
+                               'get',
+                               matterId=matterId,
+                               view='BASIC')
+            body.setdefault('name', result['name'])
+            body.setdefault('description', result.get('description'))
+        gapi.call(v.matters(), 'update', body=body, matterId=matterId)
+    if action:
+        print(f'Performing {action} on matter {sys.argv[3]}')
+        gapi.call(v.matters(), action, matterId=matterId, **action_kwargs)
+    for collaborator in add_collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {
+            'matterPermission': {
+                'role': 'COLLABORATOR',
+                'accountId': collaborator['id']
+            }
+        }
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+    for collaborator in remove_collaborators:
+        print(f' removing collaborator {collaborator["email"]}')
+        gapi.call(v.matters(),
+                  'removePermissions',
+                  matterId=matterId,
+                  body={'accountId': collaborator['id']})
+
+
+def getMatterInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    result = gapi.call(v.matters(), 'get', matterId=matterId, view='FULL')
+    if 'matterPermissions' in result:
+        cd = gam.buildGAPIObject('directory')
+        for i in range(0, len(result['matterPermissions'])):
+            uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
+            user_email = gam.convertUIDtoEmailAddress(uid, cd)
+            result['matterPermissions'][i]['email'] = user_email
+    display.print_json(result)
+
+
+def downloadExport():
+    verifyFiles = True
+    extractFiles = True
+    v = buildGAPIObject()
+    s = gapi_storage.build_gapi()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i + 1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        elif myarg == 'noverify':
+            verifyFiles = False
+            i += 1
+        elif myarg == 'noextract':
+            extractFiles = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam download export')
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
+    for s_file in export['cloudStorageSink']['files']:
+        bucket = s_file['bucketName']
+        s_object = s_file['objectName']
+        filename = os.path.join(targetFolder, s_object.replace('/', '-'))
+        print(f'saving to {filename}')
+        request = s.objects().get_media(bucket=bucket, object=s_object)
+        f = fileutils.open_file(filename, 'wb')
+        downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+        done = False
+        while not done:
+            status, done = downloader.next_chunk()
+            sys.stdout.write(' Downloaded: {0:>7.2%}\r'.format(
+                status.progress()))
+            sys.stdout.flush()
+        sys.stdout.write('\n Download complete. Flushing to disk...\n')
+        fileutils.close_file(f, True)
+        if verifyFiles:
+            expected_hash = s_file['md5Hash']
+            sys.stdout.write(f' Verifying file hash is {expected_hash}...')
+            sys.stdout.flush()
+            utils.md5_matches_file(filename, expected_hash, True)
+            print('VERIFIED')
+        if extractFiles and re.search(r'\.zip$', filename):
+            gam.extract_nested_zip(filename, targetFolder)
+
+
+def printMatters():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'name', 'description', 'state']
+    titles = initialTitles[:]
+    view = 'FULL'
+    state = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            view = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg == 'matterstate':
+            valid_states = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['Matter']['properties']['state']['enum'])
+            state = sys.argv[i + 1].upper()
+            if state not in valid_states:
+                controlflow.expected_argument_exit('state',
+                                                   ', '.join(valid_states),
+                                                   state)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print matters')
+    gam.printGettingAllItems('Vault Matters', None)
+    page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
+    matters = gapi.get_all_pages(v.matters(),
+                                 'list',
+                                 'matters',
+                                 page_message=page_message,
+                                 view=view,
+                                 state=state)
+    for matter in matters:
+        display.add_row_titles_to_csv_file(utils.flatten_json(matter), csvRows,
+                                           titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Matters', todrive)
+
+
+def printExports():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'id', 'name', 'createTime', 'status']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print exports')
+    if not matters:
+        fields = 'matters(matterId),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(),
+                                             'list',
+                                             'matters',
+                                             view='BASIC',
+                                             state='OPEN',
+                                             fields=fields)
+        for matter in matters_results:
+            matterIds.append(matter['matterId'])
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving exports for matter {matterId}\n')
+        exports = gapi.get_all_pages(v.matters().exports(),
+                                     'list',
+                                     'exports',
+                                     matterId=matterId)
+        for export in exports:
+            display.add_row_titles_to_csv_file(
+                utils.flatten_json(export, flattened={'matterId': matterId}),
+                csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Exports', todrive)
+
+
+def printHolds():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'holdId', 'name', 'corpus', 'updateTime']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print holds')
+    if not matters:
+        fields = 'matters(matterId),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(),
+                                             'list',
+                                             'matters',
+                                             view='BASIC',
+                                             state='OPEN',
+                                             fields=fields)
+        for matter in matters_results:
+            matterIds.append(matter['matterId'])
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving holds for matter {matterId}\n')
+        holds = gapi.get_all_pages(v.matters().holds(),
+                                   'list',
+                                   'holds',
+                                   matterId=matterId)
+        for hold in holds:
+            display.add_row_titles_to_csv_file(
+                utils.flatten_json(hold, flattened={'matterId': matterId}),
+                csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Holds', todrive)

src/gam/transport.py

@@ -0,0 +1,102 @@
+"""Methods related to network transport."""
+
+import google_auth_httplib2
+import httplib2
+
+from gam.var import GAM_INFO
+from gam.var import GC_CA_FILE
+from gam.var import GC_TLS_MAX_VERSION
+from gam.var import GC_TLS_MIN_VERSION
+from gam.var import GC_Values
+
+
+def create_http(cache=None,
+                timeout=None,
+                override_min_tls=None,
+                override_max_tls=None):
+    """Creates a uniform HTTP transport object.
+
+  Args:
+    cache: The HTTP cache to use.
+    timeout: The cache timeout, in seconds.
+    override_min_tls: The minimum TLS version to require. If not provided, the
+      default is used.
+    override_max_tls: The maximum TLS version to require. If not provided, the
+      default is used.
+
+  Returns:
+    httplib2.Http with the specified options.
+  """
+    tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
+        GC_TLS_MIN_VERSION)
+    tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
+        GC_TLS_MAX_VERSION)
+    httpObj = httplib2.Http(ca_certs=GC_Values.get(GC_CA_FILE),
+                            tls_maximum_version=tls_maximum_version,
+                            tls_minimum_version=tls_minimum_version,
+                            cache=cache,
+                            timeout=timeout)
+    httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
+    return httpObj
+
+
+def create_request(httpObj=None):
+    """Creates a uniform Request object with a default http, if not provided.
+
+  Args:
+    httpObj: Optional httplib2.Http compatible object to be used with the request.
+      If not provided, a default HTTP will be used.
+
+  Returns:
+    Request: A google_auth_httplib2.Request compatible Request.
+  """
+    if not httpObj:
+        httpObj = create_http()
+    return Request(httpObj)
+
+
+GAM_USER_AGENT = GAM_INFO
+
+
+def _force_user_agent(user_agent):
+    """Creates a decorator which can force a user agent in HTTP headers."""
+
+    def decorator(request_method):
+        """Wraps a request method to insert a user-agent in HTTP headers."""
+
+        def wrapped_request_method(*args, **kwargs):
+            """Modifies HTTP headers to include a specified user-agent."""
+            if kwargs.get('headers') is not None:
+                if kwargs['headers'].get('user-agent'):
+                    if user_agent not in kwargs['headers']['user-agent']:
+                        # Save the existing user-agent header and tack on our own.
+                        kwargs['headers']['user-agent'] = (
+                            f'{user_agent} '
+                            f'{kwargs["headers"]["user-agent"]}')
+                else:
+                    kwargs['headers']['user-agent'] = user_agent
+            else:
+                kwargs['headers'] = {'user-agent': user_agent}
+            return request_method(*args, **kwargs)
+
+        return wrapped_request_method
+
+    return decorator
+
+
+class Request(google_auth_httplib2.Request):
+    """A Request which forces a user agent."""
+
+    @_force_user_agent(GAM_USER_AGENT)
+    def __call__(self, *args, **kwargs):
+        """Inserts the GAM user-agent header in requests."""
+        return super(Request, self).__call__(*args, **kwargs)
+
+
+class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
+    """An AuthorizedHttp which forces a user agent during requests."""
+
+    @_force_user_agent(GAM_USER_AGENT)
+    def request(self, *args, **kwargs):
+        """Inserts the GAM user-agent header in requests."""
+        return super(AuthorizedHttp, self).request(*args, **kwargs)

src/gam/transport_test.py

@@ -0,0 +1,185 @@
+"""Tests for transport."""
+
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import google_auth_httplib2
+import httplib2
+
+from gam import transport
+
+
+class CreateHttpTest(unittest.TestCase):
+
+    def setUp(self):
+        SetGlobalVariables()
+        super(CreateHttpTest, self).setUp()
+
+    def test_create_http_sets_default_values_on_http(self):
+        http = transport.create_http()
+        self.assertIsNone(http.cache)
+        self.assertIsNone(http.timeout)
+        self.assertEqual(http.tls_minimum_version,
+                         transport.GC_Values[transport.GC_TLS_MIN_VERSION])
+        self.assertEqual(http.tls_maximum_version,
+                         transport.GC_Values[transport.GC_TLS_MAX_VERSION])
+        self.assertEqual(http.ca_certs,
+                         transport.GC_Values[transport.GC_CA_FILE])
+
+    def test_create_http_sets_tls_min_version(self):
+        http = transport.create_http(override_min_tls='TLSv1_1')
+        self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
+
+    def test_create_http_sets_tls_max_version(self):
+        http = transport.create_http(override_max_tls='TLSv1_3')
+        self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
+
+    def test_create_http_sets_cache(self):
+        fake_cache = {}
+        http = transport.create_http(cache=fake_cache)
+        self.assertEqual(http.cache, fake_cache)
+
+    def test_create_http_sets_cache_timeout(self):
+        http = transport.create_http(timeout=1234)
+        self.assertEqual(http.timeout, 1234)
+
+
+class TransportTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mock_http = MagicMock(spec=httplib2.Http)
+        self.mock_response = MagicMock(spec=httplib2.Response)
+        self.mock_content = MagicMock()
+        self.mock_http.request.return_value = (self.mock_response,
+                                               self.mock_content)
+        self.mock_credentials = MagicMock()
+        self.test_uri = 'http://example.com'
+        super(TransportTest, self).setUp()
+
+    @patch.object(transport, 'create_http')
+    def test_create_request_uses_default_http(self, mock_create_http):
+        request = transport.create_request()
+        self.assertEqual(request.http, mock_create_http.return_value)
+
+    def test_create_request_uses_provided_http(self):
+        request = transport.create_request(httpObj=self.mock_http)
+        self.assertEqual(request.http, self.mock_http)
+
+    def test_create_request_returns_request_with_forced_user_agent(self):
+        request = transport.create_request()
+        self.assertIsInstance(request, transport.Request)
+
+    def test_request_is_google_auth_httplib2_compatible(self):
+        request = transport.create_request()
+        self.assertIsInstance(request, google_auth_httplib2.Request)
+
+    def test_request_call_returns_response_content(self):
+        request = transport.Request(self.mock_http)
+        response = request(self.test_uri)
+        self.assertEqual(self.mock_response.status, response.status)
+        self.assertEqual(self.mock_content, response.data)
+
+    def test_request_call_forces_user_agent_no_provided_headers(self):
+        request = transport.Request(self.mock_http)
+
+        request(self.test_uri)
+        headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', headers)
+        self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+    def test_request_call_forces_user_agent_no_agent_in_headers(self):
+        request = transport.Request(self.mock_http)
+        fake_request_headers = {
+            'some-header-thats-not-a-user-agent': 'someData'
+        }
+
+        request(self.test_uri, headers=fake_request_headers)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+        self.assertEqual('someData',
+                         final_headers['some-header-thats-not-a-user-agent'])
+
+    def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
+        request = transport.Request(self.mock_http)
+        headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+        request(self.test_uri, headers=headers_with_user_agent)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn('existing-user-agent', final_headers['user-agent'])
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+    def test_request_call_same_user_agent_already_in_headers(self):
+        request = transport.Request(self.mock_http)
+        same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+        request(self.test_uri, headers=same_user_agent_header)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        # Make sure the header wasn't duplicated
+        self.assertEqual(len(transport.GAM_USER_AGENT),
+                         len(final_headers['user-agent']))
+
+    def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
+        http = transport.AuthorizedHttp(self.mock_credentials)
+        self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
+
+    def test_authorizedhttp_request_returns_response_content(self):
+        http = transport.AuthorizedHttp(self.mock_credentials,
+                                        http=self.mock_http)
+        response, content = http.request(self.test_uri)
+        self.assertEqual(self.mock_response, response)
+        self.assertEqual(self.mock_content, content)
+
+    def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        authorized_http.request(self.test_uri)
+        headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', headers)
+        self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+    def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        fake_request_headers = {
+            'some-header-thats-not-a-user-agent': 'someData'
+        }
+
+        authorized_http.request(self.test_uri, headers=fake_request_headers)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+        self.assertEqual('someData',
+                         final_headers['some-header-thats-not-a-user-agent'])
+
+    def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
+        self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+        authorized_http.request(self.test_uri, headers=headers_with_user_agent)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn('existing-user-agent', final_headers['user-agent'])
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+    def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+        authorized_http.request(self.test_uri, headers=same_user_agent_header)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        # Make sure the header wasn't duplicated
+        self.assertEqual(len(transport.GAM_USER_AGENT),
+                         len(final_headers['user-agent']))

src/gam/utils.py

@@ -0,0 +1,396 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import datetime
+import re
+import sys
+import time
+from hashlib import md5
+from html.entities import name2codepoint
+from html.parser import HTMLParser
+import importlib
+import json
+import dateutil.parser
+import types
+
+from gam import controlflow
+from gam import fileutils
+from gam import transport
+from gam.var import *
+
+
+class LazyLoader(types.ModuleType):
+  """Lazily import a module, mainly to avoid pulling in large dependencies.
+
+  `contrib`, and `ffmpeg` are examples of modules that are large and not always
+  needed, and this allows them to only be loaded when they are used.
+  """
+
+  # The lint error here is incorrect.
+  def __init__(self, local_name, parent_module_globals, name):  # pylint: disable=super-on-old-class
+    self._local_name = local_name
+    self._parent_module_globals = parent_module_globals
+
+    super(LazyLoader, self).__init__(name)
+
+  def _load(self):
+    # Import the target module and insert it into the parent's namespace
+    module = importlib.import_module(self.__name__)
+    self._parent_module_globals[self._local_name] = module
+
+    # Update this object's dict so that if someone keeps a reference to the
+    #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
+    #   that fail).
+    self.__dict__.update(module.__dict__)
+
+    return module
+
+  def __getattr__(self, item):
+    module = self._load()
+    return getattr(module, item)
+
+  def __dir__(self):
+    module = self._load()
+    return dir(module)
+
+
+class _DeHTMLParser(HTMLParser):
+
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.__text = []
+
+    def handle_data(self, data):
+        self.__text.append(data)
+
+    def handle_charref(self, name):
+        self.__text.append(
+            chr(int(name[1:], 16)) if name.startswith('x') else chr(int(name)))
+
+    def handle_entityref(self, name):
+        cp = name2codepoint.get(name)
+        if cp:
+            self.__text.append(chr(cp))
+        else:
+            self.__text.append('&' + name)
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'p':
+            self.__text.append('\n\n')
+        elif tag == 'br':
+            self.__text.append('\n')
+        elif tag == 'a':
+            for attr in attrs:
+                if attr[0] == 'href':
+                    self.__text.append(f'({attr[1]}) ')
+                    break
+        elif tag == 'div':
+            if not attrs:
+                self.__text.append('\n')
+        elif tag in {'http:', 'https'}:
+            self.__text.append(f' ({tag}//{attrs[0][0]}) ')
+
+    def handle_startendtag(self, tag, attrs):
+        if tag == 'br':
+            self.__text.append('\n\n')
+
+    def text(self):
+        return re.sub(r'\n{2}\n+', '\n\n',
+                      re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
+
+
+def commonprefix(m):
+    '''Given a list of strings m, return string which is prefix common to all'''
+    s1 = min(m)
+    s2 = max(m)
+    for i, c in enumerate(s1):
+        if c != s2[i]:
+            return s1[:i]
+    return s1
+
+
+def dehtml(text):
+    try:
+        parser = _DeHTMLParser()
+        parser.feed(str(text))
+        parser.close()
+        return parser.text()
+    except:
+        from traceback import print_exc
+        print_exc(file=sys.stderr)
+        return text
+
+
+def indentMultiLineText(message, n=0):
+    return message.replace('\n', '\n{0}'.format(' ' * n)).rstrip()
+
+
+def flatten_json(structure, key='', path='', flattened=None, listLimit=None):
+    if flattened is None:
+        flattened = {}
+    if not isinstance(structure, (dict, list)):
+        flattened[((path + '.') if path else '') + key] = structure
+    elif isinstance(structure, list):
+        for i, item in enumerate(structure):
+            if listLimit and (i >= listLimit):
+                break
+            flatten_json(item,
+                         f'{i}',
+                         '.'.join([item for item in [path, key] if item]),
+                         flattened=flattened,
+                         listLimit=listLimit)
+    else:
+        for new_key, value in list(structure.items()):
+            if new_key in ['kind', 'etag', '@type']:
+                continue
+            if value == NEVER_TIME:
+                value = 'Never'
+            flatten_json(value,
+                         new_key,
+                         '.'.join([item for item in [path, key] if item]),
+                         flattened=flattened,
+                         listLimit=listLimit)
+    return flattened
+
+
+def formatTimestampYMD(timestamp):
+    return datetime.datetime.fromtimestamp(int(timestamp) /
+                                           1000).strftime('%Y-%m-%d')
+
+
+def formatTimestampYMDHMS(timestamp):
+    return datetime.datetime.fromtimestamp(int(timestamp) /
+                                           1000).strftime('%Y-%m-%d %H:%M:%S')
+
+
+def formatTimestampYMDHMSF(timestamp):
+    return str(datetime.datetime.fromtimestamp(int(timestamp) / 1000))
+
+
+def formatFileSize(fileSize):
+    if fileSize == 0:
+        return '0kb'
+    if fileSize < ONE_KILO_BYTES:
+        return '1kb'
+    if fileSize < ONE_MEGA_BYTES:
+        return f'{fileSize // ONE_KILO_BYTES}kb'
+    if fileSize < ONE_GIGA_BYTES:
+        return f'{fileSize // ONE_MEGA_BYTES}mb'
+    return f'{fileSize // ONE_GIGA_BYTES}gb'
+
+
+def formatMilliSeconds(millis):
+    seconds, millis = divmod(millis, 1000)
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
+
+
+def integerLimits(minVal, maxVal, item='integer'):
+    if (minVal is not None) and (maxVal is not None):
+        return f'{item} {minVal}<=x<={maxVal}'
+    if minVal is not None:
+        return f'{item} x>={minVal}'
+    if maxVal is not None:
+        return f'{item} x<={maxVal}'
+    return f'{item} x'
+
+
+def get_string(i, item, optional=False, minLen=1, maxLen=None):
+    if i < len(sys.argv):
+        argstr = sys.argv[i]
+        if argstr:
+            if (len(argstr) >= minLen) and ((maxLen is None) or
+                                            (len(argstr) <= maxLen)):
+                return argstr
+            controlflow.system_error_exit(
+                2,
+                f'expected <{integerLimits(minLen, maxLen, "string length")} for {item}>'
+            )
+        if optional or (minLen == 0):
+            return ''
+        controlflow.system_error_exit(2, f'expected a Non-empty <{item}>')
+    elif optional:
+        return ''
+    controlflow.system_error_exit(2, f'expected a <{item}>')
+
+
+def get_delta(argstr, pattern):
+    tg = pattern.match(argstr.lower())
+    if tg is None:
+        return None
+    sign = tg.group(1)
+    delta = int(tg.group(2))
+    unit = tg.group(3)
+    if unit == 'y':
+        deltaTime = datetime.timedelta(days=delta * 365)
+    elif unit == 'w':
+        deltaTime = datetime.timedelta(weeks=delta)
+    elif unit == 'd':
+        deltaTime = datetime.timedelta(days=delta)
+    elif unit == 'h':
+        deltaTime = datetime.timedelta(hours=delta)
+    elif unit == 'm':
+        deltaTime = datetime.timedelta(minutes=delta)
+    if sign == '-':
+        return -deltaTime
+    return deltaTime
+
+
+def get_delta_date(argstr):
+    deltaDate = get_delta(argstr, DELTA_DATE_PATTERN)
+    if deltaDate is None:
+        controlflow.system_error_exit(
+            2, f'expected a <{DELTA_DATE_FORMAT_REQUIRED}>; got {argstr}')
+    return deltaDate
+
+
+def get_delta_time(argstr):
+    deltaTime = get_delta(argstr, DELTA_TIME_PATTERN)
+    if deltaTime is None:
+        controlflow.system_error_exit(
+            2, f'expected a <{DELTA_TIME_FORMAT_REQUIRED}>; got {argstr}')
+    return deltaTime
+
+
+def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
+    argstr = argstr.strip()
+    if argstr:
+        if argstr[0] in ['+', '-']:
+            today = datetime.date.today()
+            argstr = (datetime.datetime(today.year, today.month, today.day) +
+                      get_delta_date(argstr)).strftime(YYYYMMDD_FORMAT)
+        try:
+            dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
+            if returnTimeStamp:
+                return time.mktime(dateTime.timetuple()) * 1000
+            if returnDateTime:
+                return dateTime
+            return argstr
+        except ValueError:
+            controlflow.system_error_exit(
+                2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>; got {argstr}')
+    elif minLen == 0:
+        return ''
+    controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>')
+
+
+def get_time_or_delta_from_now(time_string):
+    """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h') or never
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+    time_string = time_string.strip().upper()
+    if time_string:
+        if time_string == 'NEVER':
+            return NEVER_TIME
+        if time_string[0] not in ['+', '-']:
+            return time_string
+        return (datetime.datetime.utcnow() +
+                get_delta_time(time_string)).isoformat() + 'Z'
+    controlflow.system_error_exit(
+        2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+
+def get_row_filter_date_or_delta_from_now(date_string):
+    """Get an ISO 8601 date or a positive/negative delta applied to now.
+  Args:
+    date_string (string): The time or delta (e.g. '2017-09-01' or '-4y')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+    date_string = date_string.strip().upper()
+    if date_string:
+        if date_string[0] in ['+', '-']:
+            deltaDate = get_delta(date_string, DELTA_DATE_PATTERN)
+            if deltaDate is None:
+                return (False, DELTA_DATE_FORMAT_REQUIRED)
+            today = datetime.date.today()
+            return (True,
+                    (datetime.datetime(today.year, today.month, today.day) +
+                     deltaDate).isoformat() + 'Z')
+        try:
+            deltaDate = dateutil.parser.parse(date_string, ignoretz=True)
+            return (True,
+                    datetime.datetime(deltaDate.year, deltaDate.month,
+                                      deltaDate.day).isoformat() + 'Z')
+        except ValueError:
+            pass
+    return (False, YYYYMMDD_FORMAT_REQUIRED)
+
+
+def get_row_filter_time_or_delta_from_now(time_string):
+    """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  Exits:
+    2: Not a valid delta.
+  """
+    time_string = time_string.strip().upper()
+    if time_string:
+        if time_string[0] in ['+', '-']:
+            deltaTime = get_delta(time_string, DELTA_TIME_PATTERN)
+            if deltaTime is None:
+                return (False, DELTA_TIME_FORMAT_REQUIRED)
+            return (True,
+                    (datetime.datetime.utcnow() + deltaTime).isoformat() + 'Z')
+        try:
+            deltaTime = dateutil.parser.parse(time_string, ignoretz=True)
+            return (True, deltaTime.isoformat() + 'Z')
+        except ValueError:
+            pass
+    return (False, YYYYMMDDTHHMMSS_FORMAT_REQUIRED)
+
+
+def get_date_zero_time_or_full_time(time_string):
+    time_string = time_string.strip()
+    if time_string:
+        if YYYYMMDD_PATTERN.match(time_string):
+            return get_yyyymmdd(time_string) + 'T00:00:00.000Z'
+        return get_time_or_delta_from_now(time_string)
+    controlflow.system_error_exit(
+        2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+
+def md5_matches_file(local_file, expected_md5, exitOnError):
+    f = fileutils.open_file(local_file, 'rb')
+    hash_md5 = md5()
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+    actual_hash = hash_md5.hexdigest()
+    if exitOnError and actual_hash != expected_md5:
+        controlflow.system_error_exit(
+            6, f'actual hash was {actual_hash}. Exiting on corrupt file.')
+    return actual_hash == expected_md5
+
+
+URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+
+def shorten_url(long_url, httpc=None):
+    if GC_Defaults[GC_NO_SHORT_URLS]:
+        return long_url
+    if not httpc:
+        httpc = transport.create_http(timeout=10)
+    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
+    try:
+        payload = json.dumps({'long_url': long_url})
+        resp, content = httpc.request(URL_SHORTENER_ENDPOINT,
+                                      'POST',
+                                      payload,
+                                      headers=headers)
+    except:
+        return long_url
+    if resp.status != 200:
+        return long_url
+    try:
+        if isinstance(content, bytes):
+            content = content.decode()
+        return json.loads(content).get('short_url', long_url)
+    except:
+        return long_url

src/linux-build.sh

@@ -1,12 +0,0 @@
-rm -rf gam
-rm -rf build
-rm -rf dist
-rm -rf gam-$1-linux-$(arch).tar.xz
-
-export LD_LIBRARY_PATH=/usr/local/lib
-pyinstaller --clean -F --distpath=gam linux-gam.spec
-cp LICENSE gam
-cp whatsnew.txt gam
-cp GamCommands.txt gam
-
-tar cfJ gam-$1-linux-$(arch).tar.xz gam/ 

src/linux-gam.spec

@@ -1,28 +0,0 @@
-# -*- mode: python -*-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['_tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )

src/macos-build.sh

@@ -1,11 +0,0 @@
-rm -rf gam
-rm -rf build
-rm -rf dist
-rm -rf gam-$1-macos.tar.xz
-
-/Library/Frameworks/Python.framework/Versions/2.7/bin/pyinstaller --clean -F --distpath=gam macos-gam.spec
-cp LICENSE gam
-cp whatsnew.txt gam
-cp GamCommands.txt gam
-
-tar cfJ gam-$1-macos.tar.xz gam/ 

src/macos-gam.spec

@@ -1,28 +0,0 @@
-# -*- mode: python -*-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['_tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )

src/project-apis.txt

@@ -1,14 +1,19 @@
 admin.googleapis.com
 alertcenter.googleapis.com
-appsactivity.googleapis.com
 calendar-json.googleapis.com
+chat.googleapis.com
+chromemanagement.googleapis.com
+chromepolicy.googleapis.com
 classroom.googleapis.com
+cloudidentity.googleapis.com
 contacts.googleapis.com
 drive.googleapis.com
+driveactivity.googleapis.com
+iap.googleapis.com
 gmail.googleapis.com
 groupssettings.googleapis.com
+iam.googleapis.com
 licensing.googleapis.com
-plus.googleapis.com
 reseller.googleapis.com
 sheets.googleapis.com
 siteverification.googleapis.com

src/requirements-dev.txt

@@ -0,0 +1,7 @@
+# This file contains all requirements needed for GAM development work
+
+# Include all build requirements
+-r requirements.txt
+
+# Dev-specific requirements
+pre-commit

src/requirements.txt

@@ -1,7 +1,11 @@
-python-dateutil
-dnspython
-google-api-python-client
+cryptography
+distro; sys_platform == 'linux'
+filelock
+google-api-python-client>=2.1
 google-auth-httplib2
-google-oauth
-oauth2client
-passlib
+google-auth-oauthlib>=0.4.1
+google-auth>=1.11.2
+httplib2>=0.17.0
+passlib>=1.7.2
+python-dateutil
+yubikey-manager>=4.0.0

src/tools/a_atleast_b.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+#from packaging import version
+from distutils.version import LooseVersion
+import sys
+
+a = sys.argv[1]
+b = sys.argv[2]
+#result = version.parse(a) >= version.parse(b)
+result = LooseVersion(a) >= LooseVersion(b)
+if result:
+    print('OK: %s is equal or newer than %s' % (a, b))
+else:
+    print('ERROR: %s is older than %s' % (a, b))
+sys.exit(not result)

src/travis/cfg_template.json

@@ -1,12 +0,0 @@
-{
-    "_class": "OAuth2Credentials",
-    "_module": "oauth2client.client",
-    "access_token": "",
-    "client_id": "118850122376-72t6r2666n5rbjlfebftqat5qjai2def.apps.googleusercontent.com",
-    "client_secret": "",
-    "invalid": false,
-    "refresh_token": "",
-    "token_expiry": "2010-04-17T15:18:45Z",
-    "token_uri": "https://accounts.google.com/o/oauth2/token",
-    "user_agent": ""
-}

src/travis/linux-x86_64-before-install.sh

@@ -1,78 +0,0 @@
-if [ "$VMTYPE" == "test" ]; then
-  export python="python"
-  export pip="pip"
-  echo "Travis setup Python $TRAVIS_PYTHON_VERSION"
-  echo "running tests with this version"
-else
-  export whereibelong=$(pwd)
-  export dist=$(lsb_release --codename --short)
-  echo "We are running on Ubuntu $dist"
-  echo "RUNNING: apt update..."
-  sudo apt-get --yes update > /dev/null
-  echo "RUNNING: apt dist-upgrade..."
-  sudo apt-get --yes dist-upgrade > /dev/null
-  echo "Installing build tools..."
-  sudo apt-get --yes install build-essential
-
-  echo "Installing deps for python3"
-  sudo cp -v /etc/apt/sources.list /tmp
-  chmod a+rwx /tmp/sources.list
-  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
-  sudo cp -v /tmp/sources.list /etc/apt
-  sudo apt-get --yes update > /dev/null
-  sudo apt-get --yes build-dep python3
-
-  mypath=$HOME
-  echo "My Path is $mypath"
-  cpucount=$(nproc --all)
-  echo "This device has $cpucount CPUs for compiling..."
-
-
-  # Compile latest OpenSSL
-  OPENSSL_VER=1.1.1b
-  wget --quiet https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz
-  echo "Extracting OpenSSL..."
-  tar xf openssl-$OPENSSL_VER.tar.gz
-  cd openssl-$OPENSSL_VER
-  echo "Compiling OpenSSL $OPENSSL_VER..."
-  ./config shared --prefix=$mypath/ssl
-  echo "Running make for OpenSSL..."
-  make -j$cpucount -s
-  echo "Running make install for OpenSSL..."
-  make install > /dev/null
-  export LD_LIBRARY_PATH=~/ssl/lib
-  cd ~
-
-  # Compile latest Python
-  PYTHON_VER=3.7.3
-  wget --quiet https://www.python.org/ftp/python/$PYTHON_VER/Python-$PYTHON_VER.tar.xz
-  echo "Extracting Python..."
-  tar xf Python-$PYTHON_VER.tar.xz
-  cd Python-$PYTHON_VER
-  echo "Compiling Python $PYTHON_VER..."
-  ./configure --with-openssl=$mypath/ssl --enable-shared \
-	--prefix=$mypath/python --with-ensurepip=upgrade > /dev/null
-  make -j$cpucount -s
-  echo "Installing Python..."
-  make install > /dev/null
-  cd ~
-
-  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
-  python=~/python/bin/python3
-  pip=~/python/bin/pip3
-
-  $python -V
-
-  if [[ "$dist" == "xenial" ]]; then
-    echo "Installing deps for StaticX..."
-    sudo apt-get install --yes patchelf scons musl
-    $pip install git+https://github.com/JonathonReinhart/staticx.git@master
-  fi
-  $pip install pyinstaller
-  cd $whereibelong
-fi
-
-echo "Upgrading pip packages..."
-$pip freeze > upgrades.txt
-$pip install --upgrade -r upgrades.txt
-$pip install -r src/requirements.txt

src/travis/linux-x86_64-install.sh

@@ -1,32 +0,0 @@
-cd src
-if [ "$VMTYPE" == "test" ]; then
-  export gam="$python gam.py"
-  export gampath=$(readlink -e .)
-else
-  $python -OO -m PyInstaller --clean --debug -F --distpath=gam $GAMOS-gam.spec
-  export gam="gam/gam"
-  export gampath=$(readlink -e gam)
-  export GAMVERSION=`$gam version simple`
-  cp LICENSE $gampath
-  cp whatsnew.txt $gampath
-  cp GamCommands.txt $gampath
-  this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
-  GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
-  rm $gampath/lastupdatecheck.txt
-  tar cfJ $GAM_ARCHIVE gam/
-  echo "PyInstaller GAM info:"
-  du -h gam/gam
-  time $gam version extended
-
-  if [[ "$dist" == "xenial" ]]; then
-    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
-    $python -OO -m staticx gam/gam gam/gam-staticx
-    strip gam/gam-staticx
-    rm gam/gam
-    mv gam/gam-staticx gam/gam
-    tar cfJ $GAM_LEGACY_ARCHIVE gam/
-    echo "Legacy StaticX GAM info:"
-    du -h gam/gam
-    time $gam version extended
-  fi
-fi

src/travis/osx-x86_64-before-install.sh

@@ -1,48 +0,0 @@
-mypath=$HOME
-whereibelong=$(pwd)
-echo "Brew installing xz..."
-brew install xz > /dev/null
-
-# Compile latest OpenSSL
-OPENSSL_VER=1.1.1b
-wget --quiet https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz
-echo "Extracting OpenSSL..."
-tar xf openssl-$OPENSSL_VER.tar.gz
-cd openssl-$OPENSSL_VER
-echo "Compiling OpenSSL $OPENSSL_VER..."
-./config shared --prefix=$mypath/ssl
-echo "Running make for OpenSSL..."
-make -j$cpucount -s
-echo "Running make install for OpenSSL..."
-make install > /dev/null
-export LD_LIBRARY_PATH=~/ssl/lib
-cd ~
-
-# Compile latest Python
-PYTHON_VER=3.7.3
-wget --quiet https://www.python.org/ftp/python/$PYTHON_VER/Python-$PYTHON_VER.tar.xz
-echo "Extracting Python..."
-tar xf Python-$PYTHON_VER.tar.xz
-cd Python-$PYTHON_VER
-echo "Compiling Python $PYTHON_VER..."
-./configure --with-openssl=$mypath/ssl --enable-shared \
-		--prefix=$mypath/python --with-ensurepip=upgrade > /dev/null
-make -j$cpucount -s
-echo "Installing Python..."
-make install > /dev/null
-cd ~
-
-export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
-python=~/python/bin/python3
-pip=~/python/bin/pip3
-
-$python -V
-
-cd $whereibelong
-
-export PATH=/usr/local/opt/python/libexec/bin:$PATH
-$pip install --upgrade pip
-$pip freeze > upgrades.txt
-$pip install --upgrade -r upgrades.txt
-$pip install -r src/requirements.txt
-$pip install pyinstaller

src/travis/osx-x86_64-install.sh

@@ -1,12 +0,0 @@
-cd src
-$python -OO -m PyInstaller --clean --debug -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=gam
-$gam version extended
-export GAMVERSION=`gam/gam version simple`
-cp LICENSE gam
-cp whatsnew.txt gam
-cp GamCommands.txt gam
-GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.tar.xz
-rm gam/lastupdatecheck.txt
-tar cfJ $GAM_ARCHIVE gam/

src/travis/set_printer_csv_filter.sh

@@ -1 +0,0 @@
-export GAM_CSV_ROW_FILTER='{"type": "regex:^GOOGLE"}'

src/travis/svars-write.py

@@ -1,15 +0,0 @@
-import sys
-import json
-import os
-
-cfg = json.load(sys.stdin)
-cfg['client_secret'] = os.getenv('client_secret')
-jid = os.getenv('jid')
-cfg['refresh_token'] = os.getenv('refresh_%s' % jid)
-vmtype = os.getenv('VMTYPE')
-if vmtype == 'test':
-    out_file = 'oauth2.txt'
-else:
-    out_file = 'gam/oauth2.txt'
-with open(out_file, 'w') as f:
-  json.dump(cfg, f)

src/travis/windows-x86-before-install.sh

@@ -1,11 +0,0 @@
-until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cinst -y --forcex86 python3
-cinst -y --forcex86 openssl.light
-cp -v /c/Program\ Files/OpenSSL/*.dll /c/Python37/DLLs
-export PATH=$PATH:/c/Python37/scripts
-until cinst -y wixtoolset; do echo "trying again..."; done
-pip install --upgrade pip
-pip freeze > upgrades.txt
-pip install --upgrade -r upgrades.txt
-pip install -r src/requirements.txt
-pip install pyinstaller

src/travis/windows-x86-install.sh

@@ -1,18 +0,0 @@
-cd src
-pyinstaller --clean -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=$(readlink -e gam)
-$gam version extended
-export GAMVERSION=`$gam version simple`
-rm gam/lastupdatecheck.txt
-cp LICENSE gam
-cp GamCommands.txt gam
-cp whatsnew.txt gam
-cp gam-setup.bat gam
-GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
-/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
-mkdir gam-64
-cp -rf gam/* gam-64/;
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch x86 gam.wxs
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
-rm *.wixpdb

src/travis/windows-x86_64-before-install.sh

@@ -1,11 +0,0 @@
-until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cinst -y python3
-cinst -y openssl.light
-cp -v /c/Program\ Files/OpenSSL/bin/*.dll /c/Python37/DLLs
-export PATH=$PATH:/c/Python37/scripts
-until cinst -y wixtoolset; do echo "trying again..."; done
-pip install --upgrade pip
-pip freeze > upgrades.txt
-pip install --upgrade -r upgrades.txt
-pip install -r src/requirements.txt
-pip install pyinstaller

src/travis/windows-x86_64-install.sh

@@ -1,18 +0,0 @@
-cd src
-pyinstaller --clean -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=$(readlink -e gam)
-$gam version
-export GAMVERSION=`$gam version simple`
-rm gam/lastupdatecheck.txt
-cp LICENSE gam
-cp GamCommands.txt gam
-cp whatsnew.txt gam
-cp gam-setup.bat gam
-GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
-/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
-mkdir gam-64
-cp -rf gam/* gam-64/;
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch x64 gam.wxs
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
-rm *.wixpdb

src/utils.py

@@ -1,85 +0,0 @@
-import collections
-import re
-import sys
-from html.entities import name2codepoint
-from html.parser import HTMLParser
-from var import GM_Globals, GM_WINDOWS, GM_SYS_ENCODING
-
-ONE_KILO_BYTES = 1000
-ONE_MEGA_BYTES = 1000000
-ONE_GIGA_BYTES = 1000000000
-
-def convertUTF8(data):
-  return data
-
-class _DeHTMLParser(HTMLParser):
-  def __init__(self):
-    HTMLParser.__init__(self)
-    self.__text = []
-
-  def handle_data(self, data):
-    self.__text.append(data)
-
-  def handle_charref(self, name):
-    self.__text.append(chr(int(name[1:], 16)) if name.startswith('x') else chr(int(name)))
-
-  def handle_entityref(self, name):
-    cp = name2codepoint.get(name)
-    if cp:
-      self.__text.append(chr(cp))
-    else:
-      self.__text.append('&'+name)
-
-  def handle_starttag(self, tag, attrs):
-    if tag == 'p':
-      self.__text.append('\n\n')
-    elif tag == 'br':
-      self.__text.append('\n')
-    elif tag == 'a':
-      for attr in attrs:
-        if attr[0] == 'href':
-          self.__text.append('({0}) '.format(attr[1]))
-          break
-    elif tag == 'div':
-      if not attrs:
-        self.__text.append('\n')
-    elif tag in ['http:', 'https']:
-      self.__text.append(' ({0}//{1}) '.format(tag, attrs[0][0]))
-
-  def handle_startendtag(self, tag, attrs):
-    if tag == 'br':
-      self.__text.append('\n\n')
-
-  def text(self):
-    return re.sub(r'\n{2}\n+', '\n\n', re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
-
-def dehtml(text):
-  try:
-    parser = _DeHTMLParser()
-    parser.feed(str(text))
-    parser.close()
-    return parser.text()
-  except:
-    from traceback import print_exc
-    print_exc(file=sys.stderr)
-    return text
-
-def indentMultiLineText(message, n=0):
-  return message.replace('\n', '\n{0}'.format(' '*n)).rstrip()
-
-def formatFileSize(fileSize):
-  if fileSize == 0:
-    return '0kb'
-  if fileSize < ONE_KILO_BYTES:
-    return '1kb'
-  if fileSize < ONE_MEGA_BYTES:
-    return '{0}kb'.format(fileSize//ONE_KILO_BYTES)
-  if fileSize < ONE_GIGA_BYTES:
-    return '{0}mb'.format(fileSize//ONE_MEGA_BYTES)
-  return '{0}gb'.format(fileSize//ONE_GIGA_BYTES)
-
-def formatMilliSeconds(millis):
-  seconds, millis = divmod(millis, 1000)
-  minutes, seconds = divmod(seconds, 60)
-  hours, minutes = divmod(minutes, 60)
-  return '%02d:%02d:%02d' % (hours, minutes, seconds)

src/versionhistory-v1.json

@@ -0,0 +1,486 @@
+{
+  "revision": "20210322",
+  "name": "versionhistory",
+  "mtlsRootUrl": "https://versionhistory.mtls.googleapis.com/",
+  "version_module": true,
+  "basePath": "",
+  "title": "Version History API",
+  "kind": "discovery#restDescription",
+  "servicePath": "",
+  "ownerDomain": "google.com",
+  "parameters": {
+    "access_token": {
+      "location": "query",
+      "description": "OAuth access token.",
+      "type": "string"
+    },
+    "alt": {
+      "default": "json",
+      "location": "query",
+      "enum": [
+        "json",
+        "media",
+        "proto"
+      ],
+      "type": "string",
+      "enumDescriptions": [
+        "Responses with Content-Type of application/json",
+        "Media download with context-dependent Content-Type",
+        "Responses with Content-Type of application/x-protobuf"
+      ],
+      "description": "Data format for response."
+    },
+    "quotaUser": {
+      "type": "string",
+      "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.",
+      "location": "query"
+    },
+    "$.xgafv": {
+      "location": "query",
+      "enumDescriptions": [
+        "v1 error format",
+        "v2 error format"
+      ],
+      "enum": [
+        "1",
+        "2"
+      ],
+      "description": "V1 error format.",
+      "type": "string"
+    },
+    "fields": {
+      "type": "string",
+      "description": "Selector specifying which fields to include in a partial response.",
+      "location": "query"
+    },
+    "key": {
+      "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
+      "location": "query",
+      "type": "string"
+    },
+    "callback": {
+      "location": "query",
+      "description": "JSONP",
+      "type": "string"
+    },
+    "oauth_token": {
+      "description": "OAuth 2.0 token for the current user.",
+      "location": "query",
+      "type": "string"
+    },
+    "upload_protocol": {
+      "location": "query",
+      "type": "string",
+      "description": "Upload protocol for media (e.g. \"raw\", \"multipart\")."
+    },
+    "prettyPrint": {
+      "description": "Returns response with indentations and line breaks.",
+      "default": "true",
+      "location": "query",
+      "type": "boolean"
+    },
+    "uploadType": {
+      "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").",
+      "type": "string",
+      "location": "query"
+    }
+  },
+  "ownerName": "Google",
+  "protocol": "rest",
+  "resources": {
+    "platforms": {
+      "methods": {
+        "list": {
+          "httpMethod": "GET",
+          "parameterOrder": [
+            "parent"
+          ],
+          "response": {
+            "$ref": "ListPlatformsResponse"
+          },
+          "path": "v1/{+parent}/platforms",
+          "description": "Returns list of platforms that are avaialble for a given product. The resource \"product\" has no resource name in its name.",
+          "flatPath": "v1/{v1Id}/platforms",
+          "id": "versionhistory.platforms.list",
+          "parameters": {
+            "parent": {
+              "location": "path",
+              "pattern": "^[^/]+$",
+              "type": "string",
+              "required": true,
+              "description": "Required. The product, which owns this collection of platforms. Format: {product}"
+            },
+            "pageSize": {
+              "format": "int32",
+              "description": "Optional. Optional limit on the number of channels to include in the response. If unspecified, the server will pick an appropriate default.",
+              "type": "integer",
+              "location": "query"
+            },
+            "pageToken": {
+              "location": "query",
+              "description": "Optional. A page token, received from a previous `ListChannels` call. Provide this to retrieve the subsequent page.",
+              "type": "string"
+            }
+          }
+        }
+      },
+      "resources": {
+        "channels": {
+          "resources": {
+            "versions": {
+              "resources": {
+                "releases": {
+                  "methods": {
+                    "list": {
+                      "id": "versionhistory.platforms.channels.versions.releases.list",
+                      "path": "v1/{+parent}/releases",
+                      "httpMethod": "GET",
+                      "parameterOrder": [
+                        "parent"
+                      ],
+                      "response": {
+                        "$ref": "ListReleasesResponse"
+                      },
+                      "flatPath": "v1/{v1Id}/platforms/{platformsId}/channels/{channelsId}/versions/{versionsId}/releases",
+                      "parameters": {
+                        "parent": {
+                          "type": "string",
+                          "required": true,
+                          "description": "Required. The version, which owns this collection of releases. Format: {product}/platforms/{platform}/channels/{channel}/versions/{version}",
+                          "pattern": "^[^/]+/platforms/[^/]+/channels/[^/]+/versions/[^/]+$",
+                          "location": "path"
+                        },
+                        "filter": {
+                          "type": "string",
+                          "location": "query",
+                          "description": "Optional. Filter string. Format is a comma separated list of All comma separated filter clauses are conjoined with a logical \"and\". Valid field_names are \"version\", \"name\", \"platform\", \"channel\", \"fraction\" \"starttime\", and \"endtime\". Valid operators are \"\u003c\", \"\u003c=\", \"=\", \"\u003e=\", and \"\u003e\". Channel comparison is done by distance from stable. must be a valid channel when filtering by channel. Ex) stable \u003c beta, beta \u003c dev, canary \u003c canary_asan. Version comparison is done numerically. Ex) 1.0.0.8 \u003c 1.0.0.10. If version is not entirely written, the version will be appended with 0 for the missing fields. Ex) version \u003e 80 becoms version \u003e 80.0.0.0 When filtering by starttime or endtime, string must be in RFC 3339 date string format. Name and platform are filtered by string comparison. Ex) \"...?filter=channel\u003c=beta, version \u003e= 80 Ex) \"...?filter=version \u003e 80, version \u003c 81 Ex) \"...?filter=starttime\u003e2020-01-01T00:00:00Z"
+                        },
+                        "orderBy": {
+                          "location": "query",
+                          "description": "Optional. Ordering string. Valid order_by strings are \"version\", \"name\", \"starttime\", \"endtime\", \"platform\", \"channel\", and \"fraction\". Optionally, you can append \"desc\" or \"asc\" to specify the sorting order. Multiple order_by strings can be used in a comma separated list. Ordering by channel will sort by distance from the stable channel (not alphabetically). A list of channels sorted in this order is: stable, beta, dev, canary, and canary_asan. Sorting by name may cause unexpected behaviour as it is a naive string sort. For example, 1.0.0.8 will be before 1.0.0.10 in descending order. If order_by is not specified the response will be sorted by starttime in descending order. Ex) \"...?order_by=starttime asc\" Ex) \"...?order_by=platform desc, channel, startime desc\"",
+                          "type": "string"
+                        },
+                        "pageSize": {
+                          "location": "query",
+                          "format": "int32",
+                          "description": "Optional. Optional limit on the number of releases to include in the response. If unspecified, the server will pick an appropriate default.",
+                          "type": "integer"
+                        },
+                        "pageToken": {
+                          "description": "Optional. A page token, received from a previous `ListReleases` call. Provide this to retrieve the subsequent page.",
+                          "location": "query",
+                          "type": "string"
+                        }
+                      },
+                      "description": "Returns list of releases of the given version."
+                    }
+                  }
+                }
+              },
+              "methods": {
+                "list": {
+                  "response": {
+                    "$ref": "ListVersionsResponse"
+                  },
+                  "path": "v1/{+parent}/versions",
+                  "parameters": {
+                    "pageSize": {
+                      "location": "query",
+                      "format": "int32",
+                      "description": "Optional. Optional limit on the number of versions to include in the response. If unspecified, the server will pick an appropriate default.",
+                      "type": "integer"
+                    },
+                    "pageToken": {
+                      "description": "Optional. A page token, received from a previous `ListVersions` call. Provide this to retrieve the subsequent page.",
+                      "location": "query",
+                      "type": "string"
+                    },
+                    "parent": {
+                      "required": true,
+                      "location": "path",
+                      "description": "Required. The channel, which owns this collection of versions. Format: {product}/platforms/{platform}/channels/{channel}",
+                      "pattern": "^[^/]+/platforms/[^/]+/channels/[^/]+$",
+                      "type": "string"
+                    },
+                    "orderBy": {
+                      "type": "string",
+                      "location": "query",
+                      "description": "Optional. Ordering string. Valid order_by strings are \"version\", \"name\", \"platform\", and \"channel\". Optionally, you can append \" desc\" or \" asc\" to specify the sorting order. Multiple order_by strings can be used in a comma separated list. Ordering by channel will sort by distance from the stable channel (not alphabetically). A list of channels sorted in this order is: stable, beta, dev, canary, and canary_asan. Sorting by name may cause unexpected behaviour as it is a naive string sort. For example, 1.0.0.8 will be before 1.0.0.10 in descending order. If order_by is not specified the response will be sorted by version in descending order. Ex) \"...?order_by=version asc\" Ex) \"...?order_by=platform desc, channel, version\""
+                    },
+                    "filter": {
+                      "description": "Optional. Filter string. Format is a comma separated list of All comma separated filter clauses are conjoined with a logical \"and\". Valid field_names are \"version\", \"name\", \"platform\", and \"channel\". Valid operators are \"\u003c\", \"\u003c=\", \"=\", \"\u003e=\", and \"\u003e\". Channel comparison is done by distance from stable. Ex) stable \u003c beta, beta \u003c dev, canary \u003c canary_asan. Version comparison is done numerically. If version is not entirely written, the version will be appended with 0 in missing fields. Ex) version \u003e 80 becoms version \u003e 80.0.0.0 Name and platform are filtered by string comparison. Ex) \"...?filter=channel\u003c=beta, version \u003e= 80 Ex) \"...?filter=version \u003e 80, version \u003c 81",
+                      "location": "query",
+                      "type": "string"
+                    }
+                  },
+                  "id": "versionhistory.platforms.channels.versions.list",
+                  "parameterOrder": [
+                    "parent"
+                  ],
+                  "description": "Returns list of version for the given platform/channel.",
+                  "flatPath": "v1/{v1Id}/platforms/{platformsId}/channels/{channelsId}/versions",
+                  "httpMethod": "GET"
+                }
+              }
+            }
+          },
+          "methods": {
+            "list": {
+              "response": {
+                "$ref": "ListChannelsResponse"
+              },
+              "parameterOrder": [
+                "parent"
+              ],
+              "parameters": {
+                "pageToken": {
+                  "type": "string",
+                  "location": "query",
+                  "description": "Optional. A page token, received from a previous `ListChannels` call. Provide this to retrieve the subsequent page."
+                },
+                "parent": {
+                  "location": "path",
+                  "type": "string",
+                  "required": true,
+                  "pattern": "^[^/]+/platforms/[^/]+$",
+                  "description": "Required. The platform, which owns this collection of channels. Format: {product}/platforms/{platform}"
+                },
+                "pageSize": {
+                  "format": "int32",
+                  "type": "integer",
+                  "description": "Optional. Optional limit on the number of channels to include in the response. If unspecified, the server will pick an appropriate default.",
+                  "location": "query"
+                }
+              },
+              "path": "v1/{+parent}/channels",
+              "httpMethod": "GET",
+              "flatPath": "v1/{v1Id}/platforms/{platformsId}/channels",
+              "id": "versionhistory.platforms.channels.list",
+              "description": "Returns list of channels that are available for a given platform."
+            }
+          }
+        }
+      }
+    }
+  },
+  "description": "Version History API - Prod",
+  "discoveryVersion": "v1",
+  "schemas": {
+    "Channel": {
+      "id": "Channel",
+      "type": "object",
+      "description": "Each Channel is owned by a Platform and owns a collection of versions. Possible Channels are listed in the Channel enum below. Not all Channels are available for every Platform (e.g. CANARY does not exist for LINUX).",
+      "properties": {
+        "name": {
+          "description": "Channel name. Format is \"{product}/platforms/{platform}/channels/{channel}\"",
+          "type": "string"
+        },
+        "channelType": {
+          "description": "Type of channel.",
+          "enumDescriptions": [
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            ""
+          ],
+          "type": "string",
+          "enum": [
+            "CHANNEL_TYPE_UNSPECIFIED",
+            "STABLE",
+            "BETA",
+            "DEV",
+            "CANARY",
+            "CANARY_ASAN",
+            "ALL",
+            "EXTENDED"
+          ]
+        }
+      }
+    },
+    "Interval": {
+      "description": "Represents a time interval, encoded as a Timestamp start (inclusive) and a Timestamp end (exclusive). The start must be less than or equal to the end. When the start equals the end, the interval is empty (matches no time). When both start and end are unspecified, the interval matches any time.",
+      "type": "object",
+      "id": "Interval",
+      "properties": {
+        "endTime": {
+          "type": "string",
+          "format": "google-datetime",
+          "description": "Optional. Exclusive end of the interval. If specified, a Timestamp matching this interval will have to be before the end."
+        },
+        "startTime": {
+          "format": "google-datetime",
+          "type": "string",
+          "description": "Optional. Inclusive start of the interval. If specified, a Timestamp matching this interval will have to be the same or after the start."
+        }
+      }
+    },
+    "ListVersionsResponse": {
+      "description": "Response message for ListVersions.",
+      "type": "object",
+      "properties": {
+        "versions": {
+          "type": "array",
+          "description": "The list of versions.",
+          "items": {
+            "$ref": "Version"
+          }
+        },
+        "nextPageToken": {
+          "description": "A token, which can be sent as `page_token` to retrieve the next page. If this field is omitted, there are no subsequent pages.",
+          "type": "string"
+        }
+      },
+      "id": "ListVersionsResponse"
+    },
+    "ListChannelsResponse": {
+      "description": "Response message for ListChannels.",
+      "id": "ListChannelsResponse",
+      "type": "object",
+      "properties": {
+        "nextPageToken": {
+          "description": "A token, which can be sent as `page_token` to retrieve the next page. If this field is omitted, there are no subsequent pages.",
+          "type": "string"
+        },
+        "channels": {
+          "description": "The list of channels.",
+          "type": "array",
+          "items": {
+            "$ref": "Channel"
+          }
+        }
+      }
+    },
+    "Platform": {
+      "properties": {
+        "platformType": {
+          "enum": [
+            "PLATFORM_TYPE_UNSPECIFIED",
+            "WIN",
+            "WIN64",
+            "MAC",
+            "LINUX",
+            "ANDROID",
+            "WEBVIEW",
+            "IOS",
+            "ALL",
+            "MAC_ARM64",
+            "LACROS"
+          ],
+          "description": "Type of platform.",
+          "enumDescriptions": [
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            ""
+          ],
+          "type": "string"
+        },
+        "name": {
+          "description": "Platform name. Format is \"{product}/platforms/{platform}\"",
+          "type": "string"
+        }
+      },
+      "id": "Platform",
+      "type": "object",
+      "description": "Each Platform is owned by a Product and owns a collection of channels. Available platforms are listed in Platform enum below. Not all Channels are available for every Platform (e.g. CANARY does not exist for LINUX)."
+    },
+    "Version": {
+      "properties": {
+        "version": {
+          "description": "String containing just the version number. e.g. \"84.0.4147.38\"",
+          "type": "string"
+        },
+        "name": {
+          "description": "Version name. Format is \"{product}/platforms/{platform}/channels/{channel}/versions/{version}\" e.g. \"chrome/platforms/win/channels/beta/versions/84.0.4147.38\"",
+          "type": "string"
+        }
+      },
+      "id": "Version",
+      "type": "object",
+      "description": "Each Version is owned by a Channel. A Version only displays the Version number (e.g. 84.0.4147.38). A Version owns a collection of releases."
+    },
+    "ListPlatformsResponse": {
+      "description": "Response message for ListPlatforms.",
+      "id": "ListPlatformsResponse",
+      "type": "object",
+      "properties": {
+        "nextPageToken": {
+          "type": "string",
+          "description": "A token, which can be sent as `page_token` to retrieve the next page. If this field is omitted, there are no subsequent pages."
+        },
+        "platforms": {
+          "type": "array",
+          "items": {
+            "$ref": "Platform"
+          },
+          "description": "The list of platforms."
+        }
+      }
+    },
+    "ListReleasesResponse": {
+      "type": "object",
+      "id": "ListReleasesResponse",
+      "description": "Response message for ListReleases.",
+      "properties": {
+        "nextPageToken": {
+          "description": "A token, which can be sent as `page_token` to retrieve the next page. If this field is omitted, there are no subsequent pages.",
+          "type": "string"
+        },
+        "releases": {
+          "description": "The list of releases.",
+          "items": {
+            "$ref": "Release"
+          },
+          "type": "array"
+        }
+      }
+    },
+    "Release": {
+      "properties": {
+        "fraction": {
+          "format": "double",
+          "type": "number",
+          "description": "Rollout fraction. This fraction indicates the fraction of people that should receive this version in this release. If the fraction is not specified in ReleaseManager, the API will assume fraction is 1."
+        },
+        "version": {
+          "type": "string",
+          "description": "String containing just the version number. e.g. \"84.0.4147.38\""
+        },
+        "name": {
+          "type": "string",
+          "description": "Release name. Format is \"{product}/platforms/{platform}/channels/{channel}/versions/{version}/releases/{release}\""
+        },
+        "serving": {
+          "description": "Timestamp interval of when the release was live. If end_time is unspecified, the release is currently live.",
+          "$ref": "Interval"
+        }
+      },
+      "type": "object",
+      "description": "A Release is owned by a Version. A Release contains information about the release(s) of its parent version. This includes when the release began and ended, as well as what percentage it was released at. If the version is released again, or if the serving percentage changes, it will create another release under the version.",
+      "id": "Release"
+    }
+  },
+  "fullyEncodeReservedExpansion": true,
+  "documentationLink": "https://developers.chrome.com/",
+  "icons": {
+    "x16": "http://www.google.com/images/icons/product/search-16.gif",
+    "x32": "http://www.google.com/images/icons/product/search-32.gif"
+  },
+  "baseUrl": "https://versionhistory.googleapis.com/",
+  "batchPath": "batch",
+  "version": "v1",
+  "canonicalName": "Version History",
+  "id": "versionhistory:v1",
+  "rootUrl": "https://versionhistory.googleapis.com/"
+}