Merge branch 'master' of https://github.com/jay0lee/GAM

* Ensure that customer_id, domain and an admin email address are present for DASA

* Fix typos

.github/ISSUE_TEMPLATE.txt

@@ -5,7 +5,7 @@ Please confirm the following:
 * I am typing the command as described in the GAM Wiki at https://github.com/jay0lee/gam/wiki
 
 Full steps to reproduce the issue:
-1. 
+1.
 2.
 3.
 

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - pinned
+  - enhancement
+  - help wanted
+  - security
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+    python: python3.7
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: double-quote-string-fixer
+    -   id: check-yaml
+    -   id: check-docstring-first
+    -   id: name-tests-test
+    -   id: requirements-txt-fixer
+    -   id: check-merge-conflict
+
+-   repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.30.0
+    hooks:
+    -   id: yapf
+        args: [--style=google, --in-place]
+
+-   repo: https://github.com/PyCQA/pylint
+    rev: pylint-2.5.0
+    hooks:
+    -   id: pylint
+        args: [--output-format=colorized]

.travis.yml

@@ -0,0 +1,249 @@
+if: tag IS blank
+os: linux
+language: python
+dist: xenial
+
+env:
+  global:
+    - BUILD_PYTHON_VERSION=3.8.5
+    - MIN_PYTHON_VERSION=3.8.5
+    - BUILD_OPENSSL_VERSION=1.1.1g
+    - MIN_OPENSSL_VERSION=1.1.1g
+    - PATCHELF_VERSION=0.11
+    - PYINSTALLER_COMMIT=ad39eb8df209d02636399ffdc44521a97886cf8c
+cache:
+  directories:
+    - $HOME/.cache/pip
+    - $HOME/python
+    - $HOME/ssl
+
+jobs:
+  allow_failures:
+    - python: nightly
+  fast_finish: true
+  include:
+    - os: linux
+      name: "Linux 64-bit Focal"
+      dist: focal
+      language: shell
+    - os: linux
+      name: "Linux 64-bit Bionic"
+      dist: bionic
+      language: shell
+    - os: linux
+      name: "Linux 64-bit Xenial"
+      dist: xenial
+      language: shell
+    - os: linux
+      name: "Linux ARM64 Focal"
+      dist: focal
+      language: shell
+      arch: arm64
+      filter_secrets: false
+    - os: linux
+      dist: bionic
+      arch: arm64
+      name: "Linux ARM64 Bionic"
+      language: shell
+      filter_secrets: false
+    - os: linux
+      dist: xenial
+      arch: arm64
+      name: "Linux ARM64 Xenial"
+      language: shell
+      filter_secrets: false
+    - os: linux
+      name: "Python 3.6 Source Testing"
+      language: python
+      python: 3.6
+    - os: linux
+      name: "Python 3.7 Source Testing"
+      language: python
+      python: 3.7
+    - os: linux
+      name: "Python 3.9 dev Source Testing"
+      language: python
+      python: 3.9-dev
+      dist: focal
+    - os: linux
+      name: "Python trunk nightly Source Testing"
+      language: python
+      python: nightly
+    - os: linux
+      name: "Python PyPi Source Testing"
+      language: python
+      python: pypy3
+    - os: osx
+      name: "MacOS 10.13"
+      language: generic
+      osx_image: xcode10.1
+    - os: osx
+      name: "MacOS 10.14"
+      language: generic
+      osx_image: xcode11.3
+    - os: osx
+      name: "MacOS 10.15"
+      language: generic
+      osx_image: xcode11.7
+    - os: osx
+      name: "MacOS 10.15 Universal Testing"
+      language: generic
+      osx_image: xcode12u
+    - os: windows
+      name: "Windows 64-bit"
+      language: shell
+    - os: windows
+      name: "Windows 32-bit"
+      language: shell
+
+before_install:
+- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    export GAMOS="macos";
+  else
+    export GAMOS="${TRAVIS_OS_NAME}";
+  fi
+- if [ "${TRAVIS_JOB_NAME}" == "Windows 32-bit" ]; then
+    export PLATFORM="x86";
+  elif [ "${TRAVIS_CPU_ARCH}" == "amd64" ]; then
+    export PLATFORM="x86_64";
+  else
+    export PLATFORM="${TRAVIS_CPU_ARCH}";
+  fi
+- source src/travis/${TRAVIS_OS_NAME}-before-install.sh
+
+install:
+- source src/travis/${TRAVIS_OS_NAME}-install.sh
+
+script:
+# Discover and run all Python unit tests. Buffer output so that it's not sent to the build log.
+- $python -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
+- touch $gampath/nobrowser.txt
+- $gam version extended
+- $gam version | grep travis # travis should be part of the path (not /tmp or such)
+# determine which Python version GAM is built with and ensure it's at least build version from above.
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]];  then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $MIN_PYTHON_VERSION; fi
+# determine which OpenSSL version GAM is built with and ensure it's at least build version from above.
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $MIN_OPENSSL_VERSION; fi
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
+- export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
+- if [ "$TRAVIS_EVENT_TYPE" != "pull_request" ]; then export e2e=true; fi
+- if [ "$e2e" = true ]; then export gam_user=gam-travis-$jid@pdl.jaylee.us; fi
+- if [ "$e2e" = true ]; then openssl aes-256-cbc -K $encrypted_6294a53f809d_key -iv $encrypted_6294a53f809d_iv -in travis/creds.tar.enc -out travis/creds.tar -d; fi
+- if [ "$e2e" = true ]; then tar xvf travis/creds.tar -C $gampath; fi
+- if [ "$e2e" = true ]; then export OAUTHFILE=oauth2.txt-gam-travis-$jid; fi
+- if [ "$e2e" = true ]; then $gam oauth info; fi
+- if [ "$e2e" = true ]; then $gam info domain; fi
+- if [ "$e2e" = true ]; then $gam oauth refresh; fi
+- if [ "$e2e" = true ]; then $gam info user; fi
+- if [ "$e2e" = true ]; then export tstamp=$(date +%s%3N);
+  export newbase=travis-test-$jid-$tstamp;
+  export newuser=$newbase@pdl.jaylee.us;
+  export newgroup=$newbase-group@pdl.jaylee.us;
+  export newalias=$newbase-alias@pdl.jaylee.us;
+  export newbuilding=$newbase-building;
+  export newresource=$newbase-resource;
+  export GAM_THREADS=5; fi
+- if [ "$e2e" = true ]; then echo email > sample.csv;
+  for i in {01..20};
+    do echo $newbase-bulkuser-$i >> sample.csv;
+  done; fi
+- if [ "$e2e" = true ]; then $gam create user $newuser firstname Travis lastname $jid password random recoveryphone 12125121110 recoveryemail jay0lee@gmail.com travis.jid $jid; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "Travis test message"; fi
+- if [ "$e2e" = true ]; then $gam create group $newgroup name "Travis $jid group" description "This is a description" isarchived true; fi
+- if [ "$e2e" = true ]; then $gam user $newuser add license gsuitebusiness; fi
+- if [ "$e2e" = true ]; then $gam update group $newgroup add owner $gam_user; fi
+- if [ "$e2e" = true ]; then $gam update group $newgroup add member $newuser; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam create user ~~email~~ firstname "Travis Bulk" lastname ~~email~~ travis.jid $jid; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add license gsuitebusiness; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user $gam_user sendemail recipient ~~email~~@pdl.jaylee.us subject "test message $newbase" message "Travis test message"; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update group $newgroup add member ~email; fi
+- if [ "$e2e" = true ]; then $gam info group $newgroup; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user check serviceaccount; fi
+- if [ "$e2e" = true ]; then $gam user $newuser imap on; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show imap; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user $newuser delegate to ~email; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show delegates; fi
+- if [ "$e2d" = true ]; then export biohazard=$(echo -e '\xe2\x98\xa3'); fi
+- if [ "$e2e" = true ]; then $gam user $newuser label "$biohazard unicode biohazard $biohazard"; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels > labels.txt; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user importemail subject "Travis import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user insertemail subject "Travis insert $newbase" file gam.py labels INBOX,UNREAD; fi # yep body is gam code
+- if [ "$e2e" = true ]; then $gam user $gam_user sendemail subject "Travis send $gam_user $newbase" file gam.py recipient admin@pdl.jaylee.us; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user draftemail subject "Travis draft $newbase" message "Draft message test"; fi
+- if [ "$e2e" = true ]; then $gam users "$gam_user $newbase-bulkuser-01 $newbase-bulkuser-02 $newbase-bulkuser-03" delete messages query in:anywhere maxtodelete 99999 doit; fi
+- if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-04 $newbase-bulkuser-05 $newbase-bulkuser-06" trash messages query in:anywhere maxtotrash 99999 doit; fi
+- if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-07 $newbase-bulkuser-08 $newbase-bulkuser-09" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit; fi
+- if [ "$e2e" = true ]; then $gam user $newuser delete label --ALL_LABELS--; fi
+- if [ "$e2e" = true ]; then $gam create feature name Whiteboard-$newbase; fi
+- if [ "$e2e" = true ]; then $gam create feature name VC-$newbase; fi
+- if [ "$e2e" = true ]; then $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."; fi
+- if [ "$e2e" = true ]; then $gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room; fi
+- if [ "$e2e" = true ]; then $gam info resource $newresource; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show filelist; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi # clear ACLs
+- if [ "$e2e" = true ]; then $gam calendar $gam_user update read domain; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user update freebusy default; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user add editor $newuser; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user showacl; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user addevent summary "Travis test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
+- if [ "$e2e" = true ]; then matterid=uid:$($gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser | head -1 | cut -d ' ' -f 3); fi
+- if [ "$e2e" = true ]; then $gam create vaulthold matter $matterid name "Travis hold $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print vaultmatters matterstate open; fi
+- if [ "$e2e" = true ]; then $gam print vaultholds matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam create vaultexport matter $matterid name "Travis export $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add calendar id:$newresource; fi
+- if [ "$e2e" = true ]; then $gam delete resource $newresource; fi
+- if [ "$e2e" = true ]; then $gam delete feature Whiteboard-$newbase; fi
+- if [ "$e2e" = true ]; then $gam delete feature VC-$newbase; fi
+- if [ "$e2e" = true ]; then $gam delete building $newbuilding; fi
+- if [ "$e2e" = true ]; then $gam delete group $newgroup; fi
+- if [ "$e2e" = true ]; then $gam create alias $newalias user $newuser; fi
+- if [ "$e2e" = true ]; then $gam whatis $newuser; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action close; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action delete; fi
+- if [ "$e2e" = true ]; then $gam delete user $newuser; fi
+- if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
+- if [ "$e2e" = true ]; then $gam print mobile; fi
+- if [ "$e2e" = true ]; then $gam print devices; fi
+- if [ "$e2e" = true ]; then export sn="$jid$jid$jid$jid$jid-$(openssl rand -base64 32 | sed 's/[^a-zA-Z0-9]//g')"; fi
+- if [ "$e2e" = true ]; then $gam create device serialnumber $sn devicetype android; fi
+- if [ "$e2e" = true ]; then $gam print devices filter "serial:$jid$jid$jid$jid$jid-" | $gam csv - gam delete device id ~name; fi
+- if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$e2e" = true ]; then $gam report usageparameters customer; fi
+- if [ "$e2e" = true ]; then $gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins; fi
+- if [ "$e2e" = true ]; then $gam report customer todrive; fi
+- if [ "$e2e" = true ]; then $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
+- if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
+- if ([ "$e2e" = true ] && [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]); then
+    for gamfile in gam-$GAMVERSION-*; do
+      fileid=$($gam user $gam_user add drivefile localfile $gamfile drivefilename $GAMVERSION-${TRAVIS_COMMIT:0:7}-$gamfile parentid 1N2zbO33qzUQFsGM49-m9AQC1ijzd_ru1 returnidonly);
+      $gam user $gam_user add drivefileacl $fileid anyone role reader withlink;
+    done;
+  fi
+
+before_deploy:
+- export TRAVIS_TAG="preview"
+- unset LD_LIBRARY_PATH
+
+deploy:
+  provider: releases
+  token:
+    secure: bzambMcQwyv/o5c5GrKGCsZHgE5R85tg8sNFvPfpISz3+uosCjnBXas7wvCKzT75XUFi2ztfbYak6HdKf4sGnNHk0saEicB3slH+ghPyZbYzp76yvvduhFO2nWW3/F01tL+Yfqqt4/q8wFaWGjrC5km+6GLVyB4lWA/Uyu49qKnz02uSwyhBD/VFbO7DOQ65a1iWk9HngyMsu0Oi7HIbSjSLtxTHedNfOf3waW0NivTTxYXiYGX/MCu3GWhgIGj47a+H3A6FcQ/9QWvnKgnoixdgPBUz7kDb7ktsWwQsILPGStgH7iMuG49ZlXdEFmqwifBri2wvzmFEevBGZjHcupy1IGrNFRG+IUGKMotio+OkLHlLjuv7ZJtqCz/Vf5SNFgNyMSanx6jKEUJuYvndVg99IRXmYVwHFwPu5BAcJACpU6C0AfyGmmSqqwxCd46uXL62ynxNFpHuRfOqlDnmCTfZgjOciJSlDDpf+Xz9fF7+oCoeCi3mrcZVFjhd3tT6Oxw5HrsDtm0ZNld1cdLidaq8H6vOFgHMd0A9yNYZzTzXTvpmxzkXT4Zc7s+PYKN6z5fRZ+pJeckUjRXblvVEfs5HFSymavcOc5AkRwxpvOsTQMNmlnaJCBo5UNs0K/rVmRi5cFmaiwTcBCY0kTllOBJ4zWsfq8seiokWwNUNK2g=
+  file_glob: true
+  overwrite: true
+  file: gam-$GAMVERSION-*
+  skip_cleanup: true
+  draft: true
+  on:
+    repo: jay0lee/GAM
+    condition: $TRAVIS_JOB_NAME != *"Testing"

README.md

@@ -1,4 +1,4 @@
-GAM is a command line tool for Google G Suite Administrators to manage domain and user settings quickly and easily.
+GAM is a command line tool for Google G Suite Administrators to manage domain and user settings quickly and easily. [![Build Status](https://travis-ci.org/jay0lee/GAM.svg?branch=master)](https://travis-ci.org/jay0lee/GAM)
 # Quick Start
 ## Linux / MacOS
 Open a terminal and run:
@@ -12,6 +12,8 @@ Download the MSI Installer from the [GitHub Releases] page. Install the MSI and
 The GAM documentation is hosted in the [GitHub Wiki]
 # Mailing List / Discussion group
 The GAM mailing list / discussion group is hosted on [Google Groups].  You can join the list and interact via email, or just post from the web itself.
+# IM Room
+[![Join the chat at https://gitter.im/jay0lee-GAM/community](https://badges.gitter.im/jay0lee-GAM/community.svg)](https://gitter.im/jay0lee-GAM/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 # Author
 GAM is maintained by <a href="mailto:jay0lee@gmail.com">Jay Lee</a>. Please direct "how do I?" questions to [Google Groups].
 

src/.gitignore

@@ -64,7 +64,7 @@ nobrowser.txt
 nocache.txt
 noverifyssl.txt
 gamcache/
-gam/
+dist/
 gam-64/
 *.zip
 *.msi

src/GamCommands.txt

@@ -38,6 +38,7 @@ If an item contains spaces, it should be surrounded by ".
 	papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|
 	saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|
 	tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen
+<DayOfWeek> ::= mon|tue|wed|thu|fri|sat|sun
 <FileFormat> ::=
 	csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
 <LabelColorHex> ::=
@@ -74,6 +75,13 @@ If an item contains spaces, it should be surrounded by ".
 	Google-Drive-storage|
 	Google-Vault|
 	101001|101005|101031
+<ProductID> ::=
+	Google-Apps|
+	Google-Chrome-Device-Management|
+	Google-Coordinate|
+	Google-Drive-storage|
+	Google-Vault|
+	101001|101005|101006|101031|101033|101034
 <SKUID> ::=
 	cloudidentity|identity|1010010001|
 	cloudidentitypremium|identitypremium|1010050001|
@@ -81,12 +89,16 @@ If an item contains spaces, it should be surrounded by ".
 	gafb|gafw|basic|gsuitebasic|Google-Apps-For-Business|
 	gafg|gsuitegovernment|gsuitegov|Google-Apps-For-Government|
 	gams|postini|gsuitegams|gsuitepostini|gsuitemessagesecurity|Google-Apps-For-Postini|
-	gal|lite|gsuitelite|Google-Apps-Lite|
-	gau|unlimited|gsuitebusiness|Google-Apps-Unlimited|
-	gae|enterprise|gsuiteenterprise|1010020020|
+	gal|gsl|lite|gsuitelite|Google-Apps-Lite|
+	gau|gsb|unlimited|gsuitebusiness|Google-Apps-Unlimited|
+	gae|gse|enterprise|gsuiteenterprise|1010020020|
 	gsefe|e4e|gsuiteenterpriseeducation|1010310002|
+	gsefes|e4es|gsuiteenterpriseeducationstudent|1010310003|
+	gsbau|businessarchived|gsuitebusinessarchived|
+	gseau|enterprisearchived|gsuiteenterprisearchived|
 	chrome|cdm|googlechromedevicemanagement|Google-Chrome-Device-Management|
 	coordinate|googlecoordinate|Google-Coordinate|
+	d4e|driveenterprise|drive4enterprise|
 	drive20gb|20gb|googledrivestorage20gb|Google-Drive-storage-20GB|
 	drive50gb|50gb|googledrivestorage50gb|Google-Drive-storage-50GB|
 	drive200gb|200gb|googledrivestorage200gb|Google-Drive-storage-200GB|
@@ -116,14 +128,12 @@ If an item contains spaces, it should be surrounded by ".
 <MilliSeconds> ::= <Digit><Digit><Digit>
 <Date> ::=
 	<Year>-<Month>-<Day> |
-	(+|-)<Number>(d|w)
-<DateTime> ::=
-	<Year>-<Month>-<Day>(<Space>|T)<Hour>:<Minute> |
-	(+|-)<Number>(m|h|d|w)
+	(+|-)<Number>(d|w|y)
 <Time> ::=
 	<Year>-<Month>-<Day>(<Space>|T)<Hour>:<Minute>:<Second>[.<MilliSeconds>](Z|(+|-(<Hour>:<Minute>))) |
 	(+|-)<Number>(m|h|d|w)
 <RegularExpression> ::= <Python Regular Expression, see: https://docs.python.org/2/library/re.html>
+<ProjectID> ::= <String> # Must match this Python Regular Expression: [a-z][a-z0-9-]{4,28}[a-z0-9]
 <Tag> ::= <String>
 <UniqueID> ::= uid:<String>
 
@@ -131,9 +141,11 @@ If an item contains spaces, it should be surrounded by ".
 
 <AccessToken> ::= <String>
 <ACLScope> ::= [user:]<EmailAddress>|group:<EmailAddress>|domain[:<DomainName>]|default
+<APIScopeURL> ::= <String>
 <ASPID> ::= <String>
 <BuildingID> ::= <String>|id:<String>
 <CalendarACLRole> ::= editor|freebusy|freebusyreader|owner|reader|writer
+<CalendarACLRuleID> ::= user:<EmailAddress>|group:<EmailAddress>|domain:<DomainName>|default
 <CalendarColorIndex> ::= <Number in range 1-24>
 <CalendarItem> ::= <EmailAddress>|<String>
 <ChatRoom> ::= <String>
@@ -211,7 +223,9 @@ If an item contains spaces, it should be surrounded by ".
 <SchemaName> ::= <String>
 <Section> ::= <String>
 <SerialNumber> ::= <String>
-<S/MIMEID> ::= <String>	
+<ServiceAccountKey> ::= <String>
+<S/MIMEID> ::= <String>
+<SMTPHostName> ::= <String>
 <StudentItem> ::= <EmailAddress>|<UniqueID>|<String>
 <TeamDriveID> ::= <String>
 <Timezone> ::= <String>
@@ -219,6 +233,7 @@ If an item contains spaces, it should be surrounded by ".
 <URI> ::= <String>
 <URL> ::= <String>
 <UserItem> ::= <EmailAddress>|<UniqueID>|<String>
+<UserName> ::= <<String>
 
 <CourseFieldName> ::=
 	alternatelink|
@@ -245,16 +260,20 @@ If an item contains spaces, it should be surrounded by ".
 	annotatedassetid|assedid|asset|
 	annotatedlocation|location|
 	annotateduser|user|
+	autoupdateexpiration|
 	bootmode|
 	cpustatusreports|
 	devicefiles|
 	deviceid|
 	diskvolumereports|
+	dockmacaddress|
 	ethernetmacaddress|
+	ethernetmacaddress0|
 	firmwareversion|
 	lastenrollmenttime|
 	lastsync|
 	macaddress|
+	manufacturedate|
 	meid|
 	model|
 	notes|
@@ -287,6 +306,7 @@ If an item contains spaces, it should be surrounded by ".
 	cancomment|
 	canreadrevisions|
 	copyable|
+	copyrequireswriterpermission|
 	createddate|createdtime|
 	description|
 	editable|
@@ -350,46 +370,55 @@ If an item contains spaces, it should be surrounded by ".
 	admincreated|
 	aliases|
 	allowexternalmembers|
-	allowgooglecommunication|
 	allowwebposting|
 	archiveonly|
-	collaborative|
+	customfootertext|
 	customreplyto|
+	customrolesenabledforsettingstobemerged|
 	defaultmessagedenynotificationtext|
 	description|
 	directmemberscount|
 	email|
-	favoriterepliesontop|
+	enablecollaborativeinbox|collaborative|
 	id|
+	includecustomfooter|
 	includeinglobaladdresslist|gal|
 	isarchived|
-	maxmessagebytes|
 	memberscanpostasthegroup|
-	messagedisplayfont|
 	messagemoderationlevel|
-	name
+	name|
 	primarylanguage|
 	replyto|
 	sendmessagedenynotification|
 	showingroupdirectory|
 	spammoderationlevel|
 	whocanadd|
-	whocanaddreferences|
+	whocanapprovemessages|
 	whocanassigntopics|
+	whocanassistcontent|
 	whocancontactowner|
+	whocandeleteanypost|
+	whocandeletetopics|
+	whocandiscovergroup|
 	whocanenterfreeformtags|
+	whocanhideabuse|
 	whocaninvite|
 	whocanjoin|
 	whocanleavegroup|
+	whocanlocktopics|
+	whocanmaketopicssticky|
 	whocanmarkduplicate|
 	whocanmarkfavoritereplyonanytopic|
-	whocanmarkfavoritereplyonowntopic|
 	whocanmarknoresponseneeded|
+	whocanmoderatecontent|
 	whocanmodifytagsandcategories|
+	whocanmovetopicsin|
+	whocanmovetopicsout|
+	whocanpostannouncements|
 	whocanpostmessage|
 	whocantaketopics|
 	whocanunassigntopic|
-	whocanunmarkfavoritereplyonanytopic
+	whocanunmarkfavoritereplyonanytopic|
 	whocanviewgroup|
 	whocanviewmembership
 
@@ -500,6 +529,8 @@ If an item contains spaces, it should be surrounded by ".
 	phones|phone|
 	posixaccounts|posix|
 	primaryemail|username|
+	recoveryemail|
+	recoveryphone|
 	relations|relation|
 	ssh|sshkeys|sshpublickeys|
 	suspended|
@@ -524,6 +555,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
     "'it em' 'it,em' \"it'em\""
 
 <ACLList> ::= "<ACLScope>(,<ACLScope>)*"
+<APIScopeURLList> ::= "<APIScopeURL>(,<APIScopeURL>)*"
 <ASPIDList> ::= "<ASPID>(,<ASPID>)*"
 <CalendarList> ::= "<CalendarItem>(,<CalendarItem>)*"
 <ChatRoomList> ::= "<ChatRoom>(,<ChatRoom>)*"
@@ -562,6 +594,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
 <SKUIDList> ="<SKUID>(,<SKUID>)*"
 <SchemaNameList> ::= "<SchemaName>(,<SchemaName>)*"
 <SerialNumberList> ::= "<SerialNumber>(,<SerialNumber>)*"
+<ServiceAccountKeyList> ::= "<ServiceAccountKey>(,<ServiceAccountKey>)*"
 <TeamDriveIDList> ::= "<TeamDriveID>(,<TeamDriveID>)*"
 <UserFieldNameList> ::= "<UserFieldName>(,<UserFieldName>)*"
 <UserList> ::= "<UserItem>(,<UserItem>)*"
@@ -643,67 +676,82 @@ Specify a collection of Users by directly specifying them or by specifiying item
 
 <DriveFileAddAttributes> ::=
 	(localfile <FileName>)|
-	(convert)|(ocr)|(ocrlanguage <Language>)|(restricted|restrict)|(starred|star)|(trashed|trash)|(viewed|view)|
+	(convert)|(ocr)|(ocrlanguage <Language>)|
+	(restricted|restrict)|(starred|star)|(trashed|trash)|(viewed|view)|
+	copyrequireswriterpermission|
 	(lastviewedbyme <Time>)|(modifieddate|modifiedtime <Time>)|(description <String>)|(mimetype <MimeType>)|
 	(parentid <DriveFolderID>)|(parentname <DriveFolderName>)|(anyownerparentname <DriveFolderName>)|writerscantshare
 <DriveFileUpdateAttributes> ::=
 	(localfile <FileName>)|
-	(convert)|(ocr)|(ocrlanguage <Language>)|(restricted|restrict <Boolean>)|(starred|star <Boolean>)|(trashed|trash <Boolean>)|(viewed|view <Boolean>)|
+	(convert)|(ocr)|(ocrlanguage <Language>)|
+	(restricted|restrict <Boolean>)|(starred|star <Boolean>)|(trashed|trash <Boolean>)|(viewed|view <Boolean>)|
+	(copyrequireswriterpermission <Boolean>)|
 	(lastviewedbyme <Time>)|(modifieddate <Time>)|(description <String>)|(mimetype <MimeType>)|
 	(parentid <DriveFolderID>)|(parentname <DriveFolderName>)|(anyownerparentname <DriveFolderName>)|writerscantshare
-<EventAttributes> ::=
-	(anyonecanaddself)|(guestscantinviteothers)|(guestscantseeothers)|(notifyattendees)|(available)|(visibility default|public|prvate)|(tentative)|
-	(attendee <EmailAddress>)|(optionalattendee <EmailAddress>)|
-	(description <String>)|(summary <String>)|(location <String>)|(id <String>)|
-	(source <String> <URL>)|(privateproperty <PropertyKey> <PropertyValue>)|(sharedproperty <PropertyKey> <PropertyValue>)|
-	(recurrence <RRULE, EXRULE, RDATE and EXDATE line>)|
-	(start allday <Date>)|(start <Time>)|(end allday <Date>)|(end <Time>)|(timezone <Timezone>)|
-	(noreminders|(reminder <Number> email|popup|sms))|
-	(colorindex|colorid <EventColorIndex>)
-
-<GroupAttributes> ::=
+<GroupSettingsAttribute> ::=
 	(allowexternalmembers <Boolean>)|
-	(allowgooglecommunication <Boolean>)|
 	(allowwebposting <Boolean>)|
 	(archiveonly <Boolean>)|
-	(collaborative (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
 	(customfootertext <String>)|
 	(customreplyto <EmailAddress>)|
 	(defaultmessagedenynotificationtext <String>)|
 	(description <String>)|
-	(favoriterepliesontop <Boolean>)|
-	(gal|includeInGlobalAddressList <Boolean>)|
+	(enablecollaborativeinbox|collaborative <Boolean>)|
+	(includeinglobaladdresslist|gal <Boolean>)|
 	(includecustomfooter <Boolean>)|
 	(isarchived <Boolean>)|
-	(maxmessagebytes <ByteCount>)|
 	(memberscanpostasthegroup <Boolean>)|
-	(messagedisplayfont DEFAULT_FONT|FIXED_WIDTH_FONT)|
-	(messagemoderationlevel MODERATE_ALL_MESSAGES|MODERATE_NON_MEMBERS|MODERATE_NEW_MEMBERS|MODERATE_NONE)|
+	(messagemoderationlevel moderate_all_messages|moderate_non_members|moderate_new_members|moderate_none)|
 	(name <String>)|
 	(primarylanguage <Language>)|
-	(replyto REPLY_TO_CUSTOM|REPLY_TO_SENDER|REPLY_TO_LIST|REPLY_TO_OWNER|REPLY_TO_IGNORE|REPLY_TO_MANAGERS)|
+	(replyto reply_to_custom|reply_to_sender|reply_to_list|reply_to_owner|reply_to_ignore|reply_to_managers)|
 	(sendmessagedenynotification <Boolean>)|
-	(showingroupdirectory <Boolean>)|
-	(spammoderationlevel ALLOW|MODERATE|SILENTLY_MODERATE|REJECT)|
-	(whocanadd ALL_MEMBERS_CAN_ADD|ALL_MANAGERS_CAN_ADD|NONE_CAN_ADD)|
-	(whocanaddreferences (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanassigntopics (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocancontactowner ANYONE_CAN_CONTACT|ALL_IN_DOMAIN_CAN_CONTACT|ALL_MEMBERS_CAN_CONTACT|ALL_MANAGERS_CAN_CONTACT)|
-	(whocanenterfreeformtags (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocaninvite ALL_MEMBERS_CAN_INVITE|ALL_MANAGERS_CAN_INVITE|NONE_CAN_INVITE)|
-	(whocanjoin ANYONE_CAN_JOIN|ALL_IN_DOMAIN_CAN_JOIN|INVITED_CAN_JOIN|CAN_REQUEST_TO_JOIN)|
-	(whocanleavegroup ALL_MANAGERS_CAN_LEAVE|ALL_MEMBERS_CAN_LEAVE|NONE_CAN_LEAVE)|
-	(whocanmarkduplicate (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanmarkfavoritereplyonanytopic (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanmarkfavoritereplyonowntopic (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanmarknoresponseneeded (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanmodifytagsandcategories (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanpostmessage NONE_CAN_POST|ALL_MANAGERS_CAN_POST|ALL_MEMBERS_CAN_POST|ALL_IN_DOMAIN_CAN_POST|ANYONE_CAN_POST)|
-	(whocantaketopics (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanunassigntopic (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanunmarkfavoritereplyonanytopic (members|all_members)|(managers|owners_and_managers)|(managers_only)|(owners|owners_only)|none)|
-	(whocanviewgroup ANYONE_CAN_VIEW|ALL_IN_DOMAIN_CAN_VIEW|ALL_MEMBERS_CAN_VIEW|ALL_MANAGERS_CAN_VIEW)|
-	(whocanviewmembership ALL_IN_DOMAIN_CAN_VIEW|ALL_MEMBERS_CAN_VIEW|ALL_MANAGERS_CAN_VIEW)
+	(spammoderationlevel allow|moderate|silently_moderate|reject)|
+	(whocanadd all_members_can_add|all_managers_can_add|all_owners_can_add|none_can_add)|
+	(whocancontactowner anyone_can_contact|all_in_domain_can_contact|all_members_can_contact|all_managers_can_contact)|
+	(whocanjoin anyone_can_join|all_in_domain_can_join|invited_can_join|can_request_to_join)|
+	(whocanleavegroup all_members_can_leave|all_managers_can_leave|all_owners_can_leave|none_can_leave)|
+	(whocanpostmessage none_can_post|all_managers_can_post|all_members_can_post|all_owners_can_post|all_in_domain_can_post|anyone_can_post)|
+	(whocanviewgroup anyone_can_view|all_in_domain_can_view|all_members_can_view|all_managers_can_view|all_owners_can_view)|
+	(whocanviewmembership all_in_domain_can_view|all_members_can_view|all_managers_can_view|all_owners_can_view)
+<GroupWhoCanDiscoverGroupAttribute> ::=
+	(whocandiscovergroup allmemberscandiscover|allindomaincandiscover|anyonecandiscover)|
+	(showingroupdirectory <Boolean>)
+<GroupWhoCanAssistContentAttribute> ::=
+	(whocanassistcontent all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanassigntopics all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanenterfreeformtags all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanhideabuse all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanmaketopicssticky all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanmarkduplicate all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanmarkfavoritereplyonanytopic all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanmarknoresponseneeded all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanmodifytagsandcategories all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocantaketopics all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanunassigntopic all_members|owners_and_managers|managers_only|owners_only|none)|
+	(whocanunmarkfavoritereplyonanytopic all_members|owners_and_managers|managers_only|owners_only|none)
+<GroupWhoCanModerateContentAttribute> ::=
+	(whocanmoderatecontent all_members|owners_and_managers|owners_only|none)|
+	(whocanapprovemessages all_members|owners_and_managers|owners_only|none)|
+	(whocandeleteanypost all_members|owners_and_managers|owners_only|none)|
+	(whocandeletetopics all_members|owners_and_managers|owners_only|none)|
+	(whocanlocktopics all_members|owners_and_managers|owners_only|none)|
+	(whocanmovetopicsin all_members|owners_and_managers|owners_only|none)|
+	(whocanmovetopicsout all_members|owners_and_managers|owners_only|none)|
+	(whocanpostannouncements all_members|owners_and_managers|owners_only|none)
+<GroupWhoCanModerateMembersAttribute> ::=
+	(whocanmoderatemembers all_members|owners_and_managers|owners_only|none)|
+	(whocanadd all_members_can_add|all_managers_can_add|none_can_add)|
+	(whocanapprovemembers all_members_can_approve|all_managers_can_approve|all_owners_can_approve|none_can_approve)|
+	(whocanbanusers all_members|owners_and_managers|owners_only|none)|
+	(whocaninvite all_members_can_invite|all_managers_can_invite|all_owners_can_invite|none_can_invite)|
+	(whocanmodifymembers all_members|owners_and_managers|owners_only|none)
+<GroupAttribute> ::=
+	<GroupSettingsAttribute>|
+	<GroupWhoCanDiscoverGroupAttribute>|
+	<GroupWhoCanAssistContentAttribute>|
+	<GroupWhoCanModerateContentAttribute>|
+	<GroupWhoCanModerateMembersAttribute>
 
 <MobileAction> ::=
 	admin_remote_wipe|wipe|admin_account_wipe|accountwipe|wipeaccount|approve|block|cancel_remote_wipe_then_activate|cancel_remote_wipe_then_block
@@ -725,48 +773,56 @@ Specify a collection of Users by directly specifying them or by specifiying item
 	(name <String>)|
 	(type <String>)|
 	(uservisibledescription <String>)
-	
+
 <SchemaFieldDefinition> ::=
 	field <FieldName> (type bool|date|double|email|int64|phone|string) [multivalued|multivalue] [indexed] [restricted] [range <Number> <Number>] endfield
 
-<UserAttributes> ::=
-	(address clear|(type work|home|other|(custom <String>) [unstructured|formatted <String>] [pobox <String>] [extendedaddress <String>] [streetaddress <String>]
-		[locality <String>] [region <String>] [postalcode <String>] [country <String>] [countrycode <String>] notprimary|primary))|
+<UserBasicAttribute> ::=
 	(agreed2terms|agreedtoterms <Boolean>)|
 	(changepassword|changepasswordatnextlogin <Boolean>)|
-	(crypt|sha|sha1|sha-1|md5|nohash)|
+	(base64-md5|base64-sha1|crypt|sha|sha1|sha-1|md5|nohash)|
 	(customerid <String>)|
 	(email|primaryemail|username <EmailAddress>)|
-	(otheremail clear|(work|home|other|<String> <String>))|
-	(externalid clear|(account|customer|login_id|network|organization|<String> <String>))|
 	(firstname|givenname <String>)|
 	(gal|includeinglobaladdresslist <Boolean>)|
 	(gender clear|(female|male|unknown|(other <String>) [addressmeas <String>]))|
-	(im clear|(type work|home|other|(custom <String>) protocol aim|gtalk|icq|jabber|msn|net_meeting|qq|skype|yahoo|(custom_protocol <String>) <String> [notprimary|primary]))|
 	(ipwhitelisted <Boolean>)|
-	(keyword clear|(occupation|outlook|(custom <string>) <String>))|
 	(language clear|<LanguageList>)|
 	(lastname|familyname <String>)|
-	(location clear|(type default|desk|<String> area <String> [building|buildingid <BuildingID>] [floor|floorname <FloorName>] [section|floorsection <String>] [desk|deskcode <String>] endlocation))|
-	(note clear|([text_plain|text_html] <String>|(file <FileName> [charset <Charset>])))|
-	(organization clear|([type domain_only|school|unknown|work] [customtype <String>] [name <String>] [title <String>] [department <String>] [symbol <String>]
-		[costcenter <String>]  [location <String>] [description <String>] [domain <String>] notprimary|primary))|
+	(note clear|([text_html|text_plain] <String>|(file <FileName> [charset <Charset>])))|
 	(org|ou|orgunitpath <OrgUnitPath>)
 	(password random|<Password>)|
-	(phone clear|([type work|home|other|work_fax|home_fax|other_fax|main|company_main|assistant|mobile|work_mobile|pager|work_pager|car|radio|callback|isdn|telex|tty_tdd|grand_central|(custom <String>)]
-		[value <String>] notprimary|primary))|
-	(posix clear|(username <String> uid <Integer> gid <Integer> [system|systemid <String>] [home|homedirectory <String>] [shell <String>] [gecos <String>] [primary <Boolean>] endposix))|
-	(relation clear|(spouse|child|mother|father|parent|brother|sister|friend|relative|domestic_partner|manager|dotted-line_manager|assistant|admin_assistant|exec_assistant|referred_by|partner|<String> <String>))|
-	(sshkeys clear|(key <String> [expires <Integer>] endssh))|
+	(recoveryemail <EmailAddress>)|
+	(recoveryphone <string>)|
 	(suspended <Boolean>)|
-	(website clear|(home_page|blog|profile|work|home|other|ftp|reservations|app_install_page|<String> <URL> [notprimary|primary]))|
-	(<SchemaName>.<FieldName> [multivalued|multivalue|value|multinonempty [type work|home|other|(custom <String>)]] <String>)
+	(<SchemaName>.<FieldName> [multivalued|multivalue|value|multinonempty [type home|other|work|(custom <String>)]] <String>)
+<UserMultiAttributes> ::=
+	(address clear|(type home|other|work|(custom <String>) [unstructured|formatted <String>] [pobox <String>] [extendedaddress <String>] [streetaddress <String>]
+		[locality <String>] [region <String>] [postalcode <String>] [country <String>] [countrycode <String>] notprimary|primary))|
+	(otheremail clear|(home|other|work|<String> <String>))|
+	(externalid clear|(account|customer|login_id|network|organization|<String> <String>))|
+	(im clear|(type home|other|work|(custom <String>) protocol aim|gtalk|icq|jabber|msn|net_meeting|qq|skype|yahoo|(custom_protocol <String>) <String> [notprimary|primary]))|
+	(keyword clear|(mission|occupation|outlook|(custom <string>) <String>))|
+	(location clear|(type default|desk|<String> area <String> [building|buildingid <String>] [floor|floorname <String>] [section|floorsection <String>] [desk|deskcode <String>] endlocation))|
+	(organization clear|([type domain_only|school|unknown|work] [customtype <String>] [name <String>] [title <String>] [department <String>] [symbol <String>]
+		[costcenter <String>]  [location <String>] [description <String>] [domain <String>] [fulltimeequivalent <Integer>] notprimary|primary))|
+	(phone clear|([type assistant|callback|car|company_main|grand_central|home|home_fax|isdn|main|mobile|other|other_fax|pager|radio|telex|tty_tdd|work|work_fax|work_mobile|work_pager|(custom <String>)]
+		[value <String>] notprimary|primary))|
+	(posix clear|(username <String> uid <Integer> gid <Integer> [system|systemid <String>] [home|homedirectory <String>] [shell <String>]
+		[gecos <String>] [os|operatingSystemType linux|unspecified|windows] [primary <Boolean>] endposix))|
+	(relation clear|(admin_assistant|assistant|brother|child|domestic_partner|dotted-line_manager|exec_assistant|father|friend|manager|mother|parent|partner|referred_by|relative|sister|spouse|<String> <String>))|
+	(sshkeys clear|(key <String> [expires <Integer>] endssh))|
+	(website clear|(app_install_page|blog|ftp|home|home_page|other|profile|reservations|resume|work|<String> <URL> [notprimary|primary]))
+<UserAttribute> ::=
+	<UserBasicAttribute>|
+	<UserMultiAttribute>
 
-gam version [check] [simple]
+gam version [check|checkrc|simple|extended] [timeoffset] [location <HostName>]
 gam help
 
 gam batch <FileName>|- [charset <Charset>]
 gam csv <FileName>|- [charset <Charset>] gam <GAM argument list>
+gam csvtest <FileName>|- [charset <Charset>] gam <GAM argument list>
 
 You can make substitutions in <GAMArgumentList> with values from the CSV file.
 An argument containing exactly ~xxx is replaced by the value of field xxx from the CSV file
@@ -775,14 +831,27 @@ An argument containing instances of ~~xxx~~ has xxx replaced by the value of fie
 Example: gam csv Users.csv gam update user "~primaryEmail" address type work unstructured "~~Street~~, ~~City~~, ~~State~~ ~~ZIP~~"
 Each user (~primaryEmail, e.g. foo@bar.com) would have their work address updated
 
-gam create project [<EmailAddress>]
-gam update project [<EmailAddress>]
+gam create project [<EmailAddress>] [<ProjectID>]
+gam create project [admin <EmailAddress>] [project <ProjectID>] [parent <String>]
+gam use project [<EmailAddress>] [<ProjectID>]
+gam use project [admin <EmailAddress>] [project <ProjectID>]
+gam update project [<EmailAddress>] [gam|<ProjectID>|(filter <String>)]
+gam delete project [<EmailAddress>] [gam|<ProjectID>|(filter <String>)]
+gam show projects [<EmailAddress>] [all|gam|<ProjectID>|(filter <String>)]
+gam print projects [<EmailAddress>] [all|gam|<ProjectID>|(filter <String>)] [todrive]
+
+gam rotate sakey|sakeys [retain_none|retain_existing|replace_current]
+         [(algorithm KEY_ALG_RSA_1024|KEY_ALG_RSA_2048)|(localkeysize 1024|2048|4096)]
+gam delete sakey|sakeys <ServiceAccountKeyList>+ [doit]
+gam show sakey|sakeys [all|system|user]
 
 gam oauth|oauth2 create|request [<EmailAddress>]
+gam oauth|oauth2 create|request [admin <EmailAddress>] [scope|scopes <APIScopeURLList>]
 gam oauth|oauth2 delete|revoke
-gam oauth|oauth2 info|verify [<AccessToken>]
+gam oauth|oauth2 info|verify [accesstoken <AccessToken>] [idtoken <IDToken>] [showsecret]
+gam oauth|oauth2 refresh
 
-gam <UserTypeEntity> check serviceaccount
+gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
 gam whatis <EmailItem>
 
@@ -814,6 +883,25 @@ gam update resoldsubscription <CustomerID> <SKUID>
 gam delete resoldsubscription <CustomerID> <SKUID> cancel|downgrade|transfer_to_direct
 gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 
+<ActivityApplicationName> ::=
+	access|accesstransparency|
+	admin|
+	calendar|calendars|
+	chat|
+	drive|doc|docs|
+	enterprisegroups|groupsenterprise|
+	gcp|
+	google+|gplus|
+	group|groups|
+	hangoutsmeet|meet|
+	jamboard|
+	login|logins|
+	mobile|
+	oauthtoken|token|tokens|
+	rules|
+	saml|
+	useraccounts
+
 <ReportsApp> ::=
 	accounts|
 	app_maker|
@@ -824,22 +912,40 @@ gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 	device_management|
 	drive|
 	gmail|
-	gplus|
 	meet|
 	mobile|
 	sites
 <ReportsAppList> ::= "<ReportsApp>(,<ReportsApp>)*"
 
-gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
-	[(user all|<UserItem>)] [filter|filters <String>] [fields|parameters <String>]
-gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
+gam report usageparameters customer|user [todrive]
+gam report usage user [todrive]
+	[<UserTypeItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
 	[fields|parameters <String>]
-gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token [todrive]
-	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
+gam report usage customer [todrive]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+
+gam report users|user [todrive]
+	[(user all|<UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[filter|filters <String>] [fields|parameters <String>]
+gam report customers|customer|domain [todrive]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[fields|parameters <String>]
+gam report <ActivityApplicationName> [todrive]
+	[(user all|<UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[start <Time>] [end <Time>]
+	[filter|filters <String>] [event <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
 gam delete admin <RoleAssignmentId>
 gam print admins [todrive] [user <UserItem>] [role <RoleItem>]
+gam create adminrole <String> privileges all|all_ou|<PrivilegesList> [description <String>]
+gam update adminrole <RoleItem> [name <String>] [privileges all|all_ou|<PrivilegesList>] [description <String>]
+gam delete adminrole <RoleItem>
 gam print adminroles|roles [todrive]
 
 gam create domain <DomainName>
@@ -872,9 +978,12 @@ gam update customer <CustomerAttributes>*
 
 gam info customer
 
-<DataTransferService> ::= googledrive|gdrive|drive|"drive and docs"|calendar|gplus|google+|googleplus
+<DataTransferService> ::=
+	calendar|
+	googledrive|gdrive|drive|"drive and docs"
+<DataTransferServiceList> ::= "<DataTransferService>(,<DataTransferService>)*"
 
-gam create datatransfer|transfer <OldOwnerID> <DataTransferService> <NewOwnerID> (<ParameterKey> <ParameterValue>)*
+gam create datatransfer|transfer <OldOwnerID> <DataTransferServiceList> <NewOwnerID> (<ParameterKey> <ParameterValue>)*
 gam info datatransfer|transfer <TransferID>
 gam print datatransfers|transfers [todrive] [olduser|oldowner <UserItem>] [newuser|newowner <UserItem>] [status <String>]
 
@@ -896,11 +1005,63 @@ gam print aliases|nicknames [todrive] [shownoneditable] [nogroups] [nousers] [(q
 gam calendar <CalendarItem> add <CalendarACLRole> ([user] <EmailAddress>)|(group <EmailAddress>)|(domain [<DomainName>])|default [sendnotifications <Boolean>]
 gam calendar <CalendarItem> update <CalendarACLRole> ([user] <EmailAddress>)|(group <EmailAddress>)|(domain [<DomainName>])|default [sendnotifications <Boolean>]
 gam calendar <CalendarItem> del|delete <CalendarACLRole> <EmailAddress>|(domain [<DomainName>])|default
+gam calendar <CalendarItem> del|delete id <CalendarACLRuleID>
 gam calendar <CalendarItem> showacl
+gam calendar <CalendarItem> printacl [todrive]
 
-gam calendar <CalendarItem> addevent <EventAttributes>+
-gam calendar <CalendarItem> deleteevent (id|eventid <EventID>)* (query|eventquery <QueryCalendar>)* [doit] [notifyattendees]
+<EventNotificationAttribute> ::=
+	notifyattendees|(sendnotifications <Boolean>)|(sendupdates all|enternalonly|none)
+
+The following attributes are equivalent:
+	notifyattendees - sendupdates all
+	sendnotifications false - sendupdates none
+	sendnotifications true - sendupdates all
+
+<EventAttributes> ::=
+	anyonecanaddself|
+	(attendee <EmailAddress>)|
+	available|
+	(colorindex|colorid <EventColorIndex>)
+	(description <String>)|
+	(end (allday <Date>)|<Time>)|
+	guestscantinviteothers|
+	guestscantseeothers|
+	hangoutsmeet|
+	(location <String>)|
+	(noreminders| (reminder <Number> email|popup|sms))|
+	(optionalattendee <EmailAddress>)|
+	(privateproperty <PropertyKey> <PropertyValue>)|
+	(recurrence <RRULE, EXRULE, RDATE and EXDATE line>)|
+	(sharedproperty <PropertyKey> <PropertyValue>)|
+	(source <String> <URL>)|
+	(start (allday <Date>)|<Time>)|
+	(summary <String>)|
+	tentative|
+	(timezone <Timezone>)|
+	(visibility default|public|prvate)
+
+<EventUpdateAttributes> ::=
+	<EventAttributes>|
+	(removeattendee <EmailAddress>)|
+	(replacedescription <RegularExpression> <String>)
+
+<EventSelectProperty:> ::=
+	(after <Time>)|
+	(before <Time>)|
+	includeeleted|
+	includehidden|
+	(query <QueryCalendar>)|
+	(updatedmin <Time>)
+
+<EventDisplayProperty> ::=
+	(timezone <TimeZone>)
+
+gam calendar <CalendarItem> addevent [id <String>] <EventAttributes>+ [<EventNotificationAttribute>]
+gam calendar <CalendarItem> deleteevent id|eventid <EventID> [doit] [<EventNotificationAttribute>]
+gam calendar <CalendarItem> moveevent id|eventid <EventID> [doit] [<EventNotificationAttribute>]
+gam calendar <CalendarItem> updateevent <EventID> <EventUpdateAttributes>+ [<EventNotificationAttribute>]
 gam calendar <CalendarItem> wipe
+gam calendar <CalendarItem> printevents <EventSelectProperty>* <EventDisplayProperty>* [todrive]
 
 <CalendarSettings> ::=
 	summary <String>|
@@ -910,7 +1071,15 @@ gam calendar <CalendarItem> wipe
 
 gam calendar <CalendarItem> modify <CalendarSettings>+
 
-gam update cros <CrOSEntity> (<CrOSAttributes>+)|(action deprovision_same_model_replace|deprovision_different_model_replace|deprovision_retiring_device|disable|reenable [acknowledge_device_touch_requirement])
+<CrOSAction> ::=
+	deprovision_same_model_replace|
+	deprovision_different_model_replace|
+	deprovision_retiring_device|
+	deprovision_upgrade_transfer|
+	disable|
+	reenable
+
+gam update cros <CrOSEntity> (<CrOSAttributes>+)|(action <CrOSAction> [acknowledge_device_touch_requirement])
 gam info cros <CrOSEntity> [nolists] [listlimit <Number>] [start <Date>] [end <Date>]
 	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>] [downloadfile latest|<Time>] [targetfolder <FilePath>]
 
@@ -958,18 +1127,18 @@ The listlimit <Number> argument limits the number of recent users, time ranges a
 The start <Date> and end <Date> arguments filter the time ranges.
 Delimiter defaults to comma.
 
-gam update mobile <MobileID> action <MobileAction>
+gam update mobile <MobileID>|query:<QueryMobile> action <MobileAction> [doit] [if_users|match_users <UserTypeEntity>]
 gam delete mobile <MobileID>
 gam info mobile <MobileID>
 gam print mobile [todrive] [(query <QueryMobile>)|(queries <QueryMobileList>)] [basic|full] [orderby <MobileOrderByFieldName> [ascending|descending]]
 	fields <MobileFieldNameList>] [delimiter <Character>] [appslimit <Number>] [listlimit <Number>]
 
 gam create group <EmailAddress> <GroupAttributes>*
-gam update group <GroupItem> [admincreated <Boolean>] [email <EmailAddress>] <GroupAttributes>*
-gam update group <GroupItem> add [owner|manager|member] [notsuspended|suspended] [delivery allmail|daily|digest|none|nomail] <UserTypeEntity>
+gam update group <GroupItem> [email <EmailAddress>] <GroupAttributes>*
+gam update group <GroupItem> add [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
 gam update group <GroupItem> delete|remove [owner|manager|member] <UserTypeEntity>
-gam update group <GroupItem> sync [owner|manager|member] [notsuspended|suspended] [delivery allmail|daily|digest|none|nomail] <UserTypeEntity>
-gam update group <GroupItem> update [owner|manager|member] [notsuspended|suspended] [delivery allmail|daily|digest|none|nomail] <UserTypeEntity>
+gam update group <GroupItem> sync [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
+gam update group <GroupItem> update [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
 gam update group <GroupItem> clear [member] [manager] [owner] [notsuspended|suspended]
 gam delete group <GroupItem>
 gam info group <GroupItem> [nousers] [noaliases] [groups]
@@ -983,12 +1152,10 @@ gam info member <UserItem> <GroupItem>
 gam print group-members|groups-members [todrive]
 	([domain <DomainName>] ([member <UserItem>]|[query <QueryGroup>]))|[group|group_ns|group_susp <GroupItem>] [notsuspended|suspended]
 	[roles <GroupRoleList>] [membernames] [fields <MembersFieldNameList>]
+	[includederivedmembership]
 
-gam print license|licenses|licence|licences [todrive] [(products|product <ProductIDList>)|(skus|sku <SKUIDList>)]
-
-gam update notification|notifications [(id all)|(id <NotificationID>)*] unread|read
-gam delete notification|notifications [(id all)|(id <NotificationID>)*]
-gam info notification|notifications [unreadonly]
+gam print licenses [todrive] [(products|product <ProductIDList>)|(skus|sku <SKUIDList>)|allskus|gsuite] [countsonly]
+gam show license|licenses|licence|licences [(products|product <ProductIDList>)|(skus|sku <SKUIDList>)|allskus|gsuite]
 
 gam create building <Name> <BuildingAttributes>*
 gam update building <BuildIngID> <BuildingAttributes>*
@@ -1005,7 +1172,7 @@ gam create resource <ResourceID> <Name> <ResourceAttributes>*
 gam update resource <ResourceID> <ResourceAttributes>*
 gam delete resource <ResourceID>
 gam info resource <ResourceID>
-gam print resources [todrive] [allfields] <ResourceFieldName>*
+gam print resources [todrive] [allfields] <ResourceFieldName>* [query <String>]
 
 gam create schema|schemas <SchemaName> <SchemaFieldDefinition>+
 gam update schema <SchemaName> <SchemaFieldDefinition>* (deletefield <FieldName>)*
@@ -1086,9 +1253,11 @@ gam print printjobs [todrive] [printer|printerid <PrinterID>]
 gam create vaultexport|export matter <MatterItem> [name <name>] corpus <drive|mail|groups|hangouts_chat>
 	(accounts <EmailAddressList>) | (orgunit|ou <OrgUnitPath>) | (teamdrives <TeamDriveList>) | (rooms <ChatRoomList>) | everyone
 	[scope <all_data|held_data|unprocessed_data>]
-	[terms <terms>] [start|starttime <Date>|<DateTime>] [end|endtime <Date>|<DateTime>] [timezone <TimeZone>] [format mbox|pst]
-	[excludedrafts <Boolean>] [driveversiondate <Date>|<DateTime>] [includeteamdrives] [includerooms]
-	[includeaccessinfo <Boolean>]
+	[terms <terms>] [start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>] [timezone <TimeZone>]
+	[excludedrafts <Boolean>] [format mbox|pst] [showconfidentialmodecontent <Boolean>]
+	[includerooms]
+	[driveversiondate <Date>|<Time>] [includeshareddrives|includeteamdrives] [includeaccessinfo <Boolean>]
+	[region any|europe|us]
 gam delete export <MatterItem> <ExportItem>
 gam info export <MatterItem> <ExportItem>
 gam print exports [todrive] [matters <MatterItemList>]
@@ -1096,10 +1265,10 @@ gam download export <MatterItem> <ExportItem> [noverify] [noextract] [targetfold
 
 gam create vaulthold|hold corpus drive|groups|mail matter <MatterItem> [name <String>] [query <QueryVaultCorpus>]
 	[(accounts|groups|users <EmailItemList>) | (orgunit|ou <OrgUnit>)]
-	[start|starttime <Date>|<DateTime>] [end|endtime <Date>|<DateTime>] 
+	[start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>]
 gam update vaulthold|hold <HoldItem> matter <MatterItem> [query <QueryVaultCorpus>]
 	[([addaccounts|addgroups|addusers <EmailItemList>] [removeaccounts|removegroups|removeusers <EmailItemList>]) | (orgunit|ou <OrgUnit>)]
-	[start|starttime <Date>|<DateTime>] [end|endtime <Date>|<DateTime>] 
+	[start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>]
 gam delete vaulthold|hold <HoldItem> matter <MatterItem>
 gam info vaulthold|hold <HoldItem> matter <MatterItem>
 gam print vaultholds|holds [todrive] [matters <MatterItemList>]
@@ -1114,7 +1283,7 @@ gam reopen vaultmatter|matter <MatterItem>
 gam delete vaultmatter|matter <MatterItem>
 gam undelete vaultmatter|matter <MatterItem>
 gam info vaultmatter|matter <MatterItem>
-gam print vaultmatters|matters [todrive] [basic|full]
+gam print vaultmatters|matters [todrive] [basic|full] [matterstate open|closed|deleted]
 
 gam <UserTypeEntity> delete|del asp|asps|applicationspecificpasswords all|<ASPIDList>
 gam <UserTypeEntity> show asps|asp|applicationspecificpasswords
@@ -1143,10 +1312,11 @@ gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
 gam <UserTypeEntity> show filerevisions <DriveFileID>
 gam <UserTypeEntity> show filetree [anyowner] (orderby <DriveOrderByFieldName> [ascending|descending])*
 
-gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive]
+gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive] [returnidonly]
 gam <UserTypeEntity> update drivefile (id <DriveFileID)|(drivefilename <DriveFileName>)|(query <QueryDriveFile) [copy] [newfilename <DriveFileName>] <DriveFileUpdateAttributes>*
-gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>) [revision <Number>] [format <FileFormatList>]
-	[targetfolder <FilePath>] [targetname <FileName>] [overwrite] [showprogress]
+gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>)
+	[revision <Number>] [(format <FileFormatList>)|(csvsheet <String>)]
+	[targetfolder <FilePath>] [targetname -|<FileName>] [overwrite] [showprogress]
 gam <UserTypeEntity> delete|del drivefile <DriveFileID>|<DriveFileURL>|(query:<QueryDriveFile>) [purge|untrash]
 gam <UserTypeEntity> transfer drive <UserItem> [keepuser]
 gam <UserTypeEntity> delete|del emptydrivefolders
@@ -1197,7 +1367,21 @@ gam <UserTypeEntity> trash messages query <QueryGmail> [doit] [max_to_trash|max_
 gam <UserTypeEntity> untrash messages query <QueryGmail> [doit] [max_to_untrash|max_to_process <Number>]
 
 gam <UserTypeEntity> show gmailprofile [todrive]
-gam <UserTypeEntity> show gplusprofile [todrive]
+
+gam <UserTypeEntity> draftemail [recipient|to <EmailAddress>] [from <EmailAddress>]
+	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]
+gam <UserTypeEntity> importemail [recipient|to <EmailAddress>] [from <EmailAddress>]
+	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]
+	[labels <LabelNameList>] (header <String> <String>)*
+	[deleted] [date <Time>]
+	[nevercheckspam] [processforcalendar]
+gam <UserTypeEntity> insertemail [recipient|to <EmailAddress>] [from <EmailAddress>]
+	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]
+	[labels <LabelNameList>] (header <String> <String>)*
+	[deleted] [date <Time>]
+gam <UserTypeEntity> sendemail [recipient|to <EmailAddress>] [from <EmailAddress>]
+	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]
+	(header <String> <String>)*
 
 gam <UserTypeEntity> create|add delegate|delegates <EmailAddress>
 gam <UserTypeEntity> delegate|delegates to <EmailAddress>
@@ -1229,7 +1413,11 @@ gam <UserTypeEntity> show imap|imap4
 gam <UserTypeEntity> pop|pop3 <Boolean> [for allmail|newmail|mailfromnowon|fromnowown] [action keep|leaveininbox|archive|delete|trash|markread]
 gam <UserTypeEntity> show pop|pop3
 
+gam <UserTypeEntity> language <Language>
+gam <UserTypeEntity> show language
+
 gam <UserTypeEntity> [create|add] sendas <EmailAddress> <Name> [signature|sig <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)*] [html] [replyto <EmailAddress>] [default] [treatasalias <Boolean>]
+	[smtpmsa.host <SMTPHostName> smtpmsa.port 25|465|587 smtpmsa.username <UserName> smtpmsa.password <Password> [smtpmsa.securitymode none|ssl|starttls]]
 gam <UserTypeEntity> update sendas <EmailAddress> [name <Name>] [signature|sig <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)*] [html] [replyto <EmailAddress>] [default] [treatasalias <Boolean>]
 gam <UserTypeEntity> delete sendas <EmailAddress>
 gam <UserTypeEntity> show sendas [format]
@@ -1245,8 +1433,16 @@ gam <UserTypeEntity> print smime [todrive] [primaryonly]
 gam <UserTypeEntity> signature|sig <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)* [html] [name <String>] [replyto <EmailAddress>] [default] [treatasalias <Boolean>]
 gam <UserTypeEntity> show signature|sig [format]
 
+<TeamDriveRestrictionsSubfieldName> ::=
+	adminmanagedrestrictions|
+	copyrequireswriterpermission|
+	domainusersonly|
+	teammembersonly
+
 gam <UserTypeEntity> create|add teamdrive <Name>
-gam <UserTypeEntity> update teamdrive <TeamDriveID> [name <Name>] [(theme|themeid <String>) | ([customtheme <DriveFileID> <Float> <Float> <Float>] [color <ColorValue>])]
+gam <UserTypeEntity> update teamdrive <TeamDriveID> [asadmin] [name <Name>]
+	[(theme|themeid <String>) | ([customtheme <DriveFileID> <Float> <Float> <Float>] [color <ColorValue>])]
+	(<TeamDriveRestrictionsSubfieldName> <Boolean>)*
 gam <UserTypeEntity> delete teamdrive <TeamDriveID>
 gam <UserTypeEntity> show teamdriveinfo <TeamDriveID> [asadmin]
 gam <UserTypeEntity> show teamdrives [asadmin]
@@ -1257,3 +1453,7 @@ gam <UserTypeEntity> vacation <FalseValues>
 gam <UserTypeEntity> vacation <TrueValues> subject <String> (message <String>)|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)* [html]
 	[contactsonly] [domainonly] [startdate <Date>] [enddate <Date>]
 gam <UserTypeEntity> show vacation [format]
+
+gam <UserTypeEntity> signout
+gam <UserTypeEntity> turnoff2sv
+

src/LICENSE

@@ -202,12 +202,12 @@
 
 
 
-APACHE HTTP SERVER SUBCOMPONENTS: 
+APACHE HTTP SERVER SUBCOMPONENTS:
 
 The Apache HTTP Server includes a number of subcomponents with
 separate copyright notices and license terms. Your use of the source
 code for the these subcomponents is subject to the terms and
-conditions of the following licenses. 
+conditions of the following licenses.
 
 For the mod_mime_magic component:
 
@@ -273,7 +273,7 @@ For the  server\util_md5.c component:
  *  Original Code Copyright (C) 1994, Jeff Hostetler, Spyglass, Inc.
  *  Portions of Content-MD5 code Copyright (C) 1993, 1994 by Carnegie Mellon
  *     University (see Copyright below).
- *  Portions of Content-MD5 code Copyright (C) 1991 Bell Communications 
+ *  Portions of Content-MD5 code Copyright (C) 1991 Bell Communications
  *     Research, Inc. (Bellcore) (see Copyright below).
  *  Portions extracted from mpack, John G. Myers - jgm+@cmu.edu
  *  Content-MD5 Code contributed by Martin Hamilton (martin@net.lut.ac.uk)
@@ -319,10 +319,10 @@ For the  server\util_md5.c component:
  * of an authorized representative of Bellcore.  BELLCORE
  * MAKES NO REPRESENTATIONS ABOUT THE ACCURACY OR SUITABILITY
  * OF THIS MATERIAL FOR ANY PURPOSE.  IT IS PROVIDED "AS IS",
- * WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.  
+ * WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
  */
 
-For the  srclib\apr\include\apr_md5.h component: 
+For the  srclib\apr\include\apr_md5.h component:
 /*
  * This is work is derived from material Copyright RSA Data Security, Inc.
  *
@@ -501,21 +501,21 @@ This program is Copyright (C) Zeus Technology Limited 1996.
 This program may be used and copied freely providing this copyright notice
 is not removed.
 
-This software is provided "as is" and any express or implied waranties, 
+This software is provided "as is" and any express or implied waranties,
 including but not limited to, the implied warranties of merchantability and
-fitness for a particular purpose are disclaimed.  In no event shall 
-Zeus Technology Ltd. be liable for any direct, indirect, incidental, special, 
-exemplary, or consequential damaged (including, but not limited to, 
+fitness for a particular purpose are disclaimed.  In no event shall
+Zeus Technology Ltd. be liable for any direct, indirect, incidental, special,
+exemplary, or consequential damaged (including, but not limited to,
 procurement of substitute good or services; loss of use, data, or profits;
 or business interruption) however caused and on theory of liability.  Whether
-in contract, strict liability or tort (including negligence or otherwise) 
+in contract, strict liability or tort (including negligence or otherwise)
 arising in any way out of the use of this software, even if advised of the
 possibility of such damage.
 
      Written by Adam Twiss (adam@zeus.co.uk).  March 1996
 
 Thanks to the following people for their input:
-  Mike Belshe (mbelshe@netscape.com) 
+  Mike Belshe (mbelshe@netscape.com)
   Michael Campanella (campanella@stevms.enet.dec.com)
 
 */
@@ -532,10 +532,10 @@ without limitation the rights to use, copy, modify, merge, publish,
 distribute, sublicense, and/or sell copies of the Software, and to
 permit persons to whom the Software is furnished to do so, subject to
 the following conditions:
-	
+
 The above copyright notice and this permission notice shall be included
 in all copies or substantial portions of the Software.
-	
+
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

src/cachetools/__init__.py

@@ -1,112 +0,0 @@
-"""Extensible memoizing collections and decorators."""
-
-from __future__ import absolute_import
-
-import functools
-
-from . import keys
-from .cache import Cache
-from .lfu import LFUCache
-from .lru import LRUCache
-from .rr import RRCache
-from .ttl import TTLCache
-
-__all__ = (
-    'Cache', 'LFUCache', 'LRUCache', 'RRCache', 'TTLCache',
-    'cached', 'cachedmethod'
-)
-
-__version__ = '2.1.0'
-
-if hasattr(functools.update_wrapper(lambda f: f(), lambda: 42), '__wrapped__'):
-    _update_wrapper = functools.update_wrapper
-else:
-    def _update_wrapper(wrapper, wrapped):
-        functools.update_wrapper(wrapper, wrapped)
-        wrapper.__wrapped__ = wrapped
-        return wrapper
-
-
-def cached(cache, key=keys.hashkey, lock=None):
-    """Decorator to wrap a function with a memoizing callable that saves
-    results in a cache.
-
-    """
-    def decorator(func):
-        if cache is None:
-            def wrapper(*args, **kwargs):
-                return func(*args, **kwargs)
-        elif lock is None:
-            def wrapper(*args, **kwargs):
-                k = key(*args, **kwargs)
-                try:
-                    return cache[k]
-                except KeyError:
-                    pass  # key not found
-                v = func(*args, **kwargs)
-                try:
-                    cache[k] = v
-                except ValueError:
-                    pass  # value too large
-                return v
-        else:
-            def wrapper(*args, **kwargs):
-                k = key(*args, **kwargs)
-                try:
-                    with lock:
-                        return cache[k]
-                except KeyError:
-                    pass  # key not found
-                v = func(*args, **kwargs)
-                try:
-                    with lock:
-                        cache[k] = v
-                except ValueError:
-                    pass  # value too large
-                return v
-        return _update_wrapper(wrapper, func)
-    return decorator
-
-
-def cachedmethod(cache, key=keys.hashkey, lock=None):
-    """Decorator to wrap a class or instance method with a memoizing
-    callable that saves results in a cache.
-
-    """
-    def decorator(method):
-        if lock is None:
-            def wrapper(self, *args, **kwargs):
-                c = cache(self)
-                if c is None:
-                    return method(self, *args, **kwargs)
-                k = key(self, *args, **kwargs)
-                try:
-                    return c[k]
-                except KeyError:
-                    pass  # key not found
-                v = method(self, *args, **kwargs)
-                try:
-                    c[k] = v
-                except ValueError:
-                    pass  # value too large
-                return v
-        else:
-            def wrapper(self, *args, **kwargs):
-                c = cache(self)
-                if c is None:
-                    return method(self, *args, **kwargs)
-                k = key(self, *args, **kwargs)
-                try:
-                    with lock(self):
-                        return c[k]
-                except KeyError:
-                    pass  # key not found
-                v = method(self, *args, **kwargs)
-                try:
-                    with lock(self):
-                        c[k] = v
-                except ValueError:
-                    pass  # value too large
-                return v
-        return _update_wrapper(wrapper, method)
-    return decorator

src/cachetools/abc.py

@@ -1,48 +0,0 @@
-from __future__ import absolute_import
-
-import collections
-from abc import abstractmethod
-
-
-class DefaultMapping(collections.MutableMapping):
-
-    __slots__ = ()
-
-    @abstractmethod
-    def __contains__(self, key):  # pragma: nocover
-        return False
-
-    @abstractmethod
-    def __getitem__(self, key):  # pragma: nocover
-        if hasattr(self.__class__, '__missing__'):
-            return self.__class__.__missing__(self, key)
-        else:
-            raise KeyError(key)
-
-    def get(self, key, default=None):
-        if key in self:
-            return self[key]
-        else:
-            return default
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        if key in self:
-            value = self[key]
-            del self[key]
-        elif default is self.__marker:
-            raise KeyError(key)
-        else:
-            value = default
-        return value
-
-    def setdefault(self, key, default=None):
-        if key in self:
-            value = self[key]
-        else:
-            self[key] = value = default
-        return value
-
-
-DefaultMapping.register(dict)

src/cachetools/cache.py

@@ -1,110 +0,0 @@
-from __future__ import absolute_import
-
-from warnings import warn
-
-from .abc import DefaultMapping
-
-
-class _DefaultSize(object):
-    def __getitem__(self, _):
-        return 1
-
-    def __setitem__(self, _, value):
-        assert value == 1
-
-    def pop(self, _):
-        return 1
-
-
-_deprecated = object()
-
-
-class Cache(DefaultMapping):
-    """Mutable mapping to serve as a simple cache or cache base class."""
-
-    __size = _DefaultSize()
-
-    def __init__(self, maxsize, missing=_deprecated, getsizeof=None):
-        if missing is not _deprecated:
-            warn("Cache constructor parameter 'missing' is deprecated",
-                 DeprecationWarning, 3)
-            if missing:
-                self.__missing = missing
-        if getsizeof:
-            self.getsizeof = getsizeof
-        if self.getsizeof is not Cache.getsizeof:
-            self.__size = dict()
-        self.__data = dict()
-        self.__currsize = 0
-        self.__maxsize = maxsize
-
-    def __repr__(self):
-        return '%s(%r, maxsize=%r, currsize=%r)' % (
-            self.__class__.__name__,
-            list(self.__data.items()),
-            self.__maxsize,
-            self.__currsize,
-        )
-
-    def __getitem__(self, key):
-        try:
-            return self.__data[key]
-        except KeyError:
-            return self.__missing__(key)
-
-    def __setitem__(self, key, value):
-        maxsize = self.__maxsize
-        size = self.getsizeof(value)
-        if size > maxsize:
-            raise ValueError('value too large')
-        if key not in self.__data or self.__size[key] < size:
-            while self.__currsize + size > maxsize:
-                self.popitem()
-        if key in self.__data:
-            diffsize = size - self.__size[key]
-        else:
-            diffsize = size
-        self.__data[key] = value
-        self.__size[key] = size
-        self.__currsize += diffsize
-
-    def __delitem__(self, key):
-        size = self.__size.pop(key)
-        del self.__data[key]
-        self.__currsize -= size
-
-    def __contains__(self, key):
-        return key in self.__data
-
-    def __missing__(self, key):
-        value = self.__missing(key)
-        try:
-            self.__setitem__(key, value)
-        except ValueError:
-            pass  # value too large
-        return value
-
-    def __iter__(self):
-        return iter(self.__data)
-
-    def __len__(self):
-        return len(self.__data)
-
-    @property
-    def maxsize(self):
-        """The maximum size of the cache."""
-        return self.__maxsize
-
-    @property
-    def currsize(self):
-        """The current size of the cache."""
-        return self.__currsize
-
-    @staticmethod
-    def getsizeof(value):
-        """Return the size of a cache element's value."""
-        return 1
-
-    @staticmethod
-    def __missing(key):
-        raise KeyError(key)

src/cachetools/func.py

@@ -1,106 +0,0 @@
-"""`functools.lru_cache` compatible memoizing function decorators."""
-
-from __future__ import absolute_import
-
-import collections
-import functools
-import random
-import time
-
-try:
-    from threading import RLock
-except ImportError:
-    from dummy_threading import RLock
-
-from . import keys
-from .lfu import LFUCache
-from .lru import LRUCache
-from .rr import RRCache
-from .ttl import TTLCache
-
-__all__ = ('lfu_cache', 'lru_cache', 'rr_cache', 'ttl_cache')
-
-
-_CacheInfo = collections.namedtuple('CacheInfo', [
-    'hits', 'misses', 'maxsize', 'currsize'
-])
-
-
-def _cache(cache, typed=False):
-    def decorator(func):
-        key = keys.typedkey if typed else keys.hashkey
-        lock = RLock()
-        stats = [0, 0]
-
-        def cache_info():
-            with lock:
-                hits, misses = stats
-                maxsize = cache.maxsize
-                currsize = cache.currsize
-            return _CacheInfo(hits, misses, maxsize, currsize)
-
-        def cache_clear():
-            with lock:
-                try:
-                    cache.clear()
-                finally:
-                    stats[:] = [0, 0]
-
-        def wrapper(*args, **kwargs):
-            k = key(*args, **kwargs)
-            with lock:
-                try:
-                    v = cache[k]
-                    stats[0] += 1
-                    return v
-                except KeyError:
-                    stats[1] += 1
-            v = func(*args, **kwargs)
-            try:
-                with lock:
-                    cache[k] = v
-            except ValueError:
-                pass  # value too large
-            return v
-        functools.update_wrapper(wrapper, func)
-        if not hasattr(wrapper, '__wrapped__'):
-            wrapper.__wrapped__ = func  # Python 2.7
-        wrapper.cache_info = cache_info
-        wrapper.cache_clear = cache_clear
-        return wrapper
-    return decorator
-
-
-def lfu_cache(maxsize=128, typed=False):
-    """Decorator to wrap a function with a memoizing callable that saves
-    up to `maxsize` results based on a Least Frequently Used (LFU)
-    algorithm.
-
-    """
-    return _cache(LFUCache(maxsize), typed)
-
-
-def lru_cache(maxsize=128, typed=False):
-    """Decorator to wrap a function with a memoizing callable that saves
-    up to `maxsize` results based on a Least Recently Used (LRU)
-    algorithm.
-
-    """
-    return _cache(LRUCache(maxsize), typed)
-
-
-def rr_cache(maxsize=128, choice=random.choice, typed=False):
-    """Decorator to wrap a function with a memoizing callable that saves
-    up to `maxsize` results based on a Random Replacement (RR)
-    algorithm.
-
-    """
-    return _cache(RRCache(maxsize, choice), typed)
-
-
-def ttl_cache(maxsize=128, ttl=600, timer=time.time, typed=False):
-    """Decorator to wrap a function with a memoizing callable that saves
-    up to `maxsize` results based on a Least Recently Used (LRU)
-    algorithm with a per-item time-to-live (TTL) value.
-    """
-    return _cache(TTLCache(maxsize, ttl, timer), typed)

src/cachetools/keys.py

@@ -1,43 +0,0 @@
-"""Key functions for memoizing decorators."""
-
-from __future__ import absolute_import
-
-__all__ = ('hashkey', 'typedkey')
-
-
-class _HashedTuple(tuple):
-
-    __hashvalue = None
-
-    def __hash__(self, hash=tuple.__hash__):
-        hashvalue = self.__hashvalue
-        if hashvalue is None:
-            self.__hashvalue = hashvalue = hash(self)
-        return hashvalue
-
-    def __add__(self, other, add=tuple.__add__):
-        return _HashedTuple(add(self, other))
-
-    def __radd__(self, other, add=tuple.__add__):
-        return _HashedTuple(add(other, self))
-
-
-_kwmark = (object(),)
-
-
-def hashkey(*args, **kwargs):
-    """Return a cache key for the specified hashable arguments."""
-
-    if kwargs:
-        return _HashedTuple(args + sum(sorted(kwargs.items()), _kwmark))
-    else:
-        return _HashedTuple(args)
-
-
-def typedkey(*args, **kwargs):
-    """Return a typed cache key for the specified hashable arguments."""
-
-    key = hashkey(*args, **kwargs)
-    key += tuple(type(v) for v in args)
-    key += tuple(type(v) for _, v in sorted(kwargs.items()))
-    return key

src/cachetools/lfu.py

@@ -1,35 +0,0 @@
-from __future__ import absolute_import
-
-import collections
-
-from .cache import Cache, _deprecated
-
-
-class LFUCache(Cache):
-    """Least Frequently Used (LFU) cache implementation."""
-
-    def __init__(self, maxsize, missing=_deprecated, getsizeof=None):
-        Cache.__init__(self, maxsize, missing, getsizeof)
-        self.__counter = collections.Counter()
-
-    def __getitem__(self, key, cache_getitem=Cache.__getitem__):
-        value = cache_getitem(self, key)
-        self.__counter[key] -= 1
-        return value
-
-    def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
-        cache_setitem(self, key, value)
-        self.__counter[key] -= 1
-
-    def __delitem__(self, key, cache_delitem=Cache.__delitem__):
-        cache_delitem(self, key)
-        del self.__counter[key]
-
-    def popitem(self):
-        """Remove and return the `(key, value)` pair least frequently used."""
-        try:
-            (key, _), = self.__counter.most_common(1)
-        except ValueError:
-            raise KeyError('%s is empty' % self.__class__.__name__)
-        else:
-            return (key, self.pop(key))

src/cachetools/lru.py

@@ -1,48 +0,0 @@
-from __future__ import absolute_import
-
-import collections
-
-from .cache import Cache, _deprecated
-
-
-class LRUCache(Cache):
-    """Least Recently Used (LRU) cache implementation."""
-
-    def __init__(self, maxsize, missing=_deprecated, getsizeof=None):
-        Cache.__init__(self, maxsize, missing, getsizeof)
-        self.__order = collections.OrderedDict()
-
-    def __getitem__(self, key, cache_getitem=Cache.__getitem__):
-        value = cache_getitem(self, key)
-        self.__update(key)
-        return value
-
-    def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
-        cache_setitem(self, key, value)
-        self.__update(key)
-
-    def __delitem__(self, key, cache_delitem=Cache.__delitem__):
-        cache_delitem(self, key)
-        del self.__order[key]
-
-    def popitem(self):
-        """Remove and return the `(key, value)` pair least recently used."""
-        try:
-            key = next(iter(self.__order))
-        except StopIteration:
-            raise KeyError('%s is empty' % self.__class__.__name__)
-        else:
-            return (key, self.pop(key))
-
-    if hasattr(collections.OrderedDict, 'move_to_end'):
-        def __update(self, key):
-            try:
-                self.__order.move_to_end(key)
-            except KeyError:
-                self.__order[key] = None
-    else:
-        def __update(self, key):
-            try:
-                self.__order[key] = self.__order.pop(key)
-            except KeyError:
-                self.__order[key] = None

src/cachetools/rr.py

@@ -1,37 +0,0 @@
-from __future__ import absolute_import
-
-import random
-
-from .cache import Cache, _deprecated
-
-
-# random.choice cannot be pickled in Python 2.7
-def _choice(seq):
-    return random.choice(seq)
-
-
-class RRCache(Cache):
-    """Random Replacement (RR) cache implementation."""
-
-    def __init__(self, maxsize, choice=random.choice, missing=_deprecated,
-                 getsizeof=None):
-        Cache.__init__(self, maxsize, missing, getsizeof)
-        # TODO: use None as default, assing to self.choice directly?
-        if choice is random.choice:
-            self.__choice = _choice
-        else:
-            self.__choice = choice
-
-    @property
-    def choice(self):
-        """The `choice` function used by the cache."""
-        return self.__choice
-
-    def popitem(self):
-        """Remove and return a random `(key, value)` pair."""
-        try:
-            key = self.__choice(list(self))
-        except IndexError:
-            raise KeyError('%s is empty' % self.__class__.__name__)
-        else:
-            return (key, self.pop(key))

src/cachetools/ttl.py

@@ -1,217 +0,0 @@
-from __future__ import absolute_import
-
-import collections
-import time
-
-from .cache import Cache, _deprecated
-
-
-class _Link(object):
-
-    __slots__ = ('key', 'expire', 'next', 'prev')
-
-    def __init__(self, key=None, expire=None):
-        self.key = key
-        self.expire = expire
-
-    def __reduce__(self):
-        return _Link, (self.key, self.expire)
-
-    def unlink(self):
-        next = self.next
-        prev = self.prev
-        prev.next = next
-        next.prev = prev
-
-
-class _Timer(object):
-
-    def __init__(self, timer):
-        self.__timer = timer
-        self.__nesting = 0
-
-    def __call__(self):
-        if self.__nesting == 0:
-            return self.__timer()
-        else:
-            return self.__time
-
-    def __enter__(self):
-        if self.__nesting == 0:
-            self.__time = time = self.__timer()
-        else:
-            time = self.__time
-        self.__nesting += 1
-        return time
-
-    def __exit__(self, *exc):
-        self.__nesting -= 1
-
-    def __reduce__(self):
-        return _Timer, (self.__timer,)
-
-    def __getattr__(self, name):
-        return getattr(self.__timer, name)
-
-
-class TTLCache(Cache):
-    """LRU Cache implementation with per-item time-to-live (TTL) value."""
-
-    def __init__(self, maxsize, ttl, timer=time.time, missing=_deprecated,
-                 getsizeof=None):
-        Cache.__init__(self, maxsize, missing, getsizeof)
-        self.__root = root = _Link()
-        root.prev = root.next = root
-        self.__links = collections.OrderedDict()
-        self.__timer = _Timer(timer)
-        self.__ttl = ttl
-
-    def __contains__(self, key):
-        try:
-            link = self.__links[key]  # no reordering
-        except KeyError:
-            return False
-        else:
-            return not (link.expire < self.__timer())
-
-    def __getitem__(self, key, cache_getitem=Cache.__getitem__):
-        try:
-            link = self.__getlink(key)
-        except KeyError:
-            expired = False
-        else:
-            expired = link.expire < self.__timer()
-        if expired:
-            return self.__missing__(key)
-        else:
-            return cache_getitem(self, key)
-
-    def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
-        with self.__timer as time:
-            self.expire(time)
-            cache_setitem(self, key, value)
-        try:
-            link = self.__getlink(key)
-        except KeyError:
-            self.__links[key] = link = _Link(key)
-        else:
-            link.unlink()
-        link.expire = time + self.__ttl
-        link.next = root = self.__root
-        link.prev = prev = root.prev
-        prev.next = root.prev = link
-
-    def __delitem__(self, key, cache_delitem=Cache.__delitem__):
-        cache_delitem(self, key)
-        link = self.__links.pop(key)
-        link.unlink()
-        if link.expire < self.__timer():
-            raise KeyError(key)
-
-    def __iter__(self):
-        root = self.__root
-        curr = root.next
-        while curr is not root:
-            # "freeze" time for iterator access
-            with self.__timer as time:
-                if not (curr.expire < time):
-                    yield curr.key
-            curr = curr.next
-
-    def __len__(self):
-        root = self.__root
-        curr = root.next
-        time = self.__timer()
-        count = len(self.__links)
-        while curr is not root and curr.expire < time:
-            count -= 1
-            curr = curr.next
-        return count
-
-    def __setstate__(self, state):
-        self.__dict__.update(state)
-        root = self.__root
-        root.prev = root.next = root
-        for link in sorted(self.__links.values(), key=lambda obj: obj.expire):
-            link.next = root
-            link.prev = prev = root.prev
-            prev.next = root.prev = link
-        self.expire(self.__timer())
-
-    def __repr__(self, cache_repr=Cache.__repr__):
-        with self.__timer as time:
-            self.expire(time)
-            return cache_repr(self)
-
-    @property
-    def currsize(self):
-        with self.__timer as time:
-            self.expire(time)
-            return super(TTLCache, self).currsize
-
-    @property
-    def timer(self):
-        """The timer function used by the cache."""
-        return self.__timer
-
-    @property
-    def ttl(self):
-        """The time-to-live value of the cache's items."""
-        return self.__ttl
-
-    def expire(self, time=None):
-        """Remove expired items from the cache."""
-        if time is None:
-            time = self.__timer()
-        root = self.__root
-        curr = root.next
-        links = self.__links
-        cache_delitem = Cache.__delitem__
-        while curr is not root and curr.expire < time:
-            cache_delitem(self, curr.key)
-            del links[curr.key]
-            next = curr.next
-            curr.unlink()
-            curr = next
-
-    def clear(self):
-        with self.__timer as time:
-            self.expire(time)
-            Cache.clear(self)
-
-    def get(self, *args, **kwargs):
-        with self.__timer:
-            return Cache.get(self, *args, **kwargs)
-
-    def pop(self, *args, **kwargs):
-        with self.__timer:
-            return Cache.pop(self, *args, **kwargs)
-
-    def setdefault(self, *args, **kwargs):
-        with self.__timer:
-            return Cache.setdefault(self, *args, **kwargs)
-
-    def popitem(self):
-        """Remove and return the `(key, value)` pair least recently used that
-        has not already expired.
-
-        """
-        with self.__timer as time:
-            self.expire(time)
-            try:
-                key = next(iter(self.__links))
-            except StopIteration:
-                raise KeyError('%s is empty' % self.__class__.__name__)
-            else:
-                return (key, self.pop(key))
-
-    if hasattr(collections.OrderedDict, 'move_to_end'):
-        def __getlink(self, key):
-            value = self.__links[key]
-            self.__links.move_to_end(key)
-            return value
-    else:
-        def __getlink(self, key):
-            value = self.__links.pop(key)
-            self.__links[key] = value
-            return value

src/cloudprint-v2.json

@@ -1,486 +0,0 @@
-{
- "kind": "discovery#restDescription",
- "discoveryVersion": "v1",
- "id": "cloudprint:v2",
- "name": "cloudprint",
- "version": "v2",
- "revision": "20150605",
- "title": "Cloud Print API",
- "description": "Lets you access Cloud Print Printers",
- "ownerDomain": "google.com",
- "ownerName": "Google",
- "icons": {
-  "x16": "http://www.google.com/images/icons/product/search-16.gif",
-  "x32": "http://www.google.com/images/icons/product/search-32.gif"
- },
- "documentationLink": "https://developers.google.com/cloud-print",
- "protocol": "rest",
- "baseUrl": "https://www.google.com/",
- "basePath": "/cloudprint/",
- "rootUrl": "https://www.google.com/",
- "servicePath": "/cloudprint/",
- "parameters": {
-  "prettyPrint": {
-   "type": "boolean",
-   "description": "Returns response with indentations and line breaks.",
-   "default": "true",
-   "location": "query"
-  }
- },
- "auth": {
-  "oauth2": {
-   "scopes": {
-    "https://www.googleapis.com/auth/cloudprint": {
-     "description": "Manage Cloud Print"
-    }
-   }
-  }
- },
- "schemas": {
-  "Job": {
-   "id": "Job",
-   "type": "object",
-   "description": "Job Object",
-   "properties": {
-    "title": {
-     "type": "string",
-     "description": "Job Title"
-    },
-    "id": {
-     "type": "string",
-     "description": "Unique ID"
-    }
-   }
-  },
-  "Jobs": {
-   "id": "Jobs",
-   "type": "object",
-   "description": "List of Jobs.",
-   "properties": {
-    "jobs": {
-     "type": "array",
-     "description": "List of job objects.",
-     "items": {
-      "$ref": "Job"
-     }
-    }
-   }
-  },
-  "Printer": {
-   "id": "Printer",
-   "type": "object",
-   "description": "Printer Object",
-   "properties": {
-    "displayName": {
-     "type": "string",
-     "description": "Display Name"
-    },
-    "id": {
-     "type": "string",
-     "description": "Unique ID"
-    }
-   }
-  },
-  "Printers": {
-   "id": "Printers",
-   "type": "object",
-   "description": "List of Printers.",
-   "properties": {
-    "printers": {
-     "type": "array",
-     "description": "List of printer objects.",
-     "items": {
-      "$ref": "Printer"
-     }
-    }
-   }
-  }
- },
- "resources": {
-  "jobs": {
-   "methods": {
-    "delete": {
-     "id": "cloudprint.jobs.delete",
-     "path": "deletejob",
-     "httpMethod": "GET",
-     "parameters": {
-      "jobid": {
-       "type": "string",
-       "location": "query",
-       "required": "true"
-      }
-     }
-    },
-    "fetch": {
-     "id": "cloudprint.jobs.fetch",
-     "path": "fetch",
-     "httpMethod": "GET",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Jobs"
-     }
-    },
-    "getticket": {
-     "id": "cloudprint.jobs.getticket",
-     "path": "ticket",
-     "httpMethod": "GET",
-     "parameters": {
-      "jobid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "use_cjt": {
-       "type": "boolean",
-       "required": "true",
-       "location": "query"
-      }
-     }
-    }, 
-    "list": {
-     "id": "cloudprint.jobs.list",
-     "path": "jobs",
-     "httpMethod": "GET",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "location": "query"
-      },
-      "owner": {
-       "type": "string",
-       "location": "query"
-      },
-      "status": {
-       "type": "string",
-       "location": "query"
-      },
-      "q": {
-       "type": "string",
-       "location": "query"
-      },
-      "offset": {
-       "type": "string",
-       "location": "query"
-      },
-      "limit": {
-       "type": "string",
-       "location": "query"
-      },
-      "sortorder": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Jobs"
-     }
-    },
-    "update": {
-     "id": "cloudprint.jobs.update",
-     "path": "control",
-     "httpMethod": "GET",
-     "parameters": {
-      "jobid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "semantic_state_diff": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Jobs"
-     }
-    },
-    "resubmit": {
-     "id": "cloudprint.jobs.resubmit",
-     "path": "resubmit",
-     "httpMethod": "POST",
-     "description": "resubmit a job to new printer.",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "jobid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "ticket": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Job"
-     }
-    },
-    "submit": {
-     "id": "cloudprint.jobs.submit",
-     "path": "submit",
-     "httpMethod": "POST",
-     "description": "Send a print job to cloud print.",
-     "request": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "title": {
-       "type": "string",
-       "location": "query"
-      },
-      "ticket": {
-       "type": "string",
-       "location": "query"
-      },
-      "content": {
-       "type": "string",
-       "location": "query"
-      },
-      "contentType": {
-       "type": "string",
-       "location": "query"
-      },
-      "tag": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Job"
-     }
-    }
-   }
-  },
-  "printers": {
-   "methods": {
-    "get": {
-     "id": "cloudprint.printers.get",
-     "path": "printer",
-     "httpMethod": "GET",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "extra_fields": {
-       "type": "string",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Printer"
-     }
-    },
-    "list": {
-     "id": "cloudprint.printers.list",
-     "path": "search",
-     "httpMethod": "GET",
-     "description": "List all printers",
-     "parameters": {
-      "q": {
-       "type": "string",
-       "description": "Query list of printers",
-       "location": "query"
-      },
-      "type": {
-       "type": "string",
-       "description": "limit results to printers of type",
-       "location": "query"
-       },
-      "connection_status": {
-       "type": "string",
-       "description": "limit results to printers with this status",
-       "location": "query"
-      },
-      "extra_fields": {
-       "type": "string",
-       "description": "include extra fields",
-       "location": "query"
-      }
-     },
-     "response": {
-      "$ref": "Printers"
-     }
-    },
-    "share": {
-     "id": "cloudprint.printers.share",
-     "path": "share",
-     "httpMethod": "GET",
-     "description": "Share printer with user, group or domain",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "scope": {
-       "type": "string",
-       "location": "query"
-      },
-      "role": {
-       "type": "string",
-       "location": "query"
-      },
-      "type": {
-        "type": "string",
-        "location": "query"
-      },
-      "skip_notification": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "public": {
-       "type": "boolean",
-       "location": "query"
-      }
-     } 
-    },
-    "unshare": {
-     "id": "cloudprint.printers.unshare",
-     "path": "unshare",
-     "httpMethod": "GET",
-     "description": "unshare printer with user, group or domain",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "scope": {
-       "type": "string",
-       "location": "query"
-      },
-      "public": {
-       "type": "string",
-       "location": "query"
-      }
-     }
-    },
-    "delete": {
-     "id": "cloudprint.printers.delete",
-     "path": "delete",
-     "httpMethod": "GET",
-     "description": "delete a printer",
-     "parameters": {
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      }
-     }
-    },
-    "update": {
-     "id": "cloudprint.printers.update",
-     "path": "update",
-     "httpMethod": "GET",
-     "description": "update a printer",
-     "parameters": {
-      "isTosAccepted": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "gcpVersion": {
-       "type": "string",
-       "location": "query"
-      },
-      "setupUrl": {
-       "type": "string",
-       "location": "query"
-      },
-      "supportUrl": {
-       "type": "string",
-       "location": "query"
-      },
-      "firmware": {
-       "type": "string",
-       "location": "query"
-      },
-      "currentQuota": {
-       "type": "string",
-       "location": "query"
-      },
-      "type": {
-       "type": "string",
-       "location": "query"
-      },
-      "public": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "status": {
-       "type": "string",
-       "location": "query"
-      },
-      "proxy": {
-       "type": "string",
-       "location": "query"
-      },
-      "manufacturer": {
-       "type": "string",
-       "location": "query"
-      },
-      "defaultDisplayName": {
-       "type": "string",
-       "location": "query"
-      },
-      "displayName": {
-       "type": "string",
-       "location": "query"
-      },
-      "name": {
-       "type": "string",
-       "location": "query"
-      },
-      "uuid": {
-       "type": "string",
-       "location": "query"
-      },
-      "updateUrl": {
-       "type": "string",
-       "location": "query"
-      },
-      "ownerId": {
-       "type": "string",
-       "location": "query"
-      },
-      "model": {
-       "type": "string",
-       "location": "query"
-      },
-      "description": {
-       "type": "string",
-       "location": "query"
-      },
-      "printerid": {
-       "type": "string",
-       "required": "true",
-       "location": "query"
-      },
-      "quotaEnabled": {
-       "type": "boolean",
-       "location": "query"
-      },
-      "dailyQuota": {
-       "type": "string",
-       "location": "query"
-      }
-     }
-    } 
-   }
-  }
- }
-}

src/gam-install.sh

@@ -8,8 +8,9 @@ GAM installation script.
 OPTIONS:
    -h      show help.
    -d      Directory where gam folder will be installed. Default is \$HOME/bin/
-   -a      Architecture to install (i386, x86_64, arm). Default is to detect your arch with "uname -m".
+   -a      Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m".
    -o      OS we are running (linux, macos). Default is to detect your OS with "uname -s".
+   -b      OS version. Default is to detect on MacOS and Linux.
    -l      Just upgrade GAM to latest version. Skips project creation and auth.
    -p      Profile update (true, false). Should script add gam command to environment. Default is true.
    -u      Admin user email address to use with GAM. Default is to prompt.
@@ -21,18 +22,23 @@ EOF
 target_dir="$HOME/bin"
 gamarch=$(uname -m)
 gamos=$(uname -s)
+osversion=""
 update_profile=true
 upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-while getopts "hd:a:o:lp:u:r:v:" OPTION
+gam_glibc_vers="2.27 2.23"
+gam_macos_vers="10.15.4 10.14.6 10.13.6"
+
+while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
      case $OPTION in
          h) usage; exit;;
          d) target_dir="$OPTARG";;
          a) gamarch="$OPTARG";;
          o) gamos="$OPTARG";;
+         b) osversion="$OPTARG";;
          l) upgrade_only=true;;
          p) update_profile="$OPTARG";;
          u) adminuser="$OPTARG";;
@@ -46,7 +52,7 @@ done
 target_dir=${target_dir%/}
 
 update_profile() {
-	[ -f "$1" ] || return 1
+	[ $2 -eq 1 ] || [ -f "$1" ] || return 1
 
 	grep -F "$alias_line" "$1" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
@@ -75,28 +81,66 @@ echo -e "\x1B[1;33m$1"
 echo -e '\x1B[0m'
 }
 
+version_gt()
+{
+# MacOS < 10.13 doesn't support sort -V
+echo "" | sort -V > /dev/null 2>&1
+vsort_failed=$?
+if [ "${1}" = "${2}" ]; then
+  true
+elif (( $vsort_failed != 0 )); then
+  false
+else
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+fi
+}
+
 case $gamos in
   [lL]inux)
     gamos="linux"
+    if [ "$osversion" == "" ]; then
+      this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    else
+      this_glibc_ver=$osversion
+    fi
+    echo "This Linux distribution uses glibc $this_glibc_ver"
+    useglibc="legacy"
+    for gam_glibc_ver in $gam_glibc_vers; do
+      if version_gt $this_glibc_ver $gam_glibc_ver; then
+        useglibc="glibc$gam_glibc_ver"
+        echo_green "Using GAM compiled against $useglibc"
+        break
+      fi
+    done
     case $gamarch in
-      x86_64) gamfile="linux-x86_64.tar.xz";;
-      i?86) gamfile="linux-i686.tar.xz";;
-      arm*) gamfile="linux-armv7l.tar.xz";;
+      x86_64) gamfile="linux-x86_64-$useglibc.tar.xz";;
+      arm64|aarch64) gamfile="linux-arm64-$useglibc.tar.xz";;
       *)
-        echo_red "ERROR: this installer currently only supports i386, x86_64 and arm Linux. Looks like you're running on $gamarch. Exiting."
+        echo_red "ERROR: this installer currently only supports x86_64 and arm64 Linux. Looks like you're running on $gamarch. Exiting."
         exit
     esac
     ;;
   [Mm]ac[Oo][sS]|[Dd]arwin)
-    osver=$(sw_vers -productVersion | awk -F'.' '{print $2}')
-    if (( $osver < 10 )); then
-      echo_red "ERROR: GAM currently requires MacOS 10.10 or newer. You are running MacOS 10.$osver. Please upgrade." 
-      exit
-    else
-      echo_green "Good, you're running MacOS 10.$osver..."
-    fi
     gamos="macos"
-    gamfile="macos.tar.xz"
+    if [ "$osversion" == "" ]; then
+      this_macos_ver=$(sw_vers -productVersion)
+    else
+      this_macos_ver=$osversion
+    fi
+    echo "You are running MacOS $this_macos_ver"
+    use_macos_ver=""
+    for gam_macos_ver in $gam_macos_vers; do
+      if version_gt $this_macos_ver $gam_macos_ver; then
+        use_macos_ver="MacOS$gam_macos_ver"
+        echo_green "Using GAM compiled on $use_macos_ver"
+        break
+      fi
+    done
+    if [ "$use_macos_ver" == "" ]; then
+      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
+      exit
+    fi
+    gamfile="macos-x86_64-$use_macos_ver.tar.xz"
     ;;
   *)
     echo_red "Sorry, this installer currently only supports Linux and MacOS. Looks like you're runnning on $gamos. Exiting."
@@ -143,11 +187,16 @@ try:
 except KeyError:
   print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
-pycmd="python"
+pycmd="python3"
 $pycmd -V >/dev/null 2>&1
 rc=$?
 if (( $rc != 0 )); then
-  pycmd="python3"
+  pycmd="python"
+fi
+$pycmd -V >/dev/null 2>&1
+rc=$?
+if (( $rc != 0 )); then
+  pycmd="python2"
 fi
 $pycmd -V >/dev/null 2>&1
 rc=$?
@@ -169,6 +218,10 @@ fi
 # Temp dir for archive
 #temp_archive_dir=$(mktemp -d)
 temp_archive_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+
+# Clean up after ourselves even if we are killed with CTRL-C
+trap "rm -rf $temp_archive_dir" EXIT
+
 echo_yellow "Downloading file $name from $browser_download_url to $temp_archive_dir."
 # Save archive to temp w/o losing our path
 (cd $temp_archive_dir && curl -O -L $browser_download_url)
@@ -185,9 +238,23 @@ else
   echo_green "Finished extracting GAM archive."
 fi
 
+# Update profile to add gam command
+if [ "$update_profile" = true ]; then
+  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
+  if [ "$gamos" == "linux" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0
+    update_profile "$HOME/.zshrc" 0
+  elif [ "$gamos" == "macos" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.zshrc" 1
+  fi
+else
+  echo_yellow "skipping profile update."
+fi
+
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
-  "$target_dir/gam/gam" version
+  "$target_dir/gam/gam" version extended
   rc=$?
   if (( $rc != 0 )); then
     echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -198,18 +265,6 @@ if [ "$upgrade_only" = true ]; then
   exit
 fi
 
-# Update profile to add gam command
-if [ "$update_profile" = true ]; then
-  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
-  if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile"
-  elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile"
-  fi
-else
-  echo_yellow "skipping profile update."
-fi
-
 while true; do
   read -p "Can you run a full browser on this machine? (usually Y for MacOS, N for Linux if you SSH into this machine) " yn
   case $yn in
@@ -310,7 +365,7 @@ while $project_created; do
 done
 
 echo_green "Here's information about your new GAM installation:"
-"$target_dir/gam/gam" version
+"$target_dir/gam/gam" version extended
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -321,6 +376,3 @@ echo_green "GAM installation and setup complete!"
 if [ "$update_profile" = true ]; then
   echo_green "Please restart your terminal shell or to get started right away run:\n\n$alias_line"
 fi
-
-# Clean up after ourselves even if we are killed with CTRL-C
-trap "rm -rf $temp_archive_dir" EXIT

src/gam.spec

@@ -0,0 +1,39 @@
+# -*- mode: python -*-
+
+import sys
+
+import importlib
+from PyInstaller.utils.hooks import copy_metadata
+
+sys.modules['FixTk'] = None
+
+# dynamically determine where httplib2/cacerts.txt lives
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+extra_files = [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+
+extra_files += copy_metadata('google-api-python-client')
+
+a = Analysis(['gam/__main__.py'],
+             hiddenimports=[],
+             hookspath=None,
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             datas=extra_files,
+             runtime_hooks=None)
+
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+
+
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam',
+          debug=False,
+          strip=None,
+          upx=False,
+          console=True )

src/gam.wxs

@@ -41,7 +41,7 @@
     <ComponentGroup
         Id="ProductComponents"
         Directory="INSTALLFOLDER"
-        Source="gam-64">
+	Source="dist/gam">
       <Component Id="gam_exe" Guid="886abc07-73c5-4acc-9f71-58daf62aabc1">
         <File Name="gam.exe" KeyPath="yes" />
         <Environment Id="PATH" Name="PATH" Value="[INSTALLFOLDER]" Permanent="yes" Part="last" Action="set" System="yes" />
@@ -49,9 +49,6 @@
       <Component Id="license" Guid="7a15de2e-fb91-4d0a-b8bf-c8b19c68f569">
         <File Name="LICENSE" KeyPath="yes" />
       </Component>
-      <Component Id="whatsnew_txt" Guid="6aa9863c-90d9-412f-9b73-fda82549a950">
-        <File Name="whatsnew.txt" KeyPath="yes" />
-      </Component>
       <Component Id="gam_setup_bat" Guid="ef01f93a-4b50-488a-9c04-ec5e13e66218">
         <File Name="gam-setup.bat" KeyPath="yes" />
       </Component>

src/gam/__main__.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# GAM
+#
+# Copyright 2019, LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GAM is a command line tool which allows Administrators to control their G Suite domain and accounts.
+
+With GAM you can programmatically create users, turn on/off services for users like POP and Forwarding and much more.
+For more information, see https://git.io/gam
+"""
+
+import sys
+from multiprocessing import freeze_support
+from multiprocessing import set_start_method
+
+from gam import controlflow
+import gam
+
+
+def main(argv):
+    freeze_support()
+    if sys.platform == 'darwin':
+        # https://bugs.python.org/issue33725 in Python 3.8.0 seems
+        # to break parallel operations with errors about extra -b
+        # command line arguments
+        set_start_method('fork')
+    if sys.version_info[0] < 3 or sys.version_info[1] < 6:
+        controlflow.system_error_exit(
+            5,
+            f'GAM requires Python 3.6 or newer. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.'
+            % sys.version_info[:3])
+    sys.exit(gam.ProcessGAMCommand(sys.argv))
+
+
+# Run from command line
+if __name__ == '__main__':
+    main(sys.argv)

src/gam/auth/__init__.py

@@ -0,0 +1,46 @@
+"""Authentication/Credentials general purpose and convenience methods."""
+
+import json
+import os
+
+from google.auth.jwt import Credentials as JWTCredentials
+
+from gam.auth import oauth
+from gam.var import _FN_OAUTH2_TXT
+from gam.var import _FN_OAUTH2SERVICE_JSON
+from gam.var import GC_OAUTH2_TXT
+from gam.var import GC_OAUTH2SERVICE_JSON
+from gam.var import GC_ENABLE_DASA
+from gam.var import GC_Values
+
+# TODO: Move logic that determines file name into this module. We should be able
+# to discover the file location without accessing a private member or waiting
+# for a global initialization.
+
+
+def get_admin_credentials_filename():
+    """Gets the name of the file that stores the admin account credentials."""
+    # If the environment globals are loaded, use the set global value. It may have
+    # some custom name in it. Otherwise, just use the default name.
+    if GC_Values[GC_ENABLE_DASA]:
+        return GC_Values[GC_OAUTH2SERVICE_JSON] if GC_Values[GC_OAUTH2SERVICE_JSON] else _FN_OAUTH2SERVICE_JSON
+    else:
+        return GC_Values[GC_OAUTH2_TXT] if GC_Values[GC_OAUTH2_TXT] else _FN_OAUTH2_TXT
+
+
+def get_admin_credentials(api=None):
+    """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+    credential_file = get_admin_credentials_filename()
+    if not os.path.isfile(credential_file):
+        raise oauth.InvalidCredentialsFileError
+    with open(credential_file, 'r') as f:
+        creds_data = json.load(f)
+    # Validate that enable DASA matches content of authorization file
+    if GC_Values[GC_ENABLE_DASA] and 'private_key' in creds_data:
+        audience = f'https://{api}.googleapis.com/'
+        return JWTCredentials.from_service_account_info(creds_data,
+                                                        audience=audience)
+    elif not GC_Values[GC_ENABLE_DASA] and 'token' in creds_data:
+        return oauth.Credentials.from_credentials_file(credential_file)
+    else:
+        raise oauth.InvalidCredentialsFileError

src/gam/auth/oauth.py

@@ -0,0 +1,560 @@
+"""OAuth2.0 user credentials."""
+
+import datetime
+import json
+import os
+import re
+import threading
+from urllib.parse import urlencode
+
+from filelock import FileLock
+import google_auth_oauthlib.flow
+import google.oauth2.credentials
+import google.oauth2.id_token
+
+from gam import fileutils
+from gam import transport
+from gam.var import GM_Globals
+from gam.var import GM_WINDOWS
+from gam import utils
+
+MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
+                                        'browser:\n\n\t{url}\n')
+MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
+MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
+                                             ' visit:\n\n\t{url}\n\nIf your '
+                                             'browser is on a different machine'
+                                             ' then press CTRL+C and create a '
+                                             'file called nobrowser.txt in the '
+                                             'same folder as GAM.\n')
+MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
+                                ' close this browser window and return to GAM.')
+
+
+class CredentialsError(Exception):
+    """Base error class."""
+    pass
+
+
+class InvalidCredentialsFileError(CredentialsError):
+    """Error raised when a file cannot be opened into a credentials object."""
+    pass
+
+
+class EmptyCredentialsFileError(InvalidCredentialsFileError):
+    """Error raised when a credentials file contains no content."""
+    pass
+
+
+class InvalidClientSecretsFileFormatError(CredentialsError):
+    """Error raised when a client secrets file format is invalid."""
+    pass
+
+
+class InvalidClientSecretsFileError(CredentialsError):
+    """Error raised when client secrets file cannot be read."""
+    pass
+
+
+class Credentials(google.oauth2.credentials.Credentials):
+    """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
+
+    DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+    def __init__(self,
+                 token,
+                 refresh_token=None,
+                 id_token=None,
+                 token_uri=None,
+                 client_id=None,
+                 client_secret=None,
+                 scopes=None,
+                 quota_project_id=None,
+                 expiry=None,
+                 id_token_data=None,
+                 filename=None):
+        """A thread-safe OAuth2.0 credentials object.
+
+    Credentials adds additional utility properties and methods to a
+    standard OAuth2.0 credentials object. When used to store credentials on
+    disk, it implements a file lock to avoid collision during writes.
+
+    Args:
+      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
+        information is provided.
+      refresh_token: String, The OAuth 2.0 refresh token. If specified,
+        credentials can be refreshed.
+      id_token: String, The Open ID Connect ID Token.
+      token_uri: String, The OAuth 2.0 authorization server's token endpoint
+        URI. Must be specified for refresh, can be left as None if the token can
+        not be refreshed.
+      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
+        can be left as None if the token can not be refreshed.
+      client_secret: String, The OAuth 2.0 client secret. Must be specified for
+        refresh, can be left as None if the token can not be refreshed.
+      scopes: Sequence[str], The scopes used to obtain authorization.
+        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
+          not request additional scopes after authorization. The scopes must be
+          derivable from the refresh token if refresh information is provided
+          (e.g. The refresh token scopes are a superset of this or contain a
+          wild card scope like
+            'https://www.googleapis.com/auth/any-api').
+      quota_project_id: String, The project ID used for quota and billing. This
+        project may be different from the project used to create the
+        credentials.
+      expiry: datetime.datetime, The time at which the provided token will
+        expire.
+      id_token_data: Oauth2.0 ID Token data which was previously fetched for
+        this access token against the google.oauth2.id_token library.
+      filename: String, Path to a file that will be used to store the
+        credentials. If provided, a lock file of the same name and a ".lock"
+          extension will be created for concurrency controls. Note: New
+            credentials are not saved to disk until write() or refresh() are
+            called.
+
+    Raises:
+      TypeError: If id_token_data is not the required dict type.
+    """
+        super(Credentials, self).__init__(token=token,
+                                          refresh_token=refresh_token,
+                                          id_token=id_token,
+                                          token_uri=token_uri,
+                                          client_id=client_id,
+                                          client_secret=client_secret,
+                                          scopes=scopes,
+                                          quota_project_id=quota_project_id)
+
+        # Load data not restored by the super class
+        self.expiry = expiry
+        if id_token_data and not isinstance(id_token_data, dict):
+            raise TypeError(f'Expected type id_token_data dict but received '
+                            f'{type(id_token_data)}')
+        self._id_token_data = id_token_data.copy() if id_token_data else None
+
+        # If a filename is provided, use a lock file to control concurrent access
+        # to the resource. If no filename is provided, use a thread lock that has
+        # the same interface as FileLock in order to simplify the implementation.
+        if filename:
+            # Convert relative paths into absolute
+            self._filename = os.path.abspath(filename)
+            lock_file = os.path.abspath(f'{self._filename}.lock')
+            self._lock = FileLock(lock_file)
+        else:
+            self._filename = None
+            self._lock = _FileLikeThreadLock()
+
+    # Use a property to prevent external mutation of the filename.
+    @property
+    def filename(self):
+        return self._filename
+
+    @classmethod
+    def from_authorized_user_info(cls, info, filename=None):
+        """Generates Credentials from JSON containing authorized user info.
+
+    Args:
+      info: Dict, authorized user info in Google format.
+      filename: String, the filename used to store these credentials on disk. If
+        no filename is provided, the credentials will not be saved to disk.
+
+    Raises:
+      ValueError: If missing fields are detected in the info.
+    """
+        # We need all of these keys
+        keys_needed = set(('client_id', 'client_secret'))
+        # We need 1 or more of these keys
+        keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
+        missing = keys_needed.difference(info.keys())
+        has_one_of = set(info) & keys_need_one_of
+        if missing or not has_one_of:
+            raise ValueError(
+                'Authorized user info was not in the expected format, missing '
+                f'fields {", ".join(missing)} and one of '
+                f'{", ".join(keys_need_one_of)}.')
+
+        expiry = info.get('token_expiry')
+        if expiry:
+            # Convert the raw expiry to datetime
+            expiry = datetime.datetime.strptime(expiry,
+                                                Credentials.DATETIME_FORMAT)
+        id_token_data = info.get('decoded_id_token')
+
+        # Provide backwards compatibility with field names when loading from JSON.
+        # Some field names may be different, depending on when/how the credentials
+        # were pickled.
+        return cls(token=info.get('token', info.get('auth_token', '')),
+                   refresh_token=info.get('refresh_token', ''),
+                   id_token=info.get('id_token_jwt', info.get('id_token')),
+                   token_uri=info.get('token_uri'),
+                   client_id=info['client_id'],
+                   client_secret=info['client_secret'],
+                   scopes=info.get('scopes'),
+                   quota_project_id=info.get('quota_project_id'),
+                   expiry=expiry,
+                   id_token_data=id_token_data,
+                   filename=filename)
+
+    @classmethod
+    def from_google_oauth2_credentials(cls, credentials, filename=None):
+        """Generates Credentials from a google.oauth2.Credentials object."""
+        info = json.loads(credentials.to_json())
+        # Add properties which are not exported with the native to_json() output.
+        info['id_token'] = credentials.id_token
+        if credentials.expiry:
+            info['token_expiry'] = credentials.expiry.strftime(
+                Credentials.DATETIME_FORMAT)
+        info['quota_project_id'] = credentials.quota_project_id
+
+        return cls.from_authorized_user_info(info, filename=filename)
+
+    @classmethod
+    def from_credentials_file(cls, filename):
+        """Generates Credentials from a stored Credentials file.
+
+    The same file will be used to save the credentials when the access token is
+    refreshed.
+
+    Args:
+      filename: String, the name of a file containing JSON credentials to load.
+        The same filename will be used to save credentials back to disk.
+
+    Returns:
+      The credentials loaded from disk.
+
+    Raises:
+      InvalidCredentialsFileError: When the credentials file cannot be opened.
+      EmptyCredentialsFileError: When the provided file contains no credentials.
+    """
+        file_content = fileutils.read_file(filename,
+                                           continue_on_error=True,
+                                           display_errors=False)
+        if file_content is None:
+            raise InvalidCredentialsFileError(
+                f'File {filename} could not be opened')
+        info = json.loads(file_content)
+        if not info:
+            raise EmptyCredentialsFileError(
+                f'File {filename} contains no credential data')
+
+        try:
+            # We read the existing data from the passed in file, but we also want to
+            # save future data/tokens in the same place.
+            return cls.from_authorized_user_info(info, filename=filename)
+        except ValueError as e:
+            raise InvalidCredentialsFileError(str(e))
+
+    @classmethod
+    def from_client_secrets(cls,
+                            client_id,
+                            client_secret,
+                            scopes,
+                            access_type='offline',
+                            login_hint=None,
+                            filename=None,
+                            use_console_flow=False):
+        """Runs an OAuth Flow from client secrets to generate credentials.
+
+    Args:
+      client_id: String, The OAuth2.0 Client ID.
+      client_secret: String, The OAuth2.0 Client Secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      filename: String, the path to a file to use to save the credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Returns:
+      Credentials
+    """
+        client_config = {
+            'installed': {
+                'client_id': client_id,
+                'client_secret': client_secret,
+                'redirect_uris': [
+                    'http://localhost', 'urn:ietf:wg:oauth:2.0:oob'
+                ],
+                'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+                'token_uri': 'https://oauth2.googleapis.com/token',
+            }
+        }
+
+        flow = _ShortURLFlow.from_client_config(client_config,
+                                                scopes,
+                                                autogenerate_code_verifier=True)
+        flow_kwargs = {'access_type': access_type}
+        if login_hint:
+            flow_kwargs['login_hint'] = login_hint
+
+        # TODO: Move code for browser detection somewhere in this file so that the
+        # messaging about `nobrowser.txt` is co-located with the logic that uses it.
+        if use_console_flow:
+            flow.run_console(
+                authorization_prompt_message=
+                MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
+                authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
+                **flow_kwargs)
+        else:
+            flow.run_local_server(authorization_prompt_message=
+                                  MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
+                                  success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
+                                  **flow_kwargs)
+        return cls.from_google_oauth2_credentials(flow.credentials,
+                                                  filename=filename)
+
+    @classmethod
+    def from_client_secrets_file(cls,
+                                 client_secrets_file,
+                                 scopes,
+                                 access_type='offline',
+                                 login_hint=None,
+                                 credentials_file=None,
+                                 use_console_flow=False):
+        """Runs an OAuth Flow from secrets stored on disk to generate credentials.
+
+    Args:
+      client_secrets_file: String, path to a file containing a client ID and
+        secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      credentials_file: String, the path to a file to use to save the
+        credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Raises:
+      InvalidClientSecretsFileError: If the client secrets file cannot be
+        opened.
+      InvalidClientSecretsFileFormatError: If the client secrets file does not
+        contain the required data or the data is malformed.
+
+    Returns:
+      Credentials
+    """
+        cs_data = fileutils.read_file(client_secrets_file,
+                                      continue_on_error=True,
+                                      display_errors=False)
+        if not cs_data:
+            raise InvalidClientSecretsFileError(
+                f'File {client_secrets_file} could not be opened')
+        try:
+            cs_json = json.loads(cs_data)
+            client_id = cs_json['installed']['client_id']
+            # Chop off .apps.googleusercontent.com suffix as it's not needed
+            # and we need to keep things short for the Auth URL.
+            client_id = re.sub(r'\.apps\.googleusercontent\.com$', '',
+                               client_id)
+            client_secret = cs_json['installed']['client_secret']
+        except (ValueError, IndexError, KeyError):
+            raise InvalidClientSecretsFileFormatError(
+                f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
+            )
+
+        return cls.from_client_secrets(client_id,
+                                       client_secret,
+                                       scopes,
+                                       access_type=access_type,
+                                       login_hint=login_hint,
+                                       filename=credentials_file,
+                                       use_console_flow=use_console_flow)
+
+    def _fetch_id_token_data(self):
+        """Fetches verification details from Google for the OAuth2.0 token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Raises:
+      CredentialsError: If no id_token is present.
+    """
+        if not self.id_token:
+            raise CredentialsError(
+                'Failed to fetch token data. No id_token present.')
+
+        request = transport.create_request()
+        if self.expired:
+            # The id_token needs to be unexpired, in order to request data about it.
+            self.refresh(request)
+
+        self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
+            self.id_token, request)
+
+    def get_token_value(self, field):
+        """Retrieves data from the OAuth ID token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Args:
+      field: The name of the key/field to fetch
+
+    Returns:
+      The value associated with the given key or 'Unknown' if the key data can
+      not be found in the access token data.
+    """
+        if not self._id_token_data:
+            self._fetch_id_token_data()
+        # Maintain legacy GAM behavior here to return "Unknown" if the field is
+        # otherwise unpopulated.
+        return self._id_token_data.get(field, 'Unknown')
+
+    def to_json(self, strip=None):
+        """Creates a JSON representation of a Credentials.
+
+    Args:
+        strip: Sequence[str], Optional list of members to exclude from the
+          generated JSON.
+
+    Returns:
+        str: A JSON representation of this instance, suitable to pass to
+             from_json().
+    """
+        expiry = self.expiry.strftime(
+            Credentials.DATETIME_FORMAT) if self.expiry else None
+        prep = {
+            'token': self.token,
+            'refresh_token': self.refresh_token,
+            'token_uri': self.token_uri,
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'id_token': self.id_token,
+            # Google auth doesn't currently give us scopes back on refresh.
+            # 'scopes': sorted(self.scopes),
+            'token_expiry': expiry,
+            'decoded_id_token': self._id_token_data,
+        }
+
+        # Remove empty entries
+        prep = {k: v for k, v in prep.items() if v is not None}
+
+        # Remove entries that explicitly need to be removed
+        if strip is not None:
+            prep = {k: v for k, v in prep.items() if k not in strip}
+
+        return json.dumps(prep, indent=2, sort_keys=True)
+
+    def refresh(self, request=None):
+        """Refreshes the credential's access token.
+
+    Args:
+      request: google.auth.transport.Request, The object used to make HTTP
+        requests. If not provided, a default request will be used.
+
+    Raises:
+      google.auth.exceptions.RefreshError: If the credentials could not be
+      refreshed.
+    """
+        with self._lock:
+            if request is None:
+                request = transport.create_request()
+            self._locked_refresh(request)
+            # Save the new tokens back to disk, if these credentials are disk-backed.
+            if self._filename:
+                self._locked_write()
+
+    def _locked_refresh(self, request):
+        """Refreshes the credential's access token while the file lock is held."""
+        assert self._lock.is_locked
+        super(Credentials, self).refresh(request)
+
+    def write(self):
+        """Writes credentials to disk."""
+        with self._lock:
+            self._locked_write()
+
+    def _locked_write(self):
+        """Writes credentials to disk while the file lock is held."""
+        assert self._lock.is_locked
+        if not self.filename:
+            # If no filename was provided to the constructor, these credentials cannot
+            # be saved to disk.
+            raise CredentialsError(
+                'The credentials have no associated filename and cannot be saved '
+                'to disk.')
+        fileutils.write_file(self._filename, self.to_json())
+
+    def delete(self):
+        """Deletes all files on disk related to these credentials."""
+        with self._lock:
+            # Only attempt to remove the file if the lock we're using is a FileLock.
+            if isinstance(self._lock, FileLock):
+                os.remove(self._filename)
+                if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
+                    os.remove(self._lock.lock_file)
+
+    _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+
+    def revoke(self, http=None):
+        """Revokes this credential's access token with the server.
+
+    Args:
+      http: httplib2.Http compatible object for use as a transport. If no http
+        is provided, a default will be used.
+    """
+        with self._lock:
+            if http is None:
+                http = transport.create_http()
+            params = urlencode({'token': self.refresh_token})
+            revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
+            http.request(revoke_uri, 'GET')
+
+
+class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
+    """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
+
+    URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+    def authorization_url(self, http=None, **kwargs):
+        """Gets a shortened authorization URL."""
+        long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
+        short_url = utils.shorten_url(long_url)
+        return short_url, state
+
+
+class _FileLikeThreadLock(object):
+    """A threading.lock which has the same interface as filelock.Filelock."""
+
+    def __init__(self):
+        """A shell object that holds a threading.Lock.
+
+    Since we cannot inherit from built-in classes such as threading.Lock, we
+    just use a shell object and maintain a lock inside of it.
+    """
+        self._lock = threading.Lock()
+
+    def __enter__(self, *args, **kwargs):
+        return self._lock.__enter__(*args, **kwargs)
+
+    def __exit__(self, *args, **kwargs):
+        return self._lock.__exit__(*args, **kwargs)
+
+    def acquire(self, **kwargs):
+        return self._lock.acquire(**kwargs)
+
+    def release(self):
+        return self._lock.release()
+
+    @property
+    def is_locked(self):
+        return self._lock.locked()
+
+    @property
+    def lock_file(self):
+        return None

src/gam/auth/oauth_test.py

@@ -0,0 +1,697 @@
+"""Tests for oauth."""
+
+import datetime
+import json
+import os
+import platform
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import google.oauth2.credentials
+
+from gam.auth import oauth
+
+
+class CredentialsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_token = 'fake_token'
+        self.fake_refresh_token = 'fake_refresh_token'
+        self.fake_id_token = 'fake_id_token'
+        self.fake_token_uri = 'https://fake.token.uri'
+        self.fake_client_id = 'fake_client_id'
+        self.fake_client_secret = 'fake_client_secret'
+        self.fake_scopes = [
+            'fake_api.readonly',
+            'fake_other_api.write',
+        ]
+        self.fake_quota_project_id = 'fake_quota_project_id'
+        self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
+        self.fake_filename = 'fake_filename'
+        self.fake_token_data = {
+            'field': 'value',
+            'another-field': 'another-value',
+        }
+        self.info_with_only_required_fields = {
+            'refresh_token': self.fake_refresh_token,
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+        }
+        super(CredentialsTest, self).setUp()
+
+    def tearDown(self):
+        # Remove any credential files that may have been created.
+        if os.path.exists(self.fake_filename):
+            os.remove(self.fake_filename)
+        if os.path.exists('%s.lock' % self.fake_filename):
+            os.remove('%s.lock' % self.fake_filename)
+        super(CredentialsTest, self).tearDown()
+
+    def test_from_authorized_user_info_only_required_info(self):
+        creds = oauth.Credentials.from_authorized_user_info(
+            self.info_with_only_required_fields)
+        self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+        self.assertEqual(self.fake_client_id, creds.client_id)
+        self.assertEqual(self.fake_client_secret, creds.client_secret)
+        self.assertIsNone(creds.id_token)
+        self.assertIsNone(creds.expiry)
+        self.assertIsNone(creds.filename)
+
+    def test_from_authorized_user_info_all_info_provided(self):
+        info = {
+            'token':
+                self.fake_token,
+            'refresh_token':
+                self.fake_refresh_token,
+            'id_token':
+                self.fake_id_token,
+            'token_uri':
+                self.fake_token_uri,
+            'client_id':
+                self.fake_client_id,
+            'client_secret':
+                self.fake_client_secret,
+            'token_expiry':
+                self.fake_token_expiry.strftime(
+                    oauth.Credentials.DATETIME_FORMAT),
+            'id_token_data':
+                self.fake_token_data,
+        }
+        creds = oauth.Credentials.from_authorized_user_info(info)
+        self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+        self.assertEqual(self.fake_client_id, creds.client_id)
+        self.assertEqual(self.fake_client_secret, creds.client_secret)
+        self.assertEqual(self.fake_id_token, creds.id_token)
+        self.assertEqual(self.fake_token_uri, creds.token_uri)
+        self.assertEqual(self.fake_token_expiry, creds.expiry)
+        self.assertIsNone(creds.filename)
+
+    def test_from_authorized_user_info_missing_required_info(self):
+        info_with_missing_fields = {'token': self.fake_token}
+        with self.assertRaises(ValueError):
+            oauth.Credentials.from_authorized_user_info(
+                info_with_missing_fields)
+
+    def test_from_authorized_user_info_no_expiry_in_info(self):
+        info_with_no_token_expiry = self.info_with_only_required_fields.copy()
+        self.assertIsNone(info_with_no_token_expiry.get('expiry'))
+        creds = oauth.Credentials.from_authorized_user_info(
+            info_with_no_token_expiry)
+        self.assertIsNone(creds.expiry)
+
+    def test_init_saves_filename(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+    def test_init_loads_decoded_id_token_data(self, mock_verify_token):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=self.fake_token_data)
+        self.assertEqual(self.fake_token_data.get('field'),
+                         creds.get_token_value('field'))
+        # Verify the fetching method was not called, since the token
+        # data was supposed to be loaded from the passed in info.
+        self.assertEqual(mock_verify_token.call_count, 0)
+
+    def test_credentials_uses_file_lock_when_filename_provided(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        self.assertIsInstance(creds._lock, oauth.FileLock)
+        self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
+
+    def test_credentials_uses_thread_lock_when_filename_not_provided(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=None)
+        self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
+        self.assertIsNone(creds.filename)
+
+    def test_from_oauth2credentials(self):
+        google_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        creds = oauth.Credentials.from_google_oauth2_credentials(
+            google_creds, filename=self.fake_filename)
+        self.assertEqual(google_creds.token, creds.token)
+        self.assertEqual(google_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(google_creds.client_id, creds.client_id)
+        self.assertEqual(google_creds.client_secret, creds.client_secret)
+        self.assertEqual(google_creds.id_token, creds.id_token)
+        self.assertEqual(google_creds.expiry, creds.expiry)
+        self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+        self.assertIn('could not be opened', str(e.exception))
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_no_serialized_data_in_file_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({})
+        with self.assertRaises(oauth.EmptyCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_missing_any_token_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({
+            # This data is missing a token key/value pair
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+        })
+        with self.assertRaises(oauth.InvalidCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_missing_required_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({
+            # This data is missing a client_secret key/value pair
+            'client_id': self.fake_client_id,
+            'refresh_token': self.fake_refresh_token,
+        })
+        with self.assertRaises(oauth.InvalidCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_console_flow(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                      self.fake_client_secret,
+                                                      self.fake_scopes,
+                                                      use_console_flow=True)
+        self.assertTrue(mock_flow.return_value.run_console.called)
+        self.assertFalse(mock_flow.return_value.run_local_server.called)
+        self.assertEqual(flow_creds.token, creds.token)
+        self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(flow_creds.client_id, creds.client_id)
+        self.assertEqual(flow_creds.client_secret, creds.client_secret)
+        self.assertEqual(flow_creds.id_token, creds.id_token)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_local_server_flow(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                      self.fake_client_secret,
+                                                      self.fake_scopes,
+                                                      use_console_flow=False)
+        self.assertFalse(mock_flow.return_value.run_console.called)
+        self.assertTrue(mock_flow.return_value.run_local_server.called)
+        self.assertEqual(flow_creds.token, creds.token)
+        self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(flow_creds.client_id, creds.client_id)
+        self.assertEqual(flow_creds.client_secret, creds.client_secret)
+        self.assertEqual(flow_creds.id_token, creds.id_token)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_uses_login_hint(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                              self.fake_client_secret,
+                                              self.fake_scopes,
+                                              login_hint='someone@domain.com')
+
+        run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
+        self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
+
+    def test_from_client_secrets_uses_shortened_url_flow(self):
+        with patch.object(oauth._ShortURLFlow,
+                          'from_client_config') as mock_flow:
+            flow_creds = google.oauth2.credentials.Credentials(
+                token=self.fake_token,
+                refresh_token=self.fake_refresh_token,
+                client_id=self.fake_client_id,
+                client_secret=self.fake_client_secret,
+                id_token=self.fake_id_token)
+            mock_flow.return_value.credentials = flow_creds
+            oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                  self.fake_client_secret,
+                                                  self.fake_scopes)
+        self.assertTrue(mock_flow.called)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(
+            self.fake_client_id,
+            self.fake_client_secret,
+            self.fake_scopes,
+            filename=self.fake_filename)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(
+        self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        with self.assertRaises(oauth.InvalidClientSecretsFileError):
+            oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                       self.fake_scopes)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_client_secrets_file_missing_required_json_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({})
+        with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
+            oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                       self.fake_scopes)
+        self.assertIn('Could not extract Client ID or Client Secret',
+                      str(e.exception))
+
+    @patch.object(oauth.Credentials, 'from_client_secrets')
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_client_secrets_file_strips_domain_from_client_id(
+        self, mock_read_file, mock_creds_from_client_secrets):
+        mock_read_file.return_value = json.dumps({
+            'installed': {
+                'client_id':
+                    self.fake_client_id + '.apps.googleusercontent.com',
+                'client_secret':
+                    self.fake_client_secret,
+            }
+        })
+
+        oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                   self.fake_scopes)
+        self.assertEqual(self.fake_client_id,
+                         mock_creds_from_client_secrets.call_args[0][0])
+
+    def test_get_token_value_known_token_field(self):
+        token_data = {'known-field': 'known-value'}
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token_data=token_data)
+        self.assertEqual('known-value', creds.get_token_value('known-field'))
+
+    def test_get_token_value_unknown_field_returns_unknown(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token_data=self.fake_token_data)
+        self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
+
+    @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+    def test_get_token_value_credentials_expired(self,
+                                                 mock_verify_oauth2_token):
+        mock_verify_oauth2_token.return_value = {
+            'fetched-field': 'fetched-value'
+        }
+        time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
+            minutes=5)
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  expiry=time_earlier_than_now,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=None)
+        self.assertTrue(creds.expired)
+        creds.refresh = MagicMock()
+
+        token_value = creds.get_token_value('fetched-field')
+
+        self.assertEqual('fetched-value', token_value)
+        self.assertTrue(creds.refresh.called)
+
+    def test_to_json_contains_all_required_fields(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  refresh_token=self.fake_refresh_token,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=self.fake_token_data,
+                                  token_uri=self.fake_token_uri,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  scopes=self.fake_scopes,
+                                  quota_project_id=self.fake_quota_project_id,
+                                  expiry=self.fake_token_expiry)
+        json_string = creds.to_json()
+        json_data = json.loads(json_string)
+        keys = json_data.keys()
+        self.assertIn('token', keys)
+        self.assertEqual(self.fake_token, json_data['token'])
+        self.assertIn('refresh_token', keys)
+        self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
+        self.assertIn('id_token', keys)
+        self.assertEqual(self.fake_id_token, json_data['id_token'])
+        self.assertIn('token_uri', keys)
+        self.assertEqual(self.fake_token_uri, json_data['token_uri'])
+        self.assertIn('client_id', keys)
+        self.assertEqual(self.fake_client_id, json_data['client_id'])
+        self.assertIn('client_secret', keys)
+        self.assertEqual(self.fake_client_secret, json_data['client_secret'])
+        self.assertNotIn('scopes', keys)  # Scopes are not currently saved
+        self.assertIn('token_expiry', keys)
+        self.assertEqual(
+            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+            json_data['token_expiry'])
+        self.assertIn('decoded_id_token', keys)
+        self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
+
+    def test_credentials_to_json_and_back(self):
+        original_creds = oauth.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            id_token=self.fake_id_token,
+            id_token_data=self.fake_token_data,
+            token_uri=self.fake_token_uri,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            scopes=self.fake_scopes,
+            quota_project_id=self.fake_quota_project_id,
+            expiry=self.fake_token_expiry)
+        pickled_creds = original_creds.to_json()
+        serialized_json = json.loads(pickled_creds)
+        unpickled_creds = oauth.Credentials.from_authorized_user_info(
+            serialized_json)
+        self.assertEqual(original_creds.token, unpickled_creds.token)
+        self.assertEqual(original_creds.refresh_token,
+                         unpickled_creds.refresh_token)
+        self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
+        self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
+        self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
+        self.assertEqual(original_creds.client_secret,
+                         unpickled_creds.client_secret)
+        self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    def test_refresh_calls_super_refresh(self, mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        request = MagicMock()
+
+        creds.refresh(request)
+        self.assertTrue(mock_super_refresh.called)
+        self.assertEqual(request, mock_super_refresh.call_args[0][0])
+
+    def test_refresh_locks_resource_during_refresh(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        lock = creds._lock
+
+        def check_lock_is_locked(*unused_args, **unused_kwargs):
+            self.assertTrue(lock.is_locked)
+
+        # We need to mock the superclass refresh so it doesn't actually try to
+        # refresh our fake token.
+        # At the same time, we'll make sure the lock is held during the refresh.
+        with patch.object(oauth.google.oauth2.credentials.Credentials,
+                          'refresh') as mock_refresh:
+            mock_refresh.side_effect = check_lock_is_locked
+            creds.refresh(request=MagicMock())
+
+        # Make sure our side effect was actually performed.
+        self.assertTrue(mock_refresh.called)
+        # The lock should be released after refresh
+        self.assertFalse(lock.is_locked)
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    @patch.object(oauth.fileutils, 'write_file')
+    def test_refresh_writes_new_credentials_to_disk_after_refresh(
+        self, mock_write_file, mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+
+        def update_access_token(unused_request):
+            creds.token = 'refreshed_access_token'
+
+        mock_super_refresh.side_effect = update_access_token
+
+        self.assertIsNone(creds.token)
+        creds.refresh(request=MagicMock())
+        self.assertEqual('refreshed_access_token', creds.token,
+                         'Access token was not refreshed')
+        text_written_to_file = mock_write_file.call_args[0][1]
+        self.assertIsNotNone(text_written_to_file,
+                             'Nothing was written to file')
+        saved_json = json.loads(text_written_to_file)
+        self.assertEqual('refreshed_access_token', saved_json['token'],
+                         'Refreshed access token was not saved to disk')
+
+    def test_write_writes_credentials_to_disk(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+
+        self.assertFalse(os.path.exists(self.fake_filename))
+        creds.write()
+        self.assertTrue(os.path.exists(self.fake_filename))
+
+    def test_write_raises_error_when_no_credentials_file_is_set(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+
+        self.assertIsNone(creds.filename)
+        with self.assertRaises(oauth.CredentialsError):
+            creds.write()
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    @patch.object(oauth.fileutils, 'write_file')
+    def test_write_locks_resource_during_write(self, mock_write_file,
+                                               unused_mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        lock = creds._lock
+
+        def check_lock_is_locked(*unused_args, **unused_kwargs):
+            self.assertTrue(creds._lock.is_locked)
+
+        mock_write_file.side_effect = check_lock_is_locked
+
+        self.assertFalse(lock.is_locked)
+        creds.refresh(request=MagicMock())
+        self.assertFalse(lock.is_locked)
+        self.assertTrue(mock_write_file.called)
+
+    def test_delete_removes_credentials_file(self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        creds.write()
+        self.assertTrue(os.path.exists(self.fake_filename))
+        creds.delete()
+        self.assertFalse(os.path.exists(self.fake_filename))
+
+    @unittest.skipIf(
+        platform.system() == 'Windows',
+        reason=('On Windows, Filelock deletes the lock file each time the lock '
+                'is released. Delete does not remove it.'))
+    def test_delete_removes_lock_file(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        lock_file = '%s.lock' % creds.filename
+        creds.write()
+        self.assertTrue(os.path.exists(lock_file))
+        creds.delete()
+        self.assertFalse(os.path.exists(lock_file))
+
+    def test_delete_is_noop_when_not_using_filelock(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        self.assertIsNone(creds.filename)
+        creds.delete()  # This should not raise an exception.
+
+    def test_revoke_requests_credential_revoke(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        mock_http = MagicMock()
+
+        creds.revoke(http=mock_http)
+
+        uri = mock_http.request.call_args[0][0]
+        self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
+        params = uri[uri.index('?'):]
+        self.assertIn('token=%s' % creds.refresh_token, params)
+        self.assertEqual('GET', mock_http.request.call_args[0][1])
+
+
+class ShortUrlFlowTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_client_id = 'fake_client_id'
+        self.fake_client_secret = 'fake_client_secret'
+        self.fake_scopes = [
+            'fake_api.readonly',
+            'fake_other_api.write',
+        ]
+        self.fake_client_config = {
+            'installed': {
+                'client_id': self.fake_client_id,
+                'client_secret': self.fake_client_secret,
+                'redirect_uris': [
+                    'http://localhost', 'urn:ietf:wg:oauth:2.0:oob'
+                ],
+                'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+                'token_uri': 'https://oauth2.googleapis.com/token',
+            }
+        }
+        self.long_url = 'http://example.com/some/long/url'
+        self.short_url = 'http://ex.co/short'
+        super(ShortUrlFlowTest, self).setUp()
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = json.dumps({'short_url': self.short_url})
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.short_url, url)
+        self.assertEqual('fake_state', state)
+
+        # Verify request() was called with the expected arguments.
+        self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
+                         mock_http.request.call_args[0][0])
+        self.assertEqual('POST', mock_http.request.call_args[0][1])
+        self.assertIn(self.long_url, mock_http.request.call_args[0][2])
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_request_error(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_http.request.side_effect = Exception()
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 404  # Use a status that is not 200
+        content = json.dumps({'short_url': self.short_url})
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = None
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = json.dumps(
+            {})  # This json content contains no "short-url" key
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/controlflow.py

@@ -0,0 +1,96 @@
+"""Methods related to the central control flow of an application."""
+import random
+import sys
+import time
+
+from gam import display
+from gam.var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from gam.var import MESSAGE_INVALID_JSON
+
+
+def system_error_exit(return_code, message):
+    """Raises a system exit with the given return code and message.
+
+  Args:
+    return_code: Int, the return code to yield when the system exits.
+    message: An error message to print before the system exits.
+  """
+    if message:
+        display.print_error(message)
+    sys.exit(return_code)
+
+
+def invalid_argument_exit(argument, command):
+    """Indicate that the argument is not valid for the command.
+
+  Args:
+    argument: the invalid argument
+    command: the base GAM command
+  """
+    system_error_exit(2, f'{argument} is not a valid argument for "{command}"')
+
+
+def missing_argument_exit(argument, command):
+    """Indicate that the argument is missing for the command.
+
+  Args:
+    argument: the missingagrument
+    command: the base GAM command
+  """
+    system_error_exit(2, f'missing argument {argument} for "{command}"')
+
+
+def expected_argument_exit(name, expected, argument):
+    """Indicate that the argument does not have an expected value for the command.
+
+  Args:
+    name: the field name
+    expected: the expected values
+    argument: the invalid argument
+  """
+    system_error_exit(2, f'{name} must be one of {expected}; got {argument}')
+
+
+def csv_field_error_exit(field_name, field_names):
+    """Raises a system exit when a CSV field is malformed.
+
+  Args:
+    field_name: The CSV field name for which a header does not exist in the
+      existing CSV headers.
+    field_names: The known list of CSV headers.
+  """
+    system_error_exit(
+        2,
+        MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                       ','.join(field_names)))
+
+
+def invalid_json_exit(file_name):
+    """Raises a sysyem exit when invalid JSON content is encountered."""
+    system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+
+
+def wait_on_failure(current_attempt_num,
+                    total_num_retries,
+                    error_message,
+                    error_print_threshold=3):
+    """Executes an exponential backoff-style system sleep.
+
+  Args:
+    current_attempt_num: Int, the current number of retries.
+    total_num_retries: Int, the total number of times the current action will be
+      retried.
+    error_message: String, a message to be displayed that will give more context
+      around why the action is being retried.
+    error_print_threshold: Int, the number of attempts which will have their
+      error messages suppressed. Any current_attempt_num greater than
+      error_print_threshold will print the prescribed error.
+  """
+    wait_on_fail = min(2**current_attempt_num,
+                       60) + float(random.randint(1, 1000)) / 1000
+    if current_attempt_num > error_print_threshold:
+        sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+                          f'{int(wait_on_fail)} seconds, Retry: '
+                          f'{current_attempt_num}/{total_num_retries}\n'))
+        sys.stderr.flush()
+    time.sleep(wait_on_fail)

src/gam/controlflow_test.py

@@ -0,0 +1,108 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+from gam import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+    def test_system_error_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.system_error_exit(1, 'exit message')
+
+    def test_system_error_exit_raises_systemexit_with_return_code(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.system_error_exit(100, 'exit message')
+        self.assertEqual(context_manager.exception.code, 100)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_system_error_exit_prints_error_before_exiting(
+        self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.system_error_exit(100, 'exit message')
+        self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+    def test_csv_field_error_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+
+    def test_csv_field_error_exit_exits_code_2(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+        self.assertEqual(context_manager.exception.code, 2)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+        printed_message = mock_print_err.call_args[0][0]
+        self.assertIn('aField', printed_message)
+        self.assertIn('unusedField1', printed_message)
+        self.assertIn('unusedField2', printed_message)
+
+    def test_invalid_json_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.invalid_json_exit('filename')
+
+    def test_invalid_json_exit_exit_exits_code_17(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.invalid_json_exit('filename')
+        self.assertEqual(context_manager.exception.code, 17)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.invalid_json_exit('filename')
+        printed_message = mock_print_err.call_args[0][0]
+        self.assertIn('filename', printed_message)
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+        controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+        controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+        controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+        sleep_calls = mock_sleep.call_args_list
+        self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+        self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+        self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+        total_attempts = 20
+        for attempt in range(1, total_attempts + 1):
+            controlflow.wait_on_failure(
+                attempt,
+                total_attempts,
+                'Attempt #%s' % attempt,
+                # Suppress messages while we make a lot of attempts.
+                error_print_threshold=total_attempts + 1)
+            # Wait time may be between 60 and 61 secs, due to rand addition.
+            self.assertLessEqual(mock_sleep.call_args[0][0], 61)
+
+    # Prevent the system from actually sleeping and thus slowing down the test.
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
+        message = 'An error message to display'
+        with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+            controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+        self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_only_prints_after_threshold(
+        self, unused_mock_sleep):
+        total_attempts = 5
+        threshold = 3
+        with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+            for attempt in range(1, total_attempts + 1):
+                controlflow.wait_on_failure(attempt,
+                                            total_attempts,
+                                            'Attempt #%s' % attempt,
+                                            error_print_threshold=threshold)
+        self.assertEqual(total_attempts - threshold,
+                         mock_stderr_write.call_count)

src/gam/display.py

@@ -0,0 +1,297 @@
+"""Methods related to display of information to the user."""
+
+import csv
+import datetime
+import io
+import sys
+import webbrowser
+
+import dateutil
+import googleapiclient.http
+
+#TODO: get rid of these hacks
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+
+
+def current_count(i, count):
+    return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
+
+
+def current_count_nl(i, count):
+    return f' ({i}/{count})\n' if (
+        count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
+
+
+def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
+    fields = fieldsChoiceMap[fieldName.lower()]
+    if isinstance(fields, list):
+        fieldsList.extend(fields)
+    else:
+        fieldsList.append(fields)
+
+
+# Write a CSV file
+def add_titles_to_csv_file(addTitles, titles):
+    for title in addTitles:
+        if title not in titles:
+            titles.append(title)
+
+
+def add_row_titles_to_csv_file(row, csvRows, titles):
+    csvRows.append(row)
+    for title in row:
+        if title not in titles:
+            titles.append(title)
+
+
+# fieldName is command line argument
+# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
+#ARGUMENT_TO_PROPERTY_MAP = {
+#  u'admincreated': [u'adminCreated'],
+#  u'aliases': [u'aliases', u'nonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles,
+                          titles):
+    for ftList in fieldNameMap[fieldName]:
+        if ftList not in fieldsTitles:
+            fieldsList.append(ftList)
+            fieldsTitles[ftList] = ftList
+            add_titles_to_csv_file([ftList], titles)
+
+
+# fieldName is command line argument
+# fieldNameTitleMap maps fieldName to API field name and CSV file header
+#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
+#  u'admincreated': [u'adminCreated', u'Admin_Created'],
+#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList,
+                                fieldsTitles, titles):
+    ftList = fieldNameTitleMap[fieldName]
+    for i in range(0, len(ftList), 2):
+        if ftList[i] not in fieldsTitles:
+            fieldsList.append(ftList[i])
+            fieldsTitles[ftList[i]] = ftList[i + 1]
+            add_titles_to_csv_file([ftList[i + 1]], titles)
+
+
+def sort_csv_titles(firstTitle, titles):
+    restoreTitles = []
+    for title in firstTitle:
+        if title in titles:
+            titles.remove(title)
+            restoreTitles.append(title)
+    titles.sort()
+    for title in restoreTitles[::-1]:
+        titles.insert(0, title)
+
+
+def QuotedArgumentList(items):
+    return ' '.join([
+        item if item and (item.find(' ') == -1) and
+        (item.find(',') == -1) else '"' + item + '"' for item in items
+    ])
+
+
+def write_csv_file(csvRows, titles, list_type, todrive):
+
+    def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
+        if not rowDate or not isinstance(rowDate, str):
+            return False
+        try:
+            rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
+            if dateMode:
+                rowDate = datetime.datetime(rowTime.year, rowTime.month,
+                                            rowTime.day).isoformat() + 'Z'
+        except ValueError:
+            rowDate = NEVER_TIME
+        if op == '<':
+            return rowDate < filterDate
+        if op == '<=':
+            return rowDate <= filterDate
+        if op == '>':
+            return rowDate > filterDate
+        if op == '>=':
+            return rowDate >= filterDate
+        if op == '!=':
+            return rowDate != filterDate
+        return rowDate == filterDate
+
+    def rowCountFilterMatch(rowCount, op, filterCount):
+        if isinstance(rowCount, str):
+            if not rowCount.isdigit():
+                return False
+            rowCount = int(rowCount)
+        elif not isinstance(rowCount, int):
+            return False
+        if op == '<':
+            return rowCount < filterCount
+        if op == '<=':
+            return rowCount <= filterCount
+        if op == '>':
+            return rowCount > filterCount
+        if op == '>=':
+            return rowCount >= filterCount
+        if op == '!=':
+            return rowCount != filterCount
+        return rowCount == filterCount
+
+    def rowBooleanFilterMatch(rowBoolean, filterBoolean):
+        if not isinstance(rowBoolean, bool):
+            return False
+        return rowBoolean == filterBoolean
+
+    def headerFilterMatch(filters, title):
+        for filterStr in filters:
+            if filterStr.match(title):
+                return True
+        return False
+
+    if GC_Values[GC_CSV_ROW_FILTER]:
+        for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
+            if column not in titles:
+                sys.stderr.write(
+                    f'WARNING: Row filter column "{column}" is not in output columns\n'
+                )
+                continue
+            if filterVal[0] == 'regex':
+                csvRows = [
+                    row for row in csvRows
+                    if filterVal[1].search(str(row.get(column, '')))
+                ]
+            elif filterVal[0] == 'notregex':
+                csvRows = [
+                    row for row in csvRows
+                    if not filterVal[1].search(str(row.get(column, '')))
+                ]
+            elif filterVal[0] in ['date', 'time']:
+                csvRows = [
+                    row for row in csvRows if rowDateTimeFilterMatch(
+                        filterVal[0] == 'date', row.get(column, ''),
+                        filterVal[1], filterVal[2])
+                ]
+            elif filterVal[0] == 'count':
+                csvRows = [
+                    row for row in csvRows if rowCountFilterMatch(
+                        row.get(column, 0), filterVal[1], filterVal[2])
+                ]
+            else:  #boolean
+                csvRows = [
+                    row for row in csvRows if rowBooleanFilterMatch(
+                        row.get(column, False), filterVal[1])
+                ]
+    if GC_Values[GC_CSV_HEADER_FILTER] or GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+        if GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+            titles = [
+                t for t in titles if
+                not headerFilterMatch(GC_Values[GC_CSV_HEADER_DROP_FILTER], t)
+            ]
+        if GC_Values[GC_CSV_HEADER_FILTER]:
+            titles = [
+                t for t in titles
+                if headerFilterMatch(GC_Values[GC_CSV_HEADER_FILTER], t)
+            ]
+        if not titles:
+            controlflow.system_error_exit(
+                3,
+                'No columns selected with GAM_CSV_HEADER_FILTER and GAM_CSV_HEADER_DROP_FILTER\n'
+            )
+            return
+    csv.register_dialect('nixstdout', lineterminator='\n')
+    if todrive:
+        write_to = io.StringIO()
+    else:
+        write_to = sys.stdout
+    writer = csv.DictWriter(write_to,
+                            fieldnames=titles,
+                            dialect='nixstdout',
+                            extrasaction='ignore',
+                            quoting=csv.QUOTE_MINIMAL)
+    try:
+        writer.writerow(dict((item, item) for item in writer.fieldnames))
+        writer.writerows(csvRows)
+    except IOError as e:
+        controlflow.system_error_exit(6, e)
+    if todrive:
+        admin_email = gam._get_admin_email()
+        _, drive = gam.buildDrive3GAPIObject(admin_email)
+        if not drive:
+            print(f'''\nGAM is not authorized to create Drive files. Please run:
+gam user {admin_email} check serviceaccount
+and follow recommend steps to authorize GAM for Drive access.''')
+            sys.exit(5)
+        result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
+        columns = len(titles)
+        rows = len(csvRows)
+        cell_count = rows * columns
+        data_size = len(write_to.getvalue())
+        max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
+        if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
+            print(
+                f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}'
+            )
+            mimeType = 'text/csv'
+        else:
+            mimeType = MIMETYPE_GA_SPREADSHEET
+        body = {
+            'description': QuotedArgumentList(sys.argv),
+            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
+            'mimeType': mimeType
+        }
+        result = gapi.call(drive.files(),
+                           'create',
+                           fields='webViewLink',
+                           body=body,
+                           media_body=googleapiclient.http.MediaInMemoryUpload(
+                               write_to.getvalue().encode(),
+                               mimetype='text/csv'))
+        file_url = result['webViewLink']
+        if GC_Values[GC_NO_BROWSER]:
+            msg_txt = f'Drive file uploaded to:\n {file_url}'
+            msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
+            gam.send_email(msg_subj, msg_txt)
+            print(msg_txt)
+        else:
+            webbrowser.open(file_url)
+
+
+def print_error(message):
+    """Prints a one-line error message to stderr in a standard format."""
+    sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
+
+
+def print_warning(message):
+    """Prints a one-line warning message to stderr in a standard format."""
+    sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
+
+
+def print_json(object_value, spacing=''):
+    """Prints Dict or Array to screen in clean human-readable format.."""
+    if isinstance(object_value, list):
+        if len(object_value) == 1 and isinstance(object_value[0],
+                                                 (str, int, bool)):
+            sys.stdout.write(f'{object_value[0]}\n')
+            return
+        if spacing:
+            sys.stdout.write('\n')
+        for i, a_value in enumerate(object_value):
+            if isinstance(a_value, (str, int, bool)):
+                sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
+            else:
+                sys.stdout.write(f' {spacing}{i+1}) ')
+                print_json(a_value, f' {spacing}')
+    elif isinstance(object_value, dict):
+        for key in ['kind', 'etag', 'etags']:
+            object_value.pop(key, None)
+        for another_object, another_value in object_value.items():
+            sys.stdout.write(f' {spacing}{another_object}: ')
+            print_json(another_value, f' {spacing}')
+    else:
+        sys.stdout.write(f'{object_value}\n')

src/gam/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+from gam import display
+from gam.var import ERROR_PREFIX
+from gam.var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+    def test_print_error_prints_to_stderr(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertIn(message, printed_message)
+
+    def test_print_error_prints_error_prefix(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertLess(
+            printed_message.find(ERROR_PREFIX), printed_message.find(message),
+            'The error prefix does not appear before the error message')
+
+    def test_print_error_ends_message_with_newline(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertRegex(printed_message, '\n$',
+                         'The error message does not end in a newline.')
+
+    def test_print_warning_prints_to_stderr(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertIn(message, printed_message)
+
+    def test_print_warning_prints_error_prefix(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertLess(
+            printed_message.find(WARNING_PREFIX), printed_message.find(message),
+            'The warning prefix does not appear before the error message')
+
+    def test_print_warning_ends_message_with_newline(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertRegex(printed_message, '\n$',
+                         'The warning message does not end in a newline.')

src/gam/fileutils.py

@@ -0,0 +1,183 @@
+"""Common file operations."""
+
+import io
+import os
+import sys
+
+from gam import controlflow
+from gam import display
+from gam.var import GM_Globals
+from gam.var import GM_SYS_ENCODING
+from gam.var import UTF8_SIG
+
+
+def _open_file(filename, mode, encoding=None, newline=None):
+    """Opens a file with no error handling."""
+    # Determine which encoding to use
+    if 'b' in mode:
+        encoding = None
+    elif not encoding:
+        encoding = GM_Globals[GM_SYS_ENCODING]
+    elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
+        encoding = UTF8_SIG
+
+    return open(os.path.expanduser(filename),
+                mode,
+                newline=newline,
+                encoding=encoding)
+
+
+def open_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              strip_utf_bom=False):
+    """Opens a file.
+
+  Args:
+    filename: String, the name of the file to open, or '-' to use stdin/stdout,
+      to read/write, depending on the mode param, respectively.
+    mode: String, the common file mode to open the file with. Default is read.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    strip_utf_bom: Boolean, True if the file being opened should seek past the
+      UTF Byte Order Mark before being returned.
+        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+
+  Returns:
+    The opened file.
+  """
+    try:
+        if filename == '-':
+            # Read from stdin, rather than a file
+            if 'r' in mode:
+                return io.StringIO(str(sys.stdin.read()))
+            return sys.stdout
+
+        # Open a file on disk
+        f = _open_file(filename, mode, newline=newline, encoding=encoding)
+        if strip_utf_bom:
+            utf_bom = u'\ufeff'
+            has_bom = False
+
+            if 'b' in mode:
+                has_bom = f.read(3).decode('UTF-8') == utf_bom
+            elif f.encoding and not f.encoding.lower().startswith('utf'):
+                # Convert UTF BOM into ISO-8859-1 via Bytes
+                utf8_bom_bytes = utf_bom.encode('UTF-8')
+                iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
+                    'iso-8859-1')
+                has_bom = f.read(3).encode('iso-8859-1',
+                                           'replace') == iso_8859_1_bom
+            else:
+                has_bom = f.read(1) == utf_bom
+
+            if not has_bom:
+                f.seek(0)
+
+        return f
+
+    except IOError as e:
+        controlflow.system_error_exit(6, e)
+
+
+def close_file(f, force_flush=False):
+    """Closes a file.
+
+  Args:
+    f: The file to close
+    force_flush: Flush file to disk emptying Python and OS caches. See:
+       https://stackoverflow.com/a/13762137/1503886
+
+  Returns:
+    Boolean, True if the file was successfully closed. False if an error
+        was encountered while closing.
+  """
+    if force_flush:
+        f.flush()
+        os.fsync(f.fileno())
+    try:
+        f.close()
+        return True
+    except IOError as e:
+        display.print_error(e)
+        return False
+
+
+def read_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              continue_on_error=False,
+              display_errors=True):
+    """Reads a file from disk.
+
+  Args:
+    filename: String, the path of the file to open from disk, or "-" to read
+      from stdin.
+    mode: String, the mode in which to open the file.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    The contents of the file, or stdin if filename == "-". Returns None if
+    an error is encountered and continue_on_errors is True.
+  """
+    try:
+        if filename == '-':
+            # Read from stdin, rather than a file.
+            return str(sys.stdin.read())
+
+        with _open_file(filename, mode, newline=newline,
+                        encoding=encoding) as f:
+            return f.read()
+
+    except IOError as e:
+        if continue_on_error:
+            if display_errors:
+                display.print_warning(e)
+            return None
+        controlflow.system_error_exit(6, e)
+    except (LookupError, UnicodeDecodeError, UnicodeError) as e:
+        controlflow.system_error_exit(2, str(e))
+
+
+def write_file(filename,
+               data,
+               mode='w',
+               continue_on_error=False,
+               display_errors=True):
+    """Writes data to a file.
+
+  Args:
+    filename: String, the path of the file to write to disk.
+    data: Serializable data to write to the file.
+    mode: String, the mode in which to open the file and write to it.
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    Boolean, True if the write operation succeeded, or False if not.
+  """
+    try:
+        with _open_file(filename, mode) as f:
+            f.write(data)
+        return True
+
+    except IOError as e:
+        if continue_on_error:
+            if display_errors:
+                display.print_error(e)
+            return False
+        else:
+            controlflow.system_error_exit(6, e)

src/gam/fileutils_test.py

@@ -0,0 +1,244 @@
+"""Tests for fileutils."""
+
+import io
+import os
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import fileutils
+
+
+class FileutilsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_path = '/some/path/to/file'
+        super(FileutilsTest, self).setUp()
+
+    @patch.object(fileutils.sys, 'stdin')
+    def test_open_file_stdin(self, mock_stdin):
+        mock_stdin.read.return_value = 'some stdin content'
+        f = fileutils.open_file('-', mode='r')
+        self.assertIsInstance(f, fileutils.io.StringIO)
+        self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
+
+    def test_open_file_stdout(self):
+        f = fileutils.open_file('-', mode='w')
+        self.assertEqual(fileutils.sys.stdout, f)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_opens_correct_path(self, mock_open):
+        f = fileutils.open_file(self.fake_path)
+        self.assertEqual(self.fake_path, mock_open.call_args[0][0])
+        self.assertEqual(mock_open.return_value, f)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_expands_user_file_path(self, mock_open):
+        file_path = '~/some/path/containing/tilde/shortcut/to/home'
+        fileutils.open_file(file_path)
+        opened_path = mock_open.call_args[0][0]
+        home_path = os.environ.get('HOME')
+        self.assertIsNotNone(home_path)
+        self.assertIn(home_path, opened_path)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_opens_correct_mode(self, mock_open):
+        fileutils.open_file(self.fake_path)
+        self.assertEqual('r', mock_open.call_args[0][1])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_encoding_for_binary(self, mock_open):
+        fileutils.open_file(self.fake_path, mode='b')
+        self.assertIsNone(mock_open.call_args[1]['encoding'])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_default_system_encoding(self, mock_open):
+        fileutils.open_file(self.fake_path)
+        self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
+                         mock_open.call_args[1]['encoding'])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_utf8_encoding_specified(self, mock_open):
+        fileutils.open_file(self.fake_path, encoding='UTF-8')
+        self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
+
+    def test_open_file_strips_utf_bom_in_utf(self):
+        bom_prefixed_data = u'\ufefffoobar'
+        fake_file = io.StringIO(bom_prefixed_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            self.assertEqual('foobar', f.read())
+
+    def test_open_file_strips_utf_bom_in_non_utf(self):
+        bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
+
+        # We need to trick the method under test into believing that a StringIO
+        # instance is a file with an encoding. Since StringIO does not usually have,
+        # an encoding, we'll mock it and add our own encoding, but send the other
+        # methods in use (read and seek) back to the real StringIO object.
+        real_stringio = io.StringIO(bom_prefixed_data)
+        mock_file = MagicMock(spec=io.StringIO)
+        mock_file.read.side_effect = real_stringio.read
+        mock_file.seek.side_effect = real_stringio.seek
+        mock_file.encoding = 'iso-8859-1'
+
+        mock_open = MagicMock(spec=open, return_value=mock_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            self.assertEqual('foobar', f.read())
+
+    def test_open_file_strips_utf_bom_in_binary(self):
+        bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
+        fake_file = io.BytesIO(bom_prefixed_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path,
+                                    mode='rb',
+                                    strip_utf_bom=True)
+            self.assertEqual(b'foobar', f.read())
+
+    def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
+        no_bom_data = 'This data has no BOM'
+        fake_file = io.StringIO(no_bom_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            # Since there was no opening BOM, we should be back at the beginning of
+            # the file.
+            self.assertEqual(fake_file.tell(), 0)
+            self.assertEqual(f.read(), no_bom_data)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_exits_on_io_error(self, mock_open):
+        mock_open.side_effect = IOError('Fake IOError')
+        with self.assertRaises(SystemExit) as context:
+            fileutils.open_file(self.fake_path)
+        self.assertEqual(context.exception.code, 6)
+
+    def test_close_file_closes_file_successfully(self):
+        mock_file = MagicMock()
+        self.assertTrue(fileutils.close_file(mock_file))
+        self.assertEqual(mock_file.close.call_count, 1)
+
+    def test_close_file_with_error(self):
+        mock_file = MagicMock()
+        mock_file.close.side_effect = IOError()
+        self.assertFalse(fileutils.close_file(mock_file))
+        self.assertEqual(mock_file.close.call_count, 1)
+
+    @patch.object(fileutils.sys, 'stdin')
+    def test_read_file_from_stdin(self, mock_stdin):
+        mock_stdin.read.return_value = 'some stdin content'
+        self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_default_params(self, mock_open_file):
+        fake_content = 'some fake content'
+        mock_open_file.return_value.__enter__().read.return_value = fake_content
+        self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
+        self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+        self.assertEqual(mock_open_file.call_args[0][1], 'r')
+        self.assertIsNone(mock_open_file.call_args[1]['newline'])
+
+    @patch.object(fileutils.display, 'print_warning')
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_continues_on_errors_without_displaying(
+        self, mock_open_file, mock_print_warning):
+        mock_open_file.side_effect = IOError()
+        contents = fileutils.read_file(self.fake_path,
+                                       continue_on_error=True,
+                                       display_errors=False)
+        self.assertIsNone(contents)
+        self.assertFalse(mock_print_warning.called)
+
+    @patch.object(fileutils.display, 'print_warning')
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_displays_errors(self, mock_open_file,
+                                       mock_print_warning):
+        mock_open_file.side_effect = IOError()
+        fileutils.read_file(self.fake_path,
+                            continue_on_error=True,
+                            display_errors=True)
+        self.assertTrue(mock_print_warning.called)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_6_when_continue_on_error_is_false(
+        self, mock_open_file):
+        mock_open_file.side_effect = IOError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path, continue_on_error=False)
+        self.assertEqual(context.exception.code, 6)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
+        mock_open_file.return_value.__enter__().read.side_effect = LookupError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
+        mock_open_file.return_value.__enter__().read.side_effect = UnicodeError(
+        )
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
+        fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0,
+                                               1, 'testing only')
+        mock_open_file.return_value.__enter__(
+        ).read.side_effect = fake_decode_error
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_writes_data_to_file(self, mock_open_file):
+        fake_data = 'some fake data'
+        fileutils.write_file(self.fake_path, fake_data)
+        self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+        self.assertEqual(mock_open_file.call_args[0][1], 'w')
+
+        opened_file = mock_open_file.return_value.__enter__()
+        self.assertTrue(opened_file.write.called)
+        self.assertEqual(opened_file.write.call_args[0][0], fake_data)
+
+    @patch.object(fileutils.display, 'print_error')
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_continues_on_errors_without_displaying(
+        self, mock_open_file, mock_print_error):
+        mock_open_file.side_effect = IOError()
+        status = fileutils.write_file(self.fake_path,
+                                      'foo data',
+                                      continue_on_error=True,
+                                      display_errors=False)
+        self.assertFalse(status)
+        self.assertFalse(mock_print_error.called)
+
+    @patch.object(fileutils.display, 'print_error')
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
+        mock_open_file.side_effect = IOError()
+        fileutils.write_file(self.fake_path,
+                             'foo data',
+                             continue_on_error=True,
+                             display_errors=True)
+        self.assertTrue(mock_print_error.called)
+
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_exits_code_6_when_continue_on_error_is_false(
+        self, mock_open_file):
+        mock_open_file.side_effect = IOError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.write_file(self.fake_path,
+                                 'foo data',
+                                 continue_on_error=False)
+        self.assertEqual(context.exception.code, 6)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/__init__.py

@@ -0,0 +1,349 @@
+"""Methods related to execution of GAPI requests."""
+
+import sys
+
+import googleapiclient.errors
+import google.auth.exceptions
+import httplib2
+
+from gam import controlflow
+from gam import display
+from gam.gapi import errors
+from gam import transport
+from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+                     GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
+                     MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
+                     MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
+
+
+def call(service,
+         function,
+         silent_errors=False,
+         soft_errors=False,
+         throw_reasons=None,
+         retry_reasons=None,
+         **kwargs):
+    """Executes a single request on a Google service function.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    silent_errors: Bool, If True, error messages are suppressed when
+      encountered.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A response object for the corresponding Google API call.
+  """
+    if throw_reasons is None:
+        throw_reasons = []
+    if retry_reasons is None:
+        retry_reasons = []
+
+    method = getattr(service, function)
+    retries = 10
+    parameters = dict(
+        list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+    for n in range(1, retries + 1):
+        try:
+            return method(**parameters).execute()
+        except googleapiclient.errors.HttpError as e:
+            http_status, reason, message = errors.get_gapi_error_detail(
+                e,
+                soft_errors=soft_errors,
+                silent_errors=silent_errors,
+                retry_on_http_error=n < 3)
+            if http_status == -1:
+                # The error detail indicated that we should retry this request
+                # We'll refresh credentials and make another pass
+                service._http.credentials.refresh(transport.create_http())
+                continue
+            if http_status == 0:
+                return None
+
+            is_known_error_reason = reason in [
+                r.value for r in errors.ErrorReason
+            ]
+            if is_known_error_reason and errors.ErrorReason(
+                    reason) in throw_reasons:
+                if errors.ErrorReason(
+                        reason) in errors.ERROR_REASON_TO_EXCEPTION:
+                    raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(
+                        reason)](message)
+                raise e
+            if (n != retries) and (is_known_error_reason and errors.ErrorReason(
+                    reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
+                controlflow.wait_on_failure(n, retries, reason)
+                continue
+            if soft_errors:
+                display.print_error(
+                    f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}'
+                )
+                return None
+            controlflow.system_error_exit(
+                int(http_status), f'{http_status}: {message} - {reason}')
+        except google.auth.exceptions.RefreshError as e:
+            handle_oauth_token_error(
+                e, soft_errors or
+                errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+            if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
+                raise errors.GapiServiceNotAvailableError(str(e))
+            display.print_error(
+                f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
+            return None
+        except ValueError as e:
+            if hasattr(service._http,
+                       'cache') and service._http.cache is not None:
+                service._http.cache = None
+                continue
+            controlflow.system_error_exit(4, str(e))
+        except (httplib2.ServerNotFoundError, RuntimeError) as e:
+            if n != retries:
+                service._http.connections = {}
+                controlflow.wait_on_failure(n, retries, str(e))
+                continue
+            controlflow.system_error_exit(4, str(e))
+        except TypeError as e:
+            controlflow.system_error_exit(4, str(e))
+
+
+def get_items(service,
+              function,
+              items='items',
+              throw_reasons=None,
+              retry_reasons=None,
+              **kwargs):
+    """Gets a single page of items from a Google service function that is paged.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the service
+      method's response object.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    The list of items in the first page of a response.
+  """
+    results = call(service,
+                   function,
+                   throw_reasons=throw_reasons,
+                   retry_reasons=retry_reasons,
+                   **kwargs)
+    if results:
+        return results.get(items, [])
+    return []
+
+
+def _get_max_page_size_for_api_call(service, function, **kwargs):
+    """Gets the maximum number of results supported for a single API call.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of the service method to check for max page size.
+    **kwargs: Additional params that will be passed to the request method.
+
+  Returns:
+    Int, A value from discovery if it exists, otherwise value from
+        MAX_RESULTS_API_EXCEPTIONS, otherwise None
+  """
+    method = getattr(service, function)
+    api_id = method(**kwargs).methodId
+    for resource in service._rootDesc.get('resources', {}).values():
+        for a_method in resource.get('methods', {}).values():
+            if a_method.get('id') == api_id:
+                if not a_method.get('parameters') or a_method['parameters'].get(
+                        'pageSize'
+                ) or not a_method['parameters'].get('maxResults'):
+                    # Make sure API call supports maxResults. For now we don't care to
+                    # set pageSize since all known pageSize API calls have
+                    # default pageSize == max pageSize.
+                    return None
+                known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
+                max_results = a_method['parameters']['maxResults'].get(
+                    'maximum', known_api_max)
+                return {'maxResults': max_results}
+
+    return None
+
+
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+
+def got_total_items_msg(items, eol):
+    """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+    return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+
+def got_total_items_first_last_msg(items):
+    """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+    return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}' + '\n'
+
+
+def get_all_pages(service,
+                  function,
+                  items='items',
+                  page_message=None,
+                  message_attribute=None,
+                  soft_errors=False,
+                  throw_reasons=None,
+                  retry_reasons=None,
+                  **kwargs):
+    """Aggregates and returns all pages of a Google service function response.
+
+  All pages of items are aggregated and returned as a single list.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the method's
+      response object. The items in this field will be aggregated across all
+      pages and returned.
+    page_message: String, a message to be displayed to the user during paging.
+      Template strings allow for dynamic content to be inserted during paging.
+        Supported template strings:
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
+            pages.
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
+            display a unique property of the first item in the current page.
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
+            display a unique property of the last item in the current page.
+    message_attribute: String or list, the name of a signature field within a
+    single returned item which identifies that unique item. This field is used
+    with `page_message` to templatize a paging status message.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A list of all items received from all paged responses.
+  """
+    if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
+        page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
+        if page_key:
+            kwargs.update(page_key)
+    all_items = []
+    page_token = None
+    total_items = 0
+    while True:
+        page = call(service,
+                    function,
+                    soft_errors=soft_errors,
+                    throw_reasons=throw_reasons,
+                    retry_reasons=retry_reasons,
+                    pageToken=page_token,
+                    **kwargs)
+        if page:
+            page_token = page.get('nextPageToken')
+            page_items = page.get(items, [])
+            num_page_items = len(page_items)
+            total_items += num_page_items
+            all_items.extend(page_items)
+        else:
+            page_token = None
+            num_page_items = 0
+
+        # Show a paging message to the user that indicates paging progress
+        if page_message:
+            show_message = page_message.replace(TOTAL_ITEMS_MARKER,
+                                                str(total_items))
+            if message_attribute:
+                first_item = page_items[0] if num_page_items > 0 else {}
+                last_item = page_items[-1] if num_page_items > 1 else first_item
+                if type(message_attribute) is str:
+                    first_item = str(first_item.get(message_attribute, ''))
+                    last_item = str(last_item.get(message_attribute, ''))
+                else:
+                    for attr in message_attribute:
+                        first_item = first_item.get(attr, {})
+                        last_item = last_item.get(attr, {})
+                    first_item = str(first_item)
+                    last_item = str(last_item)
+                show_message = show_message.replace(FIRST_ITEM_MARKER, first_item)
+                show_message = show_message.replace(LAST_ITEM_MARKER, last_item)
+            sys.stderr.write('\r')
+            sys.stderr.flush()
+            sys.stderr.write(show_message)
+
+        if not page_token:
+            # End the paging status message and return all items.
+            if page_message and (page_message[-1] != '\n'):
+                sys.stderr.write('\r\n')
+                sys.stderr.flush()
+            return all_items
+
+
+# TODO: Make this private once all execution related items that use this method
+# have been brought into this file
+def handle_oauth_token_error(e, soft_errors):
+    """On a token error, exits the application and writes a message to stderr.
+
+  Args:
+    e: google.auth.exceptions.RefreshError, The error to handle.
+    soft_errors: Boolean, if True, suppresses any applicable errors and instead
+      returns to the caller.
+  """
+    token_error = str(e).replace('.', '')
+    if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
+            'Invalid response'):
+        if soft_errors:
+            return
+        if not GM_Globals[GM_CURRENT_API_USER]:
+            display.print_error(
+                MESSAGE_API_ACCESS_DENIED.format(
+                    GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+                    ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+            controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
+        else:
+            controlflow.system_error_exit(
+                19,
+                MESSAGE_SERVICE_NOT_APPLICABLE.format(
+                    GM_Globals[GM_CURRENT_API_USER]))
+    controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
+
+
+def get_enum_values_minus_unspecified(values):
+    return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gam/gapi/__init___test.py

@@ -0,0 +1,519 @@
+"""Tests for gapi."""
+
+import json
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import gam.gapi as gapi
+from gam.gapi import errors
+
+
+def create_http_error(status, reason, message):
+    """Creates a HttpError object similar to most Google API Errors.
+
+  Args:
+    status: Int, the error's HTTP response status number.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    googleapiclient.errors.HttpError
+  """
+    response = {
+        'status': status,
+        'content-type': 'application/json',
+    }
+    content = {
+        'error': {
+            'code': status,
+            'errors': [{
+                'reason': str(reason),
+                'message': message,
+            }]
+        }
+    }
+    content_bytes = json.dumps(content).encode('UTF-8')
+    return gapi.googleapiclient.errors.HttpError(response, content_bytes)
+
+
+class GapiTest(unittest.TestCase):
+
+    def setUp(self):
+        SetGlobalVariables()
+        self.mock_service = MagicMock()
+        self.mock_method_name = 'mock_method'
+        self.mock_method = getattr(self.mock_service, self.mock_method_name)
+
+        self.simple_3_page_response = [
+            {
+                'items': [{
+                    'position': 'page1,item1'
+                }, {
+                    'position': 'page1,item2'
+                }, {
+                    'position': 'page1,item3'
+                }],
+                'nextPageToken': 'page2'
+            },
+            {
+                'items': [{
+                    'position': 'page2,item1'
+                }, {
+                    'position': 'page2,item2'
+                }, {
+                    'position': 'page2,item3'
+                }],
+                'nextPageToken': 'page3'
+            },
+            {
+                'items': [{
+                    'position': 'page3,item1'
+                }, {
+                    'position': 'page3,item2'
+                }, {
+                    'position': 'page3,item3'
+                }],
+            },
+        ]
+        self.empty_items_response = {'items': []}
+
+        super(GapiTest, self).setUp()
+
+    def test_call_returns_basic_200_response(self):
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, self.mock_method().execute.return_value)
+
+    def test_call_passes_target_method_params(self):
+        gapi.call(self.mock_service,
+                  self.mock_method_name,
+                  my_param_1=1,
+                  my_param_2=2)
+        self.assertEqual(self.mock_method.call_count, 1)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(method_kwargs.get('my_param_1'), 1)
+        self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+    @patch.object(gapi.errors, 'get_gapi_error_detail')
+    def test_call_retries_with_soft_errors(self, mock_error_detail):
+        mock_error_detail.return_value = (-1, 'aReason', 'some message')
+
+        # Make the request fail first, then return the proper response on the retry.
+        fake_http_error = create_http_error(403, 'aReason', 'unused message')
+        fake_200_response = MagicMock()
+        self.mock_method.return_value.execute.side_effect = [
+            fake_http_error, fake_200_response
+        ]
+
+        response = gapi.call(self.mock_service,
+                             self.mock_method_name,
+                             soft_errors=True)
+        self.assertEqual(response, fake_200_response)
+        self.assertEqual(self.mock_service._http.credentials.refresh.call_count,
+                         1)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+
+    def test_call_throws_for_provided_reason(self):
+        throw_reason = errors.ErrorReason.USER_NOT_FOUND
+        fake_http_error = create_http_error(404, throw_reason, 'forced throw')
+        self.mock_method.return_value.execute.side_effect = fake_http_error
+
+        gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
+        with self.assertRaises(gam_exception):
+            gapi.call(self.mock_service,
+                      self.mock_method_name,
+                      throw_reasons=[throw_reason])
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_request_for_default_retry_reasons(
+        self, mock_wait_on_failure):
+
+        # Test using one of the default retry reasons
+        default_throw_reason = errors.ErrorReason.BACKEND_ERROR
+        self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
+
+        fake_http_error = create_http_error(404, default_throw_reason,
+                                            'message')
+        fake_200_response = MagicMock()
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_http_error, fake_200_response
+        ]
+
+        response = gapi.call(self.mock_service,
+                             self.mock_method_name,
+                             retry_reasons=[])
+        self.assertEqual(response, fake_200_response)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+        # Make sure a backoff technique was used for retry.
+        self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_requests_for_provided_retry_reasons(
+        self, unused_mock_wait_on_failure):
+
+        retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
+        fake_retrieable_error1 = create_http_error(400, retry_reason1,
+                                                   'Forced Error 1')
+        retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
+        fake_retrieable_error2 = create_http_error(400, retry_reason2,
+                                                   'Forced Error 2')
+        non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
+        fake_non_retriable_error = create_http_error(
+            400, non_retriable_reason,
+            'This error should not cause the request to be retried')
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_retrieable_error1, fake_retrieable_error2,
+            fake_non_retriable_error
+        ]
+
+        with self.assertRaises(SystemExit):
+            # The third call should raise the SystemExit when non_retriable_error is
+            # raised.
+            gapi.call(self.mock_service,
+                      self.mock_method_name,
+                      retry_reasons=[retry_reason1, retry_reason2])
+
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
+
+    def test_call_exits_on_oauth_token_error(self):
+        # An error with any OAUTH2_TOKEN_ERROR
+        fake_token_error = gapi.google.auth.exceptions.RefreshError(
+            errors.OAUTH2_TOKEN_ERRORS[0])
+        self.mock_method.return_value.execute.side_effect = fake_token_error
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_exits_on_nonretriable_error(self):
+        error_reason = 'unknownReason'
+        fake_http_error = create_http_error(500, error_reason,
+                                            'Testing unretriable errors')
+        self.mock_method.return_value.execute.side_effect = fake_http_error
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_exits_on_request_valueerror(self):
+        self.mock_method.return_value.execute.side_effect = ValueError()
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_clears_bad_http_cache_on_request_failure(self):
+        self.mock_service._http.cache = 'something that is not None'
+        fake_200_response = MagicMock()
+        self.mock_method.return_value.execute.side_effect = [
+            ValueError(), fake_200_response
+        ]
+
+        self.assertIsNotNone(self.mock_service._http.cache)
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, fake_200_response)
+        # Assert the cache was cleared
+        self.assertIsNone(self.mock_service._http.cache)
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_requests_with_backoff_on_servernotfounderror(
+        self, mock_wait_on_failure):
+        fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
+        fake_200_response = MagicMock()
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_servernotfounderror, fake_200_response
+        ]
+
+        http_connections = self.mock_service._http.connections
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, fake_200_response)
+        # HTTP cached connections should be cleared on receiving this error
+        self.assertNotEqual(http_connections,
+                            self.mock_service._http.connections)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+        # Make sure a backoff technique was used for retry.
+        self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+    def test_get_items_calls_correct_service_function(self):
+        gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertTrue(self.mock_method.called)
+
+    def test_get_items_returns_one_page(self):
+        fake_response = {'items': [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertEqual(page, fake_response['items'])
+
+    def test_get_items_non_default_page_field_name(self):
+        field_name = 'things'
+        fake_response = {field_name: [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_items(self.mock_service,
+                              self.mock_method_name,
+                              items=field_name)
+        self.assertEqual(page, fake_response[field_name])
+
+    def test_get_items_passes_additional_kwargs_to_service(self):
+        gapi.get_items(self.mock_service,
+                       self.mock_method_name,
+                       my_param_1=1,
+                       my_param_2=2)
+        self.assertEqual(self.mock_method.call_count, 1)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(1, method_kwargs.get('my_param_1'))
+        self.assertEqual(2, method_kwargs.get('my_param_2'))
+
+    def test_get_items_returns_empty_list_when_no_items_returned(self):
+        non_items_response = {'noItemsInThisResponse': {}}
+        self.mock_method.return_value.execute.return_value = non_items_response
+        page = gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertIsInstance(page, list)
+        self.assertEqual(0, len(page))
+
+    def test_get_all_pages_returns_all_items(self):
+        page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
+        page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
+        page_3 = {'items': ['3-1', '3-2', '3-3']}
+        self.mock_method.return_value.execute.side_effect = [
+            page_1, page_2, page_3
+        ]
+        response_items = gapi.get_all_pages(self.mock_service,
+                                            self.mock_method_name)
+        self.assertListEqual(
+            response_items, page_1['items'] + page_2['items'] + page_3['items'])
+
+    def test_get_all_pages_includes_next_pagetoken_in_request(self):
+        page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
+        page_2 = {'items': ['2-1', '2-2', '2-3']}
+        self.mock_method.return_value.execute.side_effect = [page_1, page_2]
+
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           pageSize=100)
+        self.assertEqual(self.mock_method.call_count, 2)
+        call_2_kwargs = self.mock_method.call_args_list[1][1]
+        self.assertIn('pageToken', call_2_kwargs)
+        self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
+
+    def test_get_all_pages_uses_default_max_page_size(self):
+        sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
+        sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
+        self.mock_method.return_value.methodId = sample_api_id
+        self.mock_service._rootDesc = {
+            'resources': {
+                'someResource': {
+                    'methods': {
+                        'someMethod': {
+                            'id': sample_api_id,
+                            'parameters': {
+                                'maxResults': {
+                                    'maximum': sample_api_max_results
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+        gapi.get_all_pages(self.mock_service, self.mock_method_name)
+        request_method_kwargs = self.mock_method.call_args[1]
+        self.assertIn('maxResults', request_method_kwargs)
+        self.assertEqual(request_method_kwargs['maxResults'],
+                         gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
+
+    def test_get_all_pages_max_page_size_overrided(self):
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           pageSize=123456)
+        request_method_kwargs = self.mock_method.call_args[1]
+        self.assertIn('pageSize', request_method_kwargs)
+        self.assertEqual(123456, request_method_kwargs['pageSize'])
+
+    def test_get_all_pages_prints_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        self.assertIn(paging_message, messages_written)
+
+    def test_get_all_pages_prints_paging_message_inline(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+
+        # Make sure a return carriage was written between two pages
+        paging_message_call_positions = [
+            i for i, message in enumerate(messages_written)
+            if message == paging_message
+        ]
+        self.assertGreater(len(paging_message_call_positions), 1)
+        printed_between_page_messages = messages_written[
+            paging_message_call_positions[0]:paging_message_call_positions[1]]
+        self.assertIn('\r', printed_between_page_messages)
+
+    def test_get_all_pages_ends_paging_message_with_newline(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        last_page_message_index = len(
+            messages_written) - messages_written[::-1].index(paging_message)
+        last_carriage_return_index = len(
+            messages_written) - messages_written[::-1].index('\r\n')
+        self.assertGreater(last_carriage_return_index, last_page_message_index)
+
+    def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'Total number of items discovered: %%total_items%%'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_item_count = len(self.simple_3_page_response[0]['items'])
+        page_1_message = paging_message.replace('%%total_items%%',
+                                                str(page_1_item_count))
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_item_count = len(self.simple_3_page_response[1]['items'])
+        page_2_message = paging_message.replace(
+            '%%total_items%%', str(page_1_item_count + page_2_item_count))
+        self.assertIn(page_2_message, messages_written)
+
+        page_3_item_count = len(self.simple_3_page_response[2]['items'])
+        page_3_message = paging_message.replace(
+            '%%total_items%%',
+            str(page_1_item_count + page_2_item_count + page_3_item_count))
+        self.assertIn(page_3_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%total_items', message)
+
+    def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'First item in page: %%first_item%%'
+
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message,
+                               message_attribute='position')
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_message = paging_message.replace(
+            '%%first_item%%',
+            self.simple_3_page_response[0]['items'][0]['position'])
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_message = paging_message.replace(
+            '%%first_item%%',
+            self.simple_3_page_response[1]['items'][0]['position'])
+        self.assertIn(page_2_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%first_item', message)
+
+    def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'Last item in page: %%last_item%%'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message,
+                               message_attribute='position')
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_message = paging_message.replace(
+            '%%last_item%%',
+            self.simple_3_page_response[0]['items'][-1]['position'])
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_message = paging_message.replace(
+            '%%last_item%%',
+            self.simple_3_page_response[1]['items'][-1]['position'])
+        self.assertIn(page_2_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%last_item', message)
+
+    def test_get_all_pages_prints_all_attributes_in_paging_message(self):
+        pass
+
+    def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           my_param_1=1,
+                           my_param_2=2)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(method_kwargs.get('my_param_1'), 1)
+        self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+    @patch.object(gapi, 'call')
+    def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
+        throw_for = MagicMock()
+        retry_for = MagicMock()
+        mock_call.return_value = self.empty_items_response
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           throw_reasons=throw_for,
+                           retry_reasons=retry_for)
+        method_kwargs = mock_call.call_args[1]
+        self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
+        self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
+
+    def test_get_all_pages_non_default_items_field_name(self):
+        field_name = 'things'
+        fake_response = {field_name: [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_all_pages(self.mock_service,
+                                  self.mock_method_name,
+                                  items=field_name)
+        self.assertEqual(page, fake_response[field_name])
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/calendar.py

@@ -0,0 +1,984 @@
+import csv
+import sys
+import uuid
+
+# TODO: get rid of these hacks
+import gam
+from gam.var import *
+
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
+
+
+def normalizeCalendarId(calname, checkPrimary=False):
+    if checkPrimary and calname.lower() == 'primary':
+        return calname
+    if not GC_Values[GC_DOMAIN]:
+        GC_Values[GC_DOMAIN] = gam._getValueFromOAuth('hd')
+    return gam.convertUIDtoEmailAddress(calname,
+                                        email_types=['user', 'resource'])
+
+
+def buildCalendarGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+    return (calendarId, gam.buildGAPIServiceObject('calendar', calendarId))
+
+
+def buildCalendarDataGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+
+    # Try to impersonate the calendar owner. If we fail, fall back to using
+    # admin for authentication. Resource calendars cannot be impersonated,
+    # so we need to access them as the admin.
+    cal = None
+    if not calname.endswith('.calendar.google.com'):
+        cal = gam.buildGAPIServiceObject('calendar', calendarId, False)
+    if cal is None:
+        _, cal = buildCalendarGAPIObject(gam._get_admin_email())
+    return (calendarId, cal)
+
+
+def printShowACLs(csvFormat):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    toDrive = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if csvFormat and myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            action = ['showacl', 'printacl'][csvFormat]
+            message = f'gam calendar <email> {action}'
+            controlflow.invalid_argument_exit(sys.argv[i], message)
+    acls = gapi.get_all_pages(cal.acl(), 'list', 'items', calendarId=calendarId)
+    i = 0
+    if csvFormat:
+        titles = []
+        rows = []
+    else:
+        count = len(acls)
+    for rule in acls:
+        i += 1
+        if csvFormat:
+            row = utils.flatten_json(rule, None)
+            for key in row:
+                if key not in titles:
+                    titles.append(key)
+            rows.append(row)
+        else:
+            formatted_acl = formatACLRule(rule)
+            current_count = display.current_count(i, count)
+            print(
+                f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
+    if csvFormat:
+        display.write_csv_file(rows, titles, f'{calendarId} Calendar ACLs',
+                               toDrive)
+
+
+def _getCalendarACLScope(i, body):
+    body['scope'] = {}
+    myarg = sys.argv[i].lower()
+    body['scope']['type'] = myarg
+    i += 1
+    if myarg in ['user', 'group']:
+        body['scope']['value'] = gam.normalizeEmailAddressOrUID(sys.argv[i],
+                                                                noUid=True)
+        i += 1
+    elif myarg == 'domain':
+        if i < len(sys.argv) and \
+           sys.argv[i].lower().replace('_', '') != 'sendnotifications':
+            body['scope']['value'] = sys.argv[i].lower()
+            i += 1
+        else:
+            body['scope']['value'] = GC_Values[GC_DOMAIN]
+    elif myarg != 'default':
+        body['scope']['type'] = 'user'
+        body['scope']['value'] = gam.normalizeEmailAddressOrUID(myarg,
+                                                                noUid=True)
+    return i
+
+
+CALENDAR_ACL_ROLES_MAP = {
+    'editor': 'writer',
+    'freebusy': 'freeBusyReader',
+    'freebusyreader': 'freeBusyReader',
+    'owner': 'owner',
+    'read': 'reader',
+    'reader': 'reader',
+    'writer': 'writer',
+    'none': 'none',
+}
+
+
+def addACL(function):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    myarg = sys.argv[4].lower().replace('_', '')
+    if myarg not in CALENDAR_ACL_ROLES_MAP:
+        controlflow.expected_argument_exit('Role',
+                                           ', '.join(CALENDAR_ACL_ROLES_MAP),
+                                           myarg)
+    body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]}
+    i = _getCalendarACLScope(5, body)
+    sendNotifications = True
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'sendnotifications':
+            sendNotifications = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {function.lower()}')
+    print(f'Calendar: {calendarId}, {function} ACL: {formatACLRule(body)}')
+    gapi.call(cal.acl(),
+              'insert',
+              calendarId=calendarId,
+              body=body,
+              sendNotifications=sendNotifications)
+
+
+def delACL():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    if sys.argv[4].lower() == 'id':
+        ruleId = sys.argv[5]
+        print(f'Removing rights for {ruleId} to {calendarId}')
+        gapi.call(cal.acl(), 'delete', calendarId=calendarId, ruleId=ruleId)
+    else:
+        body = {'role': 'none'}
+        _getCalendarACLScope(5, body)
+        print(f'Calendar: {calendarId}, Delete ACL: {formatACLScope(body)}')
+        gapi.call(cal.acl(),
+                  'insert',
+                  calendarId=calendarId,
+                  body=body,
+                  sendNotifications=False)
+
+
+def wipeData():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    gapi.call(cal.calendars(), 'clear', calendarId=calendarId)
+
+
+def printEvents():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    q = showDeleted = showHiddenInvitations = timeMin = \
+        timeMax = timeZone = updatedMin = None
+    toDrive = False
+    titles = []
+    csvRows = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            q = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'includedeleted':
+            showDeleted = True
+            i += 1
+        elif myarg == 'includehidden':
+            showHiddenInvitations = True
+            i += 1
+        elif myarg == 'after':
+            timeMin = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'before':
+            timeMax = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'updated':
+            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], 'gam calendar <email> printevents')
+    page_message = gapi.got_total_items_msg(f'Events for {calendarId}', '')
+    results = gapi.get_all_pages(cal.events(),
+                                 'list',
+                                 'items',
+                                 page_message=page_message,
+                                 calendarId=calendarId,
+                                 q=q,
+                                 showDeleted=showDeleted,
+                                 showHiddenInvitations=showHiddenInvitations,
+                                 timeMin=timeMin,
+                                 timeMax=timeMax,
+                                 timeZone=timeZone,
+                                 updatedMin=updatedMin)
+    for result in results:
+        row = {'calendarId': calendarId}
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(result, flattened=row), csvRows, titles)
+    display.sort_csv_titles(['calendarId', 'id', 'summary', 'status'], titles)
+    display.write_csv_file(csvRows, titles, 'Calendar Events', toDrive)
+
+
+def formatACLScope(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]})'
+    return f'(Scope: {rule["scope"]["type"]})'
+
+
+def formatACLRule(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]}, ' \
+               f'Role: {rule["role"]})'
+    return f'(Scope: {rule["scope"]["type"]}, Role: {rule["role"]})'
+
+
+def getSendUpdates(myarg, i, cal):
+    if myarg == 'notifyattendees':
+        sendUpdates = 'all'
+        i += 1
+    elif myarg == 'sendnotifications':
+        sendUpdates = 'all' if gam.getBoolean(sys.argv[i +
+                                                       1], myarg) else 'none'
+        i += 2
+    else:  # 'sendupdates':
+        sendUpdatesMap = {}
+        for val in cal._rootDesc['resources']['events']['methods']['delete'][
+                'parameters']['sendUpdates']['enum']:
+            sendUpdatesMap[val.lower()] = val
+        sendUpdates = sendUpdatesMap.get(sys.argv[i + 1].lower(), False)
+        if not sendUpdates:
+            controlflow.expected_argument_exit('sendupdates',
+                                               ', '.join(sendUpdatesMap),
+                                               sys.argv[i + 1])
+        i += 2
+    return (sendUpdates, i)
+
+
+def moveOrDeleteEvent(moveOrDelete):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    sendUpdates = 'none'
+    doit = False
+    kwargs = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg in ['id', 'eventid']:
+            eventId = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['query', 'eventquery']:
+            controlflow.system_error_exit(
+                2, f'query is no longer supported for {moveOrDelete}event. ' \
+                   f'Use "gam calendar <email> printevents query <query> | ' \
+                   f'gam csv - gam {moveOrDelete}event id ~id" instead.')
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        elif moveOrDelete == 'move' and myarg == 'destination':
+            kwargs['destination'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {moveOrDelete}event')
+    if doit:
+        print(f' going to {moveOrDelete} eventId {eventId}')
+        gapi.call(cal.events(),
+                  moveOrDelete,
+                  calendarId=calendarId,
+                  eventId=eventId,
+                  sendUpdates=sendUpdates,
+                  **kwargs)
+    else:
+        print(
+            f' would {moveOrDelete} eventId {eventId}. Add doit to command ' \
+            f'to actually {moveOrDelete} event')
+
+
+def infoEvent():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    eventId = sys.argv[4]
+    result = gapi.call(cal.events(),
+                       'get',
+                       calendarId=calendarId,
+                       eventId=eventId)
+    display.print_json(result)
+
+
+def addOrUpdateEvent(action):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    # only way for non-Google calendars to get updates is via email
+    kwargs = {}
+    body = {}
+    if action == 'add':
+        i = 4
+        func = 'insert'
+    else:
+        eventId = sys.argv[4]
+        kwargs = {'eventId': eventId}
+        i = 5
+        func = 'patch'
+        requires_full_update = [
+            'attendee', 'optionalattendee', 'removeattendee',
+            'replacedescription'
+        ]
+        for arg in sys.argv[i:]:
+            if arg.replace('_', '').lower() in requires_full_update:
+                func = 'update'
+                body = gapi.call(cal.events(),
+                                 'get',
+                                 calendarId=calendarId,
+                                 eventId=eventId)
+                break
+    sendUpdates, body = getEventAttributes(i, calendarId, cal, body, action)
+    result = gapi.call(cal.events(),
+                       func,
+                       conferenceDataVersion=1,
+                       supportsAttachments=True,
+                       calendarId=calendarId,
+                       sendUpdates=sendUpdates,
+                       body=body,
+                       fields='id',
+                       **kwargs)
+    print(f'Event {result["id"]} {action} finished')
+
+
+def _remove_attendee(attendees, remove_email):
+    return [
+        attendee for attendee in attendees
+        if not attendee['email'].lower() == remove_email
+    ]
+
+
+def getEventAttributes(i, calendarId, cal, body, action):
+    # Default to external only so non-Google
+    # calendars are notified of changes
+    sendUpdates = 'externalOnly'
+    action = 'update' if body else 'add'
+    timeZone = None
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg == 'attendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({'email': sys.argv[i + 1]})
+            i += 2
+        elif myarg == 'removeattendee' and action == 'update':
+            remove_email = sys.argv[i + 1].lower()
+            if 'attendees' in body:
+                body['attendees'] = _remove_attendee(body['attendees'],
+                                                     remove_email)
+            i += 2
+        elif myarg == 'optionalattendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({
+                'email': sys.argv[i + 1],
+                'optional': True
+            })
+            i += 2
+        elif myarg == 'anyonecanaddself':
+            body['anyoneCanAddSelf'] = True
+            i += 1
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'replacedescription' and action == 'update':
+            search = sys.argv[i + 1]
+            replace = sys.argv[i + 2]
+            if 'description' in body:
+                body['description'] = re.sub(search, replace,
+                                             body['description'])
+            i += 3
+        elif myarg == 'start':
+            if sys.argv[i + 1].lower() == 'allday':
+                body['start'] = {'date': utils.get_yyyymmdd(sys.argv[i + 2])}
+                i += 3
+            else:
+                start_time = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+                body['start'] = {'dateTime': start_time}
+                i += 2
+        elif myarg == 'end':
+            if sys.argv[i + 1].lower() == 'allday':
+                body['end'] = {'date': utils.get_yyyymmdd(sys.argv[i + 2])}
+                i += 3
+            else:
+                end_time = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+                body['end'] = {'dateTime': end_time}
+                i += 2
+        elif myarg == 'guestscantinviteothers':
+            body['guestsCanInviteOthers'] = False
+            i += 1
+        elif myarg == 'guestscaninviteothers':
+            body['guestsCanInviteTohters'] = gam.getBoolean(
+                sys.argv[i + 1], 'guestscaninviteothers')
+            i += 2
+        elif myarg == 'guestscantseeothers':
+            body['guestsCanSeeOtherGuests'] = False
+            i += 1
+        elif myarg == 'guestscanseeothers':
+            body['guestsCanSeeOtherGuests'] = gam.getBoolean(
+                sys.argv[i + 1], 'guestscanseeothers')
+            i += 2
+        elif myarg == 'guestscanmodify':
+            body['guestsCanModify'] = gam.getBoolean(sys.argv[i + 1],
+                                                     'guestscanmodify')
+            i += 2
+        elif myarg == 'id':
+            if action == 'update':
+                controlflow.invalid_argument_exit(
+                    'id', 'gam calendar <calendar> updateevent')
+            body['id'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'available':
+            body['transparency'] = 'transparent'
+            i += 1
+        elif myarg == 'transparency':
+            validTransparency = ['opaque', 'transparent']
+            if sys.argv[i + 1].lower() in validTransparency:
+                body['transparency'] = sys.argv[i + 1].lower()
+            else:
+                controlflow.expected_argument_exit('transparency',
+                                                   ', '.join(validTransparency),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg == 'visibility':
+            validVisibility = ['default', 'public', 'private']
+            if sys.argv[i + 1].lower() in validVisibility:
+                body['visibility'] = sys.argv[i + 1].lower()
+            else:
+                controlflow.expected_argument_exit('visibility',
+                                                   ', '.join(validVisibility),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg == 'tentative':
+            body['status'] = 'tentative'
+            i += 1
+        elif myarg == 'status':
+            validStatus = ['confirmed', 'tentative', 'cancelled']
+            if sys.argv[i + 1].lower() in validStatus:
+                body['status'] = sys.argv[i + 1].lower()
+            else:
+                controlflow.expected_argument_exit('visibility',
+                                                   ', '.join(validStatus),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg == 'source':
+            body['source'] = {'title': sys.argv[i + 1], 'url': sys.argv[i + 2]}
+            i += 3
+        elif myarg == 'noreminders':
+            body['reminders'] = {'useDefault': False}
+            i += 1
+        elif myarg == 'reminder':
+            minutes = \
+            gam.getInteger(sys.argv[i+1], myarg, minVal=0,
+                                maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+            reminder = {'minutes': minutes, 'method': sys.argv[i + 2]}
+            body.setdefault('reminders', {'overrides': [], 'useDefault': False})
+            body['reminders']['overrides'].append(reminder)
+            i += 3
+        elif myarg == 'recurrence':
+            body.setdefault('recurrence', [])
+            body['recurrence'].append(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'privateproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['private'][sys.argv[i +
+                                                           1]] = sys.argv[i + 2]
+            i += 3
+        elif myarg == 'sharedproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['shared'][sys.argv[i + 1]] = sys.argv[i +
+                                                                             2]
+            i += 3
+        elif myarg == 'colorindex':
+            body['colorId'] = gam.getInteger(sys.argv[i + 1], myarg,
+                                             CALENDAR_EVENT_MIN_COLOR_INDEX,
+                                             CALENDAR_EVENT_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'hangoutsmeet':
+            body['conferenceData'] = {
+                'createRequest': {
+                    'requestId': f'{str(uuid.uuid4())}'
+                }
+            }
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {action}event')
+    if ('recurrence' in body) and (('start' in body) or ('end' in body)):
+        if not timeZone:
+            timeZone = gapi.call(cal.calendars(),
+                                 'get',
+                                 calendarId=calendarId,
+                                 fields='timeZone')['timeZone']
+        if 'start' in body:
+            body['start']['timeZone'] = timeZone
+        if 'end' in body:
+            body['end']['timeZone'] = timeZone
+    return (sendUpdates, body)
+
+
+def modifySettings():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'timezone':
+            body['timeZone'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam calendar <email> modify')
+    gapi.call(cal.calendars(), 'patch', calendarId=calendarId, body=body)
+
+
+def changeAttendees(users):
+    do_it = True
+    i = 5
+    allevents = False
+    start_date = end_date = None
+    while len(sys.argv) > i:
+        myarg = sys.argv[i].lower()
+        if myarg == 'csv':
+            csv_file = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'dryrun':
+            do_it = False
+            i += 1
+        elif myarg == 'start':
+            start_date = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'end':
+            end_date = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'allevents':
+            allevents = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], 'gam <users> update calattendees')
+    attendee_map = {}
+    f = fileutils.open_file(csv_file)
+    csvFile = csv.reader(f)
+    for row in csvFile:
+        attendee_map[row[0].lower()] = row[1].lower()
+    fileutils.close_file(f)
+    for user in users:
+        sys.stdout.write(f'Checking user {user}\n')
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        page_token = None
+        while True:
+            events_page = gapi.call(cal.events(),
+                                    'list',
+                                    calendarId=user,
+                                    pageToken=page_token,
+                                    timeMin=start_date,
+                                    timeMax=end_date,
+                                    showDeleted=False,
+                                    showHiddenInvitations=False)
+            print(f'Got {len(events_page.get("items", []))}')
+            for event in events_page.get('items', []):
+                if event['status'] == 'cancelled':
+                    # print u' skipping cancelled event'
+                    continue
+                try:
+                    event_summary = event['summary']
+                except (KeyError, UnicodeEncodeError, UnicodeDecodeError):
+                    event_summary = event['id']
+                try:
+                    organizer = event['organizer']['email'].lower()
+                    if not allevents and organizer != user:
+                        #print(f' skipping not-my-event {event_summary}')
+                        continue
+                except KeyError:
+                    pass  # no email for organizer
+                needs_update = False
+                try:
+                    for attendee in event['attendees']:
+                        try:
+                            if attendee['email'].lower() in attendee_map:
+                                old_email = attendee['email'].lower()
+                                new_email = attendee_map[
+                                    attendee['email'].lower()]
+                                print(f' SWITCHING attendee {old_email} to ' \
+                                    f'{new_email} for {event_summary}')
+                                event['attendees'].remove(attendee)
+                                event['attendees'].append({'email': new_email})
+                                needs_update = True
+                        except KeyError:  # no email for that attendee
+                            pass
+                except KeyError:
+                    continue  # no attendees
+                if needs_update:
+                    body = {}
+                    body['attendees'] = event['attendees']
+                    print(f'UPDATING {event_summary}')
+                    if do_it:
+                        gapi.call(cal.events(),
+                                  'patch',
+                                  calendarId=user,
+                                  eventId=event['id'],
+                                  sendNotifications=False,
+                                  body=body)
+                    else:
+                        print(' not pulling the trigger.')
+                # else:
+                #  print(f' no update needed for {event_summary}')
+            try:
+                page_token = events_page['nextPageToken']
+            except KeyError:
+                break
+
+
+def deleteCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    for user in users:
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        gapi.call(cal.calendarList(),
+                  'delete',
+                  soft_errors=True,
+                  calendarId=calendarId)
+
+
+CALENDAR_REMINDER_MAX_MINUTES = 40320
+
+CALENDAR_MIN_COLOR_INDEX = 1
+CALENDAR_MAX_COLOR_INDEX = 24
+
+CALENDAR_EVENT_MIN_COLOR_INDEX = 1
+CALENDAR_EVENT_MAX_COLOR_INDEX = 11
+
+
+def getCalendarAttributes(i, body, function):
+    colorRgbFormat = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'selected':
+            body['selected'] = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg == 'hidden':
+            body['hidden'] = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg == 'summary':
+            body['summaryOverride'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'colorindex':
+            body['colorId'] = gam.getInteger(sys.argv[i + 1],
+                                             myarg,
+                                             minVal=CALENDAR_MIN_COLOR_INDEX,
+                                             maxVal=CALENDAR_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'backgroundcolor':
+            body['backgroundColor'] = gam.getColor(sys.argv[i + 1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'foregroundcolor':
+            body['foregroundColor'] = gam.getColor(sys.argv[i + 1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'reminder':
+            body.setdefault('defaultReminders', [])
+            method = sys.argv[i + 1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_REMINDER_METHODS:
+                    controlflow.expected_argument_exit(
+                        'Method', ', '.join(CALENDAR_REMINDER_METHODS +
+                                            CLEAR_NONE_ARGUMENT), method)
+                minutes = gam.getInteger(sys.argv[i + 2],
+                                         myarg,
+                                         minVal=0,
+                                         maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+                body['defaultReminders'].append({
+                    'method': method,
+                    'minutes': minutes
+                })
+                i += 3
+            else:
+                i += 2
+        elif myarg == 'notification':
+            body.setdefault('notificationSettings', {'notifications': []})
+            method = sys.argv[i + 1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_NOTIFICATION_METHODS:
+                    controlflow.expected_argument_exit(
+                        'Method', ', '.join(CALENDAR_NOTIFICATION_METHODS +
+                                            CLEAR_NONE_ARGUMENT), method)
+                eventType = sys.argv[i + 2].lower()
+                if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP:
+                    controlflow.expected_argument_exit(
+                        'Event', ', '.join(CALENDAR_NOTIFICATION_TYPES_MAP),
+                        eventType)
+                notice = {
+                    'method': method,
+                    'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]
+                }
+                body['notificationSettings']['notifications'].append(notice)
+                i += 3
+            else:
+                i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              f'gam {function} calendar')
+    return colorRgbFormat
+
+
+def addCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    body = {'id': calendarId, 'selected': True, 'hidden': False}
+    colorRgbFormat = getCalendarAttributes(6, body, 'add')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f'Subscribing {user} to calendar {calendarId}{current_count}')
+        gapi.call(cal.calendarList(),
+                  'insert',
+                  soft_errors=True,
+                  body=body,
+                  colorRgbFormat=colorRgbFormat)
+
+
+def updateCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    body = {}
+    colorRgbFormat = getCalendarAttributes(6, body, 'update')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f"Updating {user}'s subscription to calendar ' \
+              f'{calendarId}{current_count}")
+        calId = calendarId if calendarId != 'primary' else user
+        gapi.call(cal.calendarList(),
+                  'patch',
+                  soft_errors=True,
+                  calendarId=calId,
+                  body=body,
+                  colorRgbFormat=colorRgbFormat)
+
+
+def _showCalendar(userCalendar, j, jcount):
+    current_count = display.current_count(j, jcount)
+    summary = userCalendar.get('summaryOverride', userCalendar['summary'])
+    print(f'  Calendar: {userCalendar["id"]}{current_count}')
+    print(f'    Summary: {summary}')
+    print(f'    Description: {userCalendar.get("description", "")}')
+    print(f'    Access Level: {userCalendar["accessRole"]}')
+    print(f'    Timezone: {userCalendar["timeZone"]}')
+    print(f'    Location: {userCalendar.get("location", "")}')
+    print(f'    Hidden: {userCalendar.get("hidden", "False")}')
+    print(f'    Selected: {userCalendar.get("selected", "False")}')
+    print(f'    Color ID: {userCalendar["colorId"]}, ' \
+          f'Background Color: {userCalendar["backgroundColor"]}, ' \
+          f'Foreground Color: {userCalendar["foregroundColor"]}')
+    print(f'    Default Reminders:')
+    for reminder in userCalendar.get('defaultReminders', []):
+        print(f'      Method: {reminder["method"]}, ' \
+              f'Minutes: {reminder["minutes"]}')
+    print('    Notifications:')
+    if 'notificationSettings' in userCalendar:
+        notifications = userCalendar['notificationSettings'].get(
+            'notifications', [])
+        for notification in notifications:
+            print(f'      Method: {notification["method"]}, ' \
+                  f'Type: {notification["type"]}')
+
+
+def infoCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.call(cal.calendarList(),
+                           'get',
+                           soft_errors=True,
+                           calendarId=calendarId)
+        if result:
+            print(f'User: {user}, Calendar:{display.current_count(i, count)}')
+            _showCalendar(result, 1, 1)
+
+
+def printShowCalendars(users, csvFormat):
+    if csvFormat:
+        todrive = False
+        titles = []
+        csvRows = []
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, f"gam <users> {['show', 'print'][csvFormat]} calendars")
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.get_all_pages(cal.calendarList(),
+                                    'list',
+                                    'items',
+                                    soft_errors=True)
+        jcount = len(result)
+        if not csvFormat:
+            print(f'User: {user}, Calendars:{display.current_count(i, count)}')
+            if jcount == 0:
+                continue
+            j = 0
+            for userCalendar in result:
+                j += 1
+                _showCalendar(userCalendar, j, jcount)
+        else:
+            if jcount == 0:
+                continue
+            for userCalendar in result:
+                row = {'primaryEmail': user}
+                display.add_row_titles_to_csv_file(
+                    utils.flatten_json(userCalendar, flattened=row), csvRows,
+                    titles)
+    if csvFormat:
+        display.sort_csv_titles(['primaryEmail', 'id'], titles)
+        display.write_csv_file(csvRows, titles, 'Calendars', todrive)
+
+
+def showCalSettings(users):
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        feed = gapi.get_all_pages(cal.settings(),
+                                  'list',
+                                  'items',
+                                  soft_errors=True)
+        if feed:
+            current_count = display.current_count(i, count)
+            print(f'User: {user}, Calendar Settings:{current_count}')
+            settings = {}
+            for setting in feed:
+                settings[setting['id']] = setting['value']
+            for attr, value in sorted(settings.items()):
+                print(f'  {attr}: {value}')
+
+
+def transferSecCals(users):
+    target_user = sys.argv[5]
+    remove_source_user = sendNotifications = True
+    i = 6
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'keepuser':
+            remove_source_user = False
+            i += 1
+        elif myarg == 'sendnotifications':
+            sendNotifications = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam <users> transfer seccals')
+    if remove_source_user:
+        target_user, target_cal = buildCalendarGAPIObject(target_user)
+        if not target_cal:
+            return
+    for user in users:
+        user, source_cal = buildCalendarGAPIObject(user)
+        if not source_cal:
+            continue
+        calendars = gapi.get_all_pages(source_cal.calendarList(),
+                                       'list',
+                                       'items',
+                                       soft_errors=True,
+                                       minAccessRole='owner',
+                                       showHidden=True,
+                                       fields='items(id),nextPageToken')
+        for calendar in calendars:
+            calendarId = calendar['id']
+            if calendarId.find('@group.calendar.google.com') != -1:
+                body = {
+                    'role': 'owner',
+                    'scope': {
+                        'type': 'user',
+                        'value': target_user
+                    }
+                }
+                gapi.call(source_cal.acl(),
+                          'insert',
+                          calendarId=calendarId,
+                          body=body,
+                          sendNotifications=sendNotifications)
+                if remove_source_user:
+                    body = {
+                        'role': 'none',
+                        'scope': {
+                            'type': 'user',
+                            'value': user
+                        }
+                    }
+                    gapi.call(target_cal.acl(),
+                              'insert',
+                              calendarId=calendarId,
+                              body=body,
+                              sendNotifications=sendNotifications)

src/gam/gapi/cloudidentity/__init__.py

@@ -0,0 +1,9 @@
+import gam
+
+
+def build(api='cloudidentity'):
+    return gam.buildGAPIObject(api)
+
+def build_dwd(api='cloudidentity'):
+    admin = gam._get_admin_email()
+    return gam.buildGAPIServiceObject(api, admin, True)

src/gam/gapi/cloudidentity/devices.py

@@ -0,0 +1,297 @@
+import csv
+import sys
+
+import googleapiclient
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
+from gam.gapi import errors as gapi_errors
+from gam.gapi import cloudidentity as gapi_cloudidentity
+from gam.gapi.directory import customer as gapi_directory_customer
+from gam.gapi.directory import groups as gapi_directory_groups
+
+
+def create():
+    ci = gapi_cloudidentity.build_dwd()
+    customer = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    device_types = gapi.get_enum_values_minus_unspecified(
+        ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1Device']['properties']['deviceType']['enum'])
+    body = {}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'serialnumber':
+            body['serialNumber'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'devicetype':
+            body['deviceType'] = sys.argv[i+1].upper()
+            if body['deviceType'] not in device_types:
+                controlflow.expected_argument_exit('device_type',
+                                                   ', '.join(device_types),
+                                                   sys.argv[i+1])
+            i += 2
+        else:
+           controlflow.invalid_argument_exit(sys.argv[i], 'gam create device') 
+    if not body.get('serialNumber') or not body.get('deviceType'):
+        controlflow.system_error_exit(
+            3, 'serial_number and device_type are required arguments for "gam create device".') 
+    result = gapi.call(ci.devices(), 'create', customer=customer, body=body)
+    print(f'Created device {result["response"]["name"]}')
+
+def info():
+    ci = gapi_cloudidentity.build_dwd()
+    customer = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    name = sys.argv[3]
+    if not name.startswith('devices/'):
+        name = f'devices/{name}'
+    device = gapi.call(ci.devices(), 'get', name=name, customer=customer)
+    device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
+        'deviceUsers', parent=name, customer=customer)
+    display.print_json(device)
+    print('Device Users:')
+    display.print_json(device_users)
+
+def _generic_action(action, device_user=False):
+    ci = gapi_cloudidentity.build_dwd()
+    customer = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    
+    # bah, inconsistencies in API
+    if action == 'delete':
+        kwargs = {'customer': customer}
+    else:
+        kwargs = {'body': {'customer': customer}}
+
+    if device_user:
+        endpoint = ci.devices().deviceUsers()
+    else:
+        endpoint = ci.devices()
+    name = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        # The API calls it "name" but GAM will expose as "id" to avoid admin confusion.
+        if myarg == 'id':
+            name = sys.argv[i+1]
+            if not name.startswith('devices/'):
+                name = f'devices/{name}'
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], f'gam {action} device')
+    if not name:
+        controlflow.system_error_exit(3, f'id is a required argument for "gam {action} device".')
+    op = gapi.call(endpoint, action, name=name, **kwargs)
+    print(op) 
+
+def delete():
+     _generic_action('delete')
+
+def cancel_wipe():
+     _generic_action('cancelWipe')
+
+def wipe():
+     _generic_action('wipe')
+
+def approve_user():
+    _generic_action('approve', True)
+
+def block_user():
+    _generic_action('block', True)
+
+def cancel_wipe_user():
+    _generic_action('cancelWipe', True)
+
+def delete_user():
+    _generic_action('delete', True)
+
+def wipe_user():
+    _generic_action('wipe', True)
+
+def print_():
+    ci = gapi_cloudidentity.build_dwd()
+    customer = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    parent = 'devices/-'
+    filter = None
+    get_device_users = True
+    get_device_views = ['COMPANY_INVENTORY', 'USER_ASSIGNED_DEVICES']
+    titles = []
+    csvRows = []
+    todrive = False
+    sortHeaders = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['filter', 'query']:
+          filter = sys.argv[i+1]
+          i += 2
+        elif myarg == 'nocompanydevices':
+            get_device_views.remove('COMPANY_INVENTORY')
+            i += 1
+        elif myarg == 'nopersonaldevices':
+            get_device_views.remove('USER_ASSIGNED_DEVICES')
+            i += 1
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        else:
+           controlflow.invalid_argument_exit(sys.argv[i], 'gam print devices')
+    view_name_map = {
+      'COMPANY_INVENTORY': 'Company Devices',
+      'USER_ASSIGNED_DEVICES': 'Personal Devices',
+      }
+    devices = []
+    for view in get_device_views:
+        view_name = view_name_map.get(view, 'Devices')
+        page_message = gapi.got_total_items_msg(view_name, '...\n')
+        devices += gapi.get_all_pages(ci.devices(), 'list', 'devices',
+            customer=customer, page_message=page_message,
+            pageSize=100, filter=filter, view=view)
+    if get_device_users:
+        page_message = gapi.got_total_items_msg('Device Users', '...\n')
+        device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
+            'deviceUsers', customer=customer, parent=parent,
+            page_message=page_message, pageSize=20, filter=filter)
+        for device_user in device_users:
+            for device in devices:
+                if device_user.get('name').startswith(device.get('name')):
+                    if 'users' not in device:
+                        device['users'] = []
+                    device['users'].append(device_user)
+                    break
+    for device in devices:
+        device = utils.flatten_json(device)
+        for a_key in device:
+            if a_key not in titles:
+                titles.append(a_key)
+        csvRows.append(device)
+    if sortHeaders:
+        display.sort_csv_titles(['name',], titles)
+    display.write_csv_file(csvRows, titles, 'Devices', todrive)
+
+
+def sync():
+    ci = gapi_cloudidentity.build_dwd()
+    device_types = gapi.get_enum_values_minus_unspecified(
+        ci._rootDesc['schemas']['GoogleAppsCloudidentityDevicesV1Device']['properties']['deviceType']['enum'])
+    customer = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    filter = None
+    csv_file = None
+    serialnumber_column = 'serialNumber'
+    devicetype_column = 'deviceType'
+    static_devicetype = None
+    assetid_column = None
+    unassigned_missing_action = 'delete'
+    assigned_missing_action = 'donothing'
+    missing_actions = ['delete', 'wipe', 'donothing']
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['filter', 'query']:
+          filter = sys.argv[i+1]
+          i += 2
+        elif myarg == 'csvfile':
+          csv_file = sys.argv[i+1]
+          i += 2
+        elif myarg == 'serialnumbercolumn':
+          serialnumber_column = sys.argv[i+1]
+          i += 2
+        elif myarg == 'devicetypecolumn':
+          devicetype_column = sys.argv[i+1]
+          i += 2
+        elif myarg == 'staticdevicetype':
+          static_devicetype = sys.argv[i+1].upper()
+          if static_devicetype not in device_types:
+                controlflow.expected_argument_exit('device_type',
+                                                   ', '.join(device_types),
+                                                   sys.argv[i+1])
+          i += 2
+        elif myarg == 'assetidcolumn':
+          assetid_column = sys.argv[i+1]
+          i += 2
+        elif myarg == 'unassigned_missing_action':
+          unassigned_missing_action = sys.argv[i+1].lower().replace('_', '')
+          if unassigned_missing_action not in missing_actions:
+              controlflow.expected_argument_exit('unassigned_missing_action',
+                                                   ', '.join(missing_actions),
+                                                   sys.argv[i+1])
+          i += 2
+        elif myarg == 'assigned_missing_action':
+          assigned_missing_action = sys.argv[i+1].lower().replace('_', '')
+          if assigned_missing_action not in missing_actions:
+              controlflow.expected_argument_exit('assigned_missing_action',
+                                                   ', '.join(missing_actions),
+                                                   sys.argv[i+1])
+          i += 2
+        else:
+           controlflow.invalid_argument_exit(sys.argv[i], 'gam sync devices')
+    f = fileutils.open_file(csv_file)
+    input_file = csv.DictReader(f, restval='')
+    if serialnumber_column not in input_file.fieldnames:
+        controlflow.csv_field_error_exit(serialnumber_column, input_file.fieldnames)
+    if not static_devicetype and devicetype_column not in input_file.fieldnames:
+        controlflow.csv_field_error_exit(devicetype_column, input_file.fieldnames)
+    if assetid_column and assetid_column not in input_file.fieldnames:
+        controlflow.csv_field_error_exit(assetid_column, input_file.fieldnames)
+    local_devices = []
+    for row in input_file:
+        # upper() is very important to comparison since Google
+        # always return uppercase serials
+        serialnumber = row[serialnumber_column].strip().upper()
+        local_device = {'serialNumber': serialnumber}
+        if static_devicetype:
+            local_device['deviceType'] = static_devicetype
+        else:
+            local_device['deviceType'] = row[devicetype_column].strip()
+        if assetid_column:
+            local_device['assetTag'] = row[assetid_column].strip()
+        local_devices.append(local_device)
+    fileutils.close_file(f)
+    page_message = gapi.got_total_items_msg('Company Devices', '...\n')
+    device_fields = ['serialNumber', 'deviceType', 'lastSyncTime', 'name']
+    if assetid_column:
+       device_fields.append('assetTag')
+    fields = f'nextPageToken,devices({",".join(device_fields)})'
+    remote_devices = gapi.get_all_pages(ci.devices(), 'list', 'devices',
+            customer=customer, page_message=page_message,
+            pageSize=100, filter=filter, view='COMPANY_INVENTORY', fields=fields)
+    remote_device_map = {}
+    for remote_device in remote_devices:
+        sn = remote_device['serialNumber']
+        last_sync = remote_device.pop('lastSyncTime')
+        name = remote_device.pop('name')
+        remote_device_map[sn] = {'name': name}
+        if last_sync == '1970-01-01T00:00:00Z':
+            remote_device_map[sn]['unassigned'] = True
+    devices_to_add = [device for device in local_devices if device not in remote_devices]
+    missing_devices = [device for device in remote_devices if device not in local_devices]
+    print(f'Need to add {len(devices_to_add)} and remove {len(missing_devices)} devices...')
+    for add_device in devices_to_add:
+        print(f'Creating {add_device["serialNumber"]}')
+        try:
+            result = gapi.call(ci.devices(), 'create', customer=customer,
+              throw_reasons=[gapi_errors.ErrorReason.FOUR_O_NINE], body=add_device)
+            print(f' created {result["response"]["deviceType"]} device {result["response"]["name"]} with serial {result["response"]["serialNumber"]}')
+        except googleapiclient.errors.HttpError:
+            print(f' {add_device["serialNumber"]} already exists')
+    for missing_device in missing_devices:
+        sn = missing_device['serialNumber']
+        name = remote_device_map[sn]['name']
+        unassigned = remote_device_map[sn].get('unassigned')
+        action = unassigned_missing_action if unassigned else assigned_missing_action
+        if action == 'donothing':
+            pass
+        else:
+            if action == 'delete':
+                kwargs = {'customer': customer}
+            else:
+                kwargs = {'body': {'customer': customer}}
+            gapi.call(ci.devices(), unassigned_missing_action,
+                name=name, **kwargs)
+            print(f'{action}d {sn}')

src/gam/gapi/cloudidentity/groups.py

@@ -0,0 +1,752 @@
+import csv
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam import utils
+from gam.gapi import errors as gapi_errors
+from gam.gapi import cloudidentity as gapi_cloudidentity
+from gam.gapi.directory import customer as gapi_directory_customer
+from gam.gapi.directory import groups as gapi_directory_groups
+
+
+def create():
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    initialGroupConfig = 'EMPTY'
+    gapi_directory_customer.setTrueCustomerId()
+    parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    body = {
+        'groupKey': {
+            'id': gam.normalizeEmailAddressOrUID(sys.argv[3], noUid=True)
+        },
+        'parent': parent,
+        'labels': {
+            'cloudidentity.googleapis.com/groups.discussion_forum': ''
+        },
+    }
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['displayName'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['alias', 'aliases']:
+            # As of 2020/06/25 this doesn't work (yet?)
+            aliases = sys.argv[i + 1].split(' ')
+            body['additionalGroupKeys'] = []
+            for alias in aliases:
+                body['additionalGroupKeys'].append({'id': alias})
+            i += 2
+        elif myarg in ['dynamic']:
+            # As of 2020/06/25 this doesn't work (yet?)
+            body['dynamicGroupMetadata'] = {
+                'queries': [{
+                    'query': sys.argv[i + 1],
+                    'resourceType': 'USER'
+                }]
+            }
+            i += 2
+        elif myarg in ['makeowner']:
+            initialGroupConfig = 'WITH_INITIAL_OWNER'
+            i += 1
+        else:
+            print('should not get here')
+            sys.exit(5)
+    print(f'Creating group {body["groupKey"]["id"]}')
+    gapi.call(ci.groups(),
+              'create',
+              initialGroupConfig=initialGroupConfig,
+              body=body)
+
+
+def delete():
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    group = sys.argv[3]
+    name = group_email_to_id(ci, group)
+    print(f'Deleting group {group}')
+    gapi.call(ci.groups(), 'delete', name=name)
+
+
+def info():
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    group = gam.normalizeEmailAddressOrUID(sys.argv[3])
+    getUsers = True
+    showJoinDate = True
+    showUpdateDate = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nousers':
+            getUsers = False
+            i += 1
+        elif myarg == 'nojoindate':
+            showJoinDate = False
+            i += 1
+        elif myarg == 'showupdatedate':
+            showUpdateDate = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam info cigroup')
+    name = group_email_to_id(ci, group)
+    basic_info = gapi.call(ci.groups(), 'get', name=name)
+    display.print_json(basic_info)
+    if getUsers:
+        if not showJoinDate and not showUpdateDate:
+            view = 'BASIC'
+            pageSize = 1000
+        else:
+            view = 'FULL'
+            pageSize = 500
+        members = gapi.get_all_pages(ci.groups().memberships(),
+                                     'list',
+                                     'memberships',
+                                     parent=name,
+                                     fields='*',
+                                     pageSize=pageSize,
+                                     view=view)
+        print('Members:')
+        for member in members:
+            role = get_single_role(member.get('roles', [])).lower()
+            email = member.get('memberKey', {}).get('id')
+            jc_string = ''
+            if showJoinDate:
+                joined = member.get('createTime', 'Unknown')
+                jc_string += f'  joined {joined}'
+            if showUpdateDate:
+                updated = member.get('updateTime', 'Unknown')
+                jc_string += f'  updated {updated}'
+            print(
+                f'{role}: {email}{jc_string}'
+                # f' {member.get("role", ROLE_MEMBER).lower()}: {member.get("email", member["id"])} ({member["type"].lower()})'
+            )
+        print(f'Total {len(members)} users in group')
+
+
+def info_member():
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    member = gam.normalizeEmailAddressOrUID(sys.argv[3])
+    group = gam.normalizeEmailAddressOrUID(sys.argv[4])
+    group_name = gapi.call(ci.groups(),
+                           'lookup',
+                           groupKey_id=group,
+                           fields='name').get('name')
+    member_name = gapi.call(ci.groups().memberships(),
+                            'lookup',
+                            parent=group_name,
+                            memberKey_id=member,
+                            fields='name').get('name')
+    member_details = gapi.call(ci.groups().memberships(),
+                               'get',
+                               name=member_name)
+    display.print_json(member_details)
+
+
+UPDATE_GROUP_SUBCMDS = ['add', 'clear', 'delete', 'remove', 'sync', 'update']
+GROUP_ROLES_MAP = {
+    'owner': ROLE_OWNER,
+    'owners': ROLE_OWNER,
+    'manager': ROLE_MANAGER,
+    'managers': ROLE_MANAGER,
+    'member': ROLE_MEMBER,
+    'members': ROLE_MEMBER,
+}
+
+
+def print_():
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    i = 3
+    members = membersCountOnly = managers = managersCountOnly = owners = ownersCountOnly = False
+    gapi_directory_customer.setTrueCustomerId()
+    parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    memberDelimiter = '\n'
+    todrive = False
+    titles = []
+    csvRows = []
+    roles = []
+    sortHeaders = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'delimiter':
+            memberDelimiter = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in ['members', 'memberscount']:
+            roles.append(ROLE_MEMBER)
+            members = True
+            if myarg == 'memberscount':
+                membersCountOnly = True
+            i += 1
+        elif myarg in ['owners', 'ownerscount']:
+            roles.append(ROLE_OWNER)
+            owners = True
+            if myarg == 'ownerscount':
+                ownersCountOnly = True
+            i += 1
+        elif myarg in ['managers', 'managerscount']:
+            roles.append(ROLE_MANAGER)
+            managers = True
+            if myarg == 'managerscount':
+                managersCountOnly = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print cigroups')
+    if roles:
+        if members:
+            display.add_titles_to_csv_file([
+                'MembersCount',
+            ], titles)
+            if not membersCountOnly:
+                display.add_titles_to_csv_file([
+                    'Members',
+                ], titles)
+        if managers:
+            display.add_titles_to_csv_file([
+                'ManagersCount',
+            ], titles)
+            if not managersCountOnly:
+                display.add_titles_to_csv_file([
+                    'Managers',
+                ], titles)
+        if owners:
+            display.add_titles_to_csv_file([
+                'OwnersCount',
+            ], titles)
+            if not ownersCountOnly:
+                display.add_titles_to_csv_file([
+                    'Owners',
+                ], titles)
+    gam.printGettingAllItems('Groups', None)
+    page_message = gapi.got_total_items_first_last_msg('Groups')
+    entityList = gapi.get_all_pages(ci.groups(),
+                                    'list',
+                                    'groups',
+                                    page_message=page_message,
+                                    message_attribute=['groupKey', 'id'],
+                                    parent=parent,
+                                    view='FULL',
+                                    pageSize=500)
+    i = 0
+    count = len(entityList)
+    for groupEntity in entityList:
+        i += 1
+        groupEmail = groupEntity['groupKey']['id']
+        group = utils.flatten_json(groupEntity)
+        for a_key in group:
+            if a_key not in titles:
+                titles.append(a_key)
+        groupKey_id = groupEntity['name']
+        if roles:
+            sys.stderr.write(
+                f' Getting {roles} for {groupEmail}{gam.currentCountNL(i, count)}'
+            )
+            page_message = gapi.got_total_items_first_last_msg('Members')
+            validRoles, _, _ = gam._getRoleVerification(
+                '.'.join(roles), 'nextPageToken,members(email,id,role)')
+            groupMembers = gapi.get_all_pages(ci.groups().memberships(),
+                                              'list',
+                                              'memberships',
+                                              page_message=page_message,
+                                              message_attribute=['memberKey', 'id'],
+                                              soft_errors=True,
+                                              parent=groupKey_id,
+                                              view='BASIC')
+            if members:
+                membersList = []
+                membersCount = 0
+            if managers:
+                managersList = []
+                managersCount = 0
+            if owners:
+                ownersList = []
+                ownersCount = 0
+            for member in groupMembers:
+                member_email = member['memberKey']['id']
+                role = get_single_role(member.get('roles'))
+                if not validRoles or role in validRoles:
+                    if role == ROLE_MEMBER:
+                        if members:
+                            membersCount += 1
+                            if not membersCountOnly:
+                                membersList.append(member_email)
+                    elif role == ROLE_MANAGER:
+                        if managers:
+                            managersCount += 1
+                            if not managersCountOnly:
+                                managersList.append(member_email)
+                    elif role == ROLE_OWNER:
+                        if owners:
+                            ownersCount += 1
+                            if not ownersCountOnly:
+                                ownersList.append(member_email)
+                    elif members:
+                        membersCount += 1
+                        if not membersCountOnly:
+                            membersList.append(member_email)
+            if members:
+                group['MembersCount'] = membersCount
+                if not membersCountOnly:
+                    group['Members'] = memberDelimiter.join(membersList)
+            if managers:
+                group['ManagersCount'] = managersCount
+                if not managersCountOnly:
+                    group['Managers'] = memberDelimiter.join(managersList)
+            if owners:
+                group['OwnersCount'] = ownersCount
+                if not ownersCountOnly:
+                    group['Owners'] = memberDelimiter.join(ownersList)
+        csvRows.append(group)
+    if sortHeaders:
+        display.sort_csv_titles([
+            'Email',
+        ], titles)
+    display.write_csv_file(csvRows, titles, 'Groups', todrive)
+
+
+def print_members():
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    todrive = False
+    gapi_directory_customer.setTrueCustomerId()
+    parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    roles = []
+    titles = ['group']
+    csvRows = []
+    groups_to_get = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['role', 'roles']:
+            for role in sys.argv[i + 1].lower().replace(',', ' ').split():
+                if role in GROUP_ROLES_MAP:
+                    roles.append(GROUP_ROLES_MAP[role])
+                else:
+                    controlflow.system_error_exit(
+                        2,
+                        f'{role} is not a valid role for "gam print group-members {myarg}"'
+                    )
+            i += 2
+        elif myarg in ['cigroup', 'cigroups']:
+            group_email = gam.normalizeEmailAddressOrUID(sys.argv[i + 1])
+            groups_to_get = [group_email]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print cigroup-members')
+    if not groups_to_get:
+        gam.printGettingAllItems('Groups', None)
+        page_message = gapi.got_total_items_first_last_msg('Groups')
+        groups_to_get = gapi.get_all_pages(
+            ci.groups(),
+            'list',
+            'groups',
+            message_attribute=['groupKey', 'id'],
+            page_message=page_message,
+            parent=parent,
+            view='BASIC',
+            pageSize=1000,
+            fields='nextPageToken,groups(groupKey(id))')
+        groups_to_get = [group['groupKey']['id'] for group in groups_to_get]
+    i = 0
+    count = len(groups_to_get)
+    for group_email in groups_to_get:
+        i += 1
+
+        sys.stderr.write(
+            f'Getting members for {group_email}{gam.currentCountNL(i, count)}')
+        group_id = group_email_to_id(ci, group_email)
+        print(f'Getting members of cigroup {group_email}...')
+        page_message = f' {gapi.got_total_items_first_last_msg("Members")}'
+        group_members = gapi.get_all_pages(
+            ci.groups().memberships(),
+            'list',
+            'memberships',
+            soft_errors=True,
+            parent=group_id,
+            view='FULL',
+            pageSize=500,
+            page_message=page_message,
+            message_attribute=['memberKey', 'id'])
+        #fields='nextPageToken,memberships(memberKey,roles,createTime,updateTime)')
+        if roles:
+            group_members = filter_members_to_roles(group_members, roles)
+        for member in group_members:
+            # reduce role to a single value
+            member['role'] = get_single_role(member.pop('roles'))
+            member = utils.flatten_json(member)
+            for title in member:
+                if title not in titles:
+                    titles.append(title)
+            member['group'] = group_email
+            csvRows.append(member)
+    display.write_csv_file(csvRows, titles, 'Group Members', todrive)
+
+
+def update():
+
+    # Convert foo@googlemail.com to foo@gmail.com; eliminate periods in name for foo.bar@gmail.com
+    def _cleanConsumerAddress(emailAddress, mapCleanToOriginal):
+        atLoc = emailAddress.find('@')
+        if atLoc > 0:
+            if emailAddress[atLoc + 1:] in ['gmail.com', 'googlemail.com']:
+                cleanEmailAddress = emailAddress[:atLoc].replace(
+                    '.', '') + '@gmail.com'
+                if cleanEmailAddress != emailAddress:
+                    mapCleanToOriginal[cleanEmailAddress] = emailAddress
+                    return cleanEmailAddress
+        return emailAddress
+
+    def _getRoleAndUsers():
+        checkSuspended = None
+        role = None
+        i = 5
+        if sys.argv[i].lower() in GROUP_ROLES_MAP:
+            role = GROUP_ROLES_MAP[sys.argv[i].lower()]
+            i += 1
+        if sys.argv[i].lower() in ['suspended', 'notsuspended']:
+            checkSuspended = sys.argv[i].lower() == 'suspended'
+            i += 1
+        if sys.argv[i].lower() in usergroup_types:
+            users_email = gam.getUsersToModify(entity_type=sys.argv[i].lower(),
+                                               entity=sys.argv[i + 1],
+                                               checkSuspended=checkSuspended,
+                                               groupUserMembersOnly=False)
+        else:
+            users_email = [
+                gam.normalizeEmailAddressOrUID(sys.argv[i],
+                                               checkForCustomerId=True)
+            ]
+        return (role, users_email)
+
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
+    group = sys.argv[3]
+    myarg = sys.argv[4].lower()
+    items = []
+    if myarg in UPDATE_GROUP_SUBCMDS:
+        group = gam.normalizeEmailAddressOrUID(group)
+        if group.startswith('groups/'):
+            parent = group
+        else:
+            parent = group_email_to_id(ci, group)
+        if not parent:
+            return
+        if myarg == 'add':
+            role, users_email = _getRoleAndUsers()
+            if not role:
+                role = ROLE_MEMBER
+            if len(users_email) > 1:
+                sys.stderr.write(
+                    f'Group: {group}, Will add {len(users_email)} {role}s.\n')
+                for user_email in users_email:
+                    item = [
+                        'gam', 'update', 'cigroup', f'id:{parent}', 'add', role,
+                        user_email
+                    ]
+                    items.append(item)
+            elif len(users_email) > 0:
+                body = {
+                    'memberKey': {
+                        'id': users_email[0]
+                    },
+                    'roles': [{
+                        'name': ROLE_MEMBER
+                    }]
+                }
+                if role != ROLE_MEMBER:
+                    body['roles'].append({'name': role})
+                add_text = [f'as {role}']
+                for i in range(2):
+                    try:
+                        gapi.call(
+                            ci.groups().memberships(),
+                            'create',
+                            throw_reasons=[
+                                gapi_errors.ErrorReason.FOUR_O_NINE,
+                                gapi_errors.ErrorReason.MEMBER_NOT_FOUND,
+                                gapi_errors.ErrorReason.RESOURCE_NOT_FOUND,
+                                gapi_errors.ErrorReason.INVALID_MEMBER,
+                                gapi_errors.ErrorReason.
+                                CYCLIC_MEMBERSHIPS_NOT_ALLOWED
+                            ],
+                            parent=parent,
+                            body=body)
+                        print(
+                            f' Group: {group}, {users_email[0]} Added {" ".join(add_text)}'
+                        )
+                        break
+                    except (gapi_errors.GapiMemberNotFoundError,
+                            gapi_errors.GapiResourceNotFoundError,
+                            gapi_errors.GapiInvalidMemberError,
+                            gapi_errors.GapiCyclicMembershipsNotAllowedError
+                           ) as e:
+                        print(
+                            f' Group: {group}, {users_email[0]} Add {" ".join(add_text)} Failed: {str(e)}'
+                        )
+                        break
+        elif myarg == 'sync':
+            syncMembersSet = set()
+            syncMembersMap = {}
+            role, users_email = _getRoleAndUsers()
+            for user_email in users_email:
+                if user_email in ('*', GC_Values[GC_CUSTOMER_ID]):
+                    syncMembersSet.add(GC_Values[GC_CUSTOMER_ID])
+                else:
+                    syncMembersSet.add(
+                        _cleanConsumerAddress(user_email.lower(),
+                                              syncMembersMap))
+            currentMembersSet = set()
+            currentMembersMap = {}
+            for current_email in gam.getUsersToModify(
+                    entity_type='cigroup',
+                    entity=group,
+                    member_type=role,
+                    groupUserMembersOnly=False):
+                if current_email == GC_Values[GC_CUSTOMER_ID]:
+                    currentMembersSet.add(current_email)
+                else:
+                    currentMembersSet.add(
+                        _cleanConsumerAddress(current_email.lower(),
+                                              currentMembersMap))
+            to_add = [
+                syncMembersMap.get(emailAddress, emailAddress)
+                for emailAddress in syncMembersSet - currentMembersSet
+            ]
+            to_remove = [
+                currentMembersMap.get(emailAddress, emailAddress)
+                for emailAddress in currentMembersSet - syncMembersSet
+            ]
+            sys.stderr.write(
+                f'Group: {group}, Will add {len(to_add)} and remove {len(to_remove)} {role}s.\n'
+            )
+            for user in to_add:
+                item = [
+                    'gam', 'update', 'cigroup', f'id:{parent}', 'add', role,
+                    user
+                ]
+                items.append(item)
+            for user in to_remove:
+                items.append([
+                    'gam', 'update', 'cigroup', f'id:{parent}', 'remove', user
+                ])
+        elif myarg in ['delete', 'remove']:
+            _, users_email = _getRoleAndUsers()
+            if len(users_email) > 1:
+                sys.stderr.write(
+                    f'Group: {group}, Will remove {len(users_email)} emails.\n')
+                for user_email in users_email:
+                    items.append([
+                        'gam', 'update', 'cigroup', f'id:{parent}', 'remove',
+                        user_email
+                    ])
+            elif len(users_email) == 1:
+                name = membership_email_to_id(ci, parent, users_email[0])
+                try:
+                    gapi.call(ci.groups().memberships(),
+                              'delete',
+                              throw_reasons=[
+                                  gapi_errors.ErrorReason.MEMBER_NOT_FOUND,
+                                  gapi_errors.ErrorReason.INVALID_MEMBER
+                              ],
+                              name=name)
+                    print(f' Group: {group}, {users_email[0]} Removed')
+                except (gapi_errors.GapiMemberNotFoundError,
+                        gapi_errors.GapiInvalidMemberError) as e:
+                    print(
+                        f' Group: {group}, {users_email[0]} Remove Failed: {str(e)}'
+                    )
+        elif myarg == 'update':
+            role, users_email = _getRoleAndUsers()
+            if not role:
+                role = ROLE_MEMBER
+            if len(users_email) > 1:
+                sys.stderr.write(
+                    f'Group: {group}, Will update {len(users_email)} {role}s.\n'
+                )
+                for user_email in users_email:
+                    item = [
+                        'gam', 'update', 'cigroup', f'id:{parent}', 'update',
+                        role, user_email
+                    ]
+                    items.append(item)
+            elif len(users_email) > 0:
+                name = membership_email_to_id(ci, parent, users_email[0])
+                addRoles = []
+                removeRoles = []
+                new_role = {'role': role}
+                current_roles = gapi.call(ci.groups().memberships(),
+                                          'get',
+                                          name=name,
+                                          fields='roles').get('roles', [])
+                current_roles = [role['name'] for role in current_roles]
+                for crole in current_roles:
+                    if crole != ROLE_MEMBER and crole != role:
+                        removeRoles.append(crole)
+                if role not in current_roles:
+                    addRoles.append({'name': role})
+                bodys = []
+                if addRoles:
+                    bodys.append({'addRoles': addRoles})
+                if removeRoles:
+                    bodys.append({'removeRoles': removeRoles})
+                for body in bodys:
+                    try:
+                        gapi.call(ci.groups().memberships(),
+                                  'modifyMembershipRoles',
+                                  throw_reasons=[
+                                      gapi_errors.ErrorReason.MEMBER_NOT_FOUND,
+                                      gapi_errors.ErrorReason.INVALID_MEMBER
+                                  ],
+                                  name=name,
+                                  body=body)
+                    except (gapi_errors.GapiMemberNotFoundError,
+                            gapi_errors.GapiInvalidMemberError) as e:
+                        print(
+                            f' Group: {group}, {users_email[0]} Update to {role} Failed: {str(e)}'
+                        )
+                        break
+                print(
+                  f' Group: {group}, {users_email[0]} Updated to {role}'
+                )
+
+        else:  # clear
+            roles = []
+            i = 5
+            while i < len(sys.argv):
+                myarg = sys.argv[i].lower()
+                if myarg.upper() in [ROLE_OWNER, ROLE_MANAGER, ROLE_MEMBER]:
+                    roles.append(myarg.upper())
+                    i += 1
+                else:
+                    controlflow.invalid_argument_exit(
+                        sys.argv[i], 'gam update cigroup clear')
+            if not roles:
+                roles = [ROLE_MEMBER]
+            group = gam.normalizeEmailAddressOrUID(group)
+            member_type_message = f'{",".join(roles).lower()}s'
+            sys.stderr.write(
+                f'Getting {member_type_message} of {group} (may take some time for large groups)...\n'
+            )
+            page_message = gapi.got_total_items_msg(f'{member_type_message}',
+                                                    '...')
+            try:
+                result = gapi.get_all_pages(
+                    ci.groups().memberships(),
+                    'list',
+                    'memberships',
+                    page_message=page_message,
+                    throw_reasons=gapi_errors.MEMBERS_THROW_REASONS,
+                    parent=parent,
+                    fields='nextPageToken,memberships(memberKey,roles)')
+                result = filter_members_to_roles(result, roles)
+                if not result:
+                    print('Group already has 0 members')
+                    return
+                users_email = [member['memberKey']['id'] for member in result]
+                sys.stderr.write(
+                    f'Group: {group}, Will remove {len(users_email)} {", ".join(roles).lower()}s.\n'
+                )
+                for user_email in users_email:
+                    items.append([
+                        'gam', 'update', 'cigroup', group, 'remove', user_email
+                    ])
+            except (gapi_errors.GapiGroupNotFoundError,
+                    gapi_errors.GapiDomainNotFoundError,
+                    gapi_errors.GapiInvalidError,
+                    gapi_errors.GapiForbiddenError):
+                gam.entityUnknownWarning('Group', group, 0, 0)
+        if items:
+            gam.run_batch(items)
+    else:
+        i = 4
+        body = {}
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower().replace('_', '')
+            if myarg == 'name':
+                body['displayName'] = sys.argv[i + 1]
+                i += 2
+            elif myarg == 'description':
+                body['description'] = sys.argv[i + 1]
+                i += 2
+            elif myarg == 'security':
+                body['labels'] = {
+                    'cloudidentity.googleapis.com/groups.security': '',
+                    'cloudidentity.googleapis.com/groups.discussion_forum': ''
+                }
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i],
+                                                  'gam update cigroup')
+        updateMask = ','.join(body.keys())
+        name = group_email_to_id(ci, group)
+        print(f'Updating group {group}')
+        gapi.call(ci.groups(),
+                  'patch',
+                  updateMask=updateMask,
+                  name=name,
+                  body=body)
+
+
+def group_email_to_id(ci, group, i=0, count=0):
+    group = gam.normalizeEmailAddressOrUID(group)
+    try:
+        return gapi.call(ci.groups(),
+                         'lookup',
+                         throw_reasons=gapi_errors.GROUP_GET_THROW_REASONS,
+                         retry_reasons=gapi_errors.GROUP_GET_RETRY_REASONS,
+                         groupKey_id=group,
+                         fields='name').get('name')
+    except (gapi_errors.GapiGroupNotFoundError,
+            gapi_errors.GapiDomainNotFoundError,
+            gapi_errors.GapiDomainCannotUseApisError,
+            gapi_errors.GapiForbiddenError, gapi_errors.GapiBadRequestError):
+        gam.entityUnknownWarning('Group', group, i, count)
+        return None
+
+
+def membership_email_to_id(ci, parent, membership, i=0, count=0):
+    membership = gam.normalizeEmailAddressOrUID(membership)
+    try:
+        return gapi.call(ci.groups().memberships(),
+                         'lookup',
+                         throw_reasons=gapi_errors.GROUP_GET_THROW_REASONS,
+                         retry_reasons=gapi_errors.GROUP_GET_RETRY_REASONS,
+                         parent=parent,
+                         memberKey_id=membership,
+                         fields='name').get('name')
+    except (gapi_errors.GapiGroupNotFoundError,
+            gapi_errors.GapiDomainNotFoundError,
+            gapi_errors.GapiDomainCannotUseApisError,
+            gapi_errors.GapiForbiddenError, gapi_errors.GapiBadRequestError):
+        gam.entityUnknownWarning('Membership', membership, i, count)
+        return None
+
+
+def get_single_role(roles):
+    ''' returns the highest role of member '''
+    roles = [role.get('name') for role in roles]
+    if not roles:
+        return ROLE_MEMBER
+    for a_role in [ROLE_OWNER, ROLE_MANAGER, ROLE_MEMBER]:
+        if a_role in roles:
+            return a_role
+    return roles[0]
+
+
+def filter_members_to_roles(members, roles):
+    filtered_members = []
+    for member in members:
+        role = get_single_role(member.get('roles', []))
+        if role in roles:
+            filtered_members.append(member)
+    return filtered_members

src/gam/gapi/directory/__init__.py

@@ -0,0 +1,5 @@
+import gam
+
+
+def build():
+    return gam.buildGAPIObject('directory')

src/gam/gapi/directory/asps.py

@@ -0,0 +1,58 @@
+import sys
+
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def info(users):
+    cd = gapi_directory.build()
+    for user in users:
+        asps = gapi.get_items(cd.asps(), 'list', 'items', userKey=user)
+        if asps:
+            print(f'Application-Specific Passwords for {user}')
+            for asp in asps:
+                if asp['creationTime'] == '0':
+                    created_date = 'Unknown'
+                else:
+                    created_date = utils.formatTimestampYMDHMS(
+                        asp['creationTime'])
+                if asp['lastTimeUsed'] == '0':
+                    used_date = 'Never'
+                else:
+                    last_used = asp['lastTimeUsed']
+                    used_date = utils.formatTimestampYMDHMS(last_used)
+                print(f' ID: {asp["codeId"]}\n' \
+                      f'  Name: {asp["name"]}\n' \
+                      f'  Created: {created_date}\n' \
+                      f'  Last Used: {used_date}\n')
+        else:
+            print(f' no ASPs for {user}\n')
+
+
+def delete(users, cd=None, codeIdList=None):
+    if not cd:
+        cd = gapi_directory.build()
+    if not codeIdList:
+        codeIdList = sys.argv[5].lower()
+    if codeIdList == 'all':
+        allCodeIds = True
+    else:
+        allCodeIds = False
+        codeIds = codeIdList.replace(',', ' ').split()
+    for user in users:
+        if allCodeIds:
+            print(f'Getting Application Specific Passwords for {user}')
+            asps = gapi.get_items(cd.asps(),
+                                  'list',
+                                  'items',
+                                  userKey=user,
+                                  fields='items/codeId')
+            codeIds = [asp['codeId'] for asp in asps]
+        if not codeIds:
+            print('No ASPs')
+        for codeId in codeIds:
+            gapi.call(cd.asps(), 'delete', userKey=user, codeId=codeId)
+            print(f'deleted ASP {codeId} for {user}')

src/gam/gapi/directory/cros.py

@@ -0,0 +1,832 @@
+import datetime
+
+from gam.var import *
+import gam
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+from gam import utils
+
+
+def doUpdateCros():
+    cd = gapi_directory.build()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    update_body = {}
+    action_body = {}
+    orgUnitPath = None
+    ack_wipe = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'user':
+            update_body['annotatedUser'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'location':
+            update_body['annotatedLocation'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'notes':
+            update_body['notes'] = sys.argv[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg in ['tag', 'asset', 'assetid']:
+            update_body['annotatedAssetId'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['ou', 'org']:
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'action':
+            action = sys.argv[i + 1].lower().replace('_', '').replace('-', '')
+            deprovisionReason = None
+            if action in [
+                    'deprovisionsamemodelreplace',
+                    'deprovisionsamemodelreplacement'
+            ]:
+                action = 'deprovision'
+                deprovisionReason = 'same_model_replacement'
+            elif action in [
+                    'deprovisiondifferentmodelreplace',
+                    'deprovisiondifferentmodelreplacement'
+            ]:
+                action = 'deprovision'
+                deprovisionReason = 'different_model_replacement'
+            elif action in ['deprovisionretiringdevice']:
+                action = 'deprovision'
+                deprovisionReason = 'retiring_device'
+            elif action == 'deprovisionupgradetransfer':
+                action = 'deprovision'
+                deprovisionReason = 'upgrade_transfer'
+            elif action not in ['disable', 'reenable']:
+                controlflow.system_error_exit(2, f'expected action of ' \
+                    f'deprovision_same_model_replace, ' \
+                    f'deprovision_different_model_replace, ' \
+                    f'deprovision_retiring_device, ' \
+                    f'deprovision_upgrade_transfer, disable or reenable,'
+                    f' got {action}')
+            action_body = {'action': action}
+            if deprovisionReason:
+                action_body['deprovisionReason'] = deprovisionReason
+            i += 2
+        elif myarg == 'acknowledgedevicetouchrequirement':
+            ack_wipe = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update cros')
+    i = 0
+    count = len(devices)
+    if action_body:
+        if action_body['action'] == 'deprovision' and not ack_wipe:
+            print(f'WARNING: Refusing to deprovision {count} devices because '
+                  'acknowledge_device_touch_requirement not specified. ' \
+                  'Deprovisioning a device means the device will have to ' \
+                  'be physically wiped and re-enrolled to be managed by ' \
+                  'your domain again. This requires physical access to ' \
+                  'the device and is very time consuming to perform for ' \
+                  'each device. Please add ' \
+                  '"acknowledge_device_touch_requirement" to the GAM ' \
+                  'command if you understand this and wish to proceed ' \
+                  'with the deprovision. Please also be aware that ' \
+                  'deprovisioning can have an effect on your device ' \
+                  'license count. See ' \
+                  'https://support.google.com/chrome/a/answer/3523633 '\
+                  'for full details.')
+            sys.exit(3)
+        for deviceId in devices:
+            i += 1
+            cur_count = gam.currentCount(i, count)
+            print(f' performing action {action} for {deviceId}{cur_count}')
+            gapi.call(cd.chromeosdevices(),
+                      function='action',
+                      customerId=GC_Values[GC_CUSTOMER_ID],
+                      resourceId=deviceId,
+                      body=action_body)
+    else:
+        if update_body:
+            for deviceId in devices:
+                i += 1
+                current_count = gam.currentCount(i, count)
+                print(f' updating {deviceId}{current_count}')
+                gapi.call(cd.chromeosdevices(),
+                          'update',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          deviceId=deviceId,
+                          body=update_body)
+        if orgUnitPath:
+            # split moves into max 50 devices per batch
+            for l in range(0, len(devices), 50):
+                move_body = {'deviceIds': devices[l:l + 50]}
+                print(f' moving {len(move_body["deviceIds"])} devices to ' \
+                      f'{orgUnitPath}')
+                gapi.call(cd.chromeosdevices(),
+                          'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath,
+                          body=move_body)
+
+
+def doGetCrosInfo():
+    cd = gapi_directory.build()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    downloadfile = None
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    projection = None
+    fieldsList = []
+    noLists = False
+    startDate = endDate = None
+    listLimit = 0
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nolists':
+            noLists = True
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            fieldsList = []
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg])
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i + 1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field])
+                    if field in CROS_ACTIVE_TIME_RANGES_ARGUMENTS + \
+                                CROS_DEVICE_FILES_ARGUMENTS + \
+                                CROS_RECENT_USERS_ARGUMENTS:
+                        projection = 'FULL'
+                        noLists = False
+                else:
+                    controlflow.invalid_argument_exit(field,
+                                                      'gam info cros fields')
+            i += 2
+        elif myarg == 'downloadfile':
+            downloadfile = sys.argv[i + 1]
+            if downloadfile.lower() == 'latest':
+                downloadfile = downloadfile.lower()
+            i += 2
+        elif myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i + 1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam info cros')
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = ','.join(set(fieldsList)).replace('.', '/')
+    else:
+        fields = None
+    i = 0
+    device_count = len(devices)
+    for deviceId in devices:
+        i += 1
+        cros = gapi.call(cd.chromeosdevices(),
+                         'get',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         deviceId=deviceId,
+                         projection=projection,
+                         fields=fields)
+        print(f'CrOS Device: {deviceId} ({i} of {device_count})')
+        if 'notes' in cros:
+            cros['notes'] = cros['notes'].replace('\n', '\\n')
+        if 'autoUpdateExpiration' in cros:
+            cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                cros['autoUpdateExpiration'])
+        _checkTPMVulnerability(cros)
+        for up in CROS_SCALAR_PROPERTY_PRINT_ORDER:
+            if up in cros:
+                if isinstance(cros[up], str):
+                    print(f'  {up}: {cros[up]}')
+                else:
+                    sys.stdout.write(f'  {up}:')
+                    display.print_json(cros[up], '  ')
+        if not noLists:
+            activeTimeRanges = _filterTimeRanges(
+                cros.get('activeTimeRanges', []), startDate, endDate)
+            lenATR = len(activeTimeRanges)
+            if lenATR:
+                print('  activeTimeRanges')
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    active_date = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    duration = utils.formatMilliSeconds(active_time)
+                    minutes = active_time // 60000
+                    print(f'    date: {active_date}')
+                    print(f'      activeTime: {active_time}')
+                    print(f'      duration: {duration}')
+                    print(f'      minutes: {minutes}')
+            recentUsers = cros.get('recentUsers', [])
+            lenRU = len(recentUsers)
+            if lenRU:
+                print('  recentUsers')
+                num_ranges = min(lenRU, listLimit or lenRU)
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get('email')
+                    if not useremail:
+                        if recentUser['type'] == 'USER_TYPE_UNMANAGED':
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    print(f'    type: {recentUser["type"]}')
+                    print(f'      email: {useremail}')
+            deviceFiles = _filterCreateReportTime(cros.get('deviceFiles',
+                                                           []), 'createTime',
+                                                  startDate, endDate)
+            lenDF = len(deviceFiles)
+            if lenDF:
+                num_ranges = min(lenDF, listLimit or lenDF)
+                print('  deviceFiles')
+                for deviceFile in deviceFiles[:num_ranges]:
+                    device_type = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    print(f'    {device_type}: {create_time}')
+            if downloadfile:
+                deviceFiles = cros.get('deviceFiles', [])
+                lenDF = len(deviceFiles)
+                if lenDF:
+                    if downloadfile == 'latest':
+                        deviceFile = deviceFiles[-1]
+                    else:
+                        for deviceFile in deviceFiles:
+                            if deviceFile['createTime'] == downloadfile:
+                                break
+                        else:
+                            print(f'ERROR: file {downloadfile} not ' \
+                                  f'available to download.')
+                            deviceFile = None
+                    if deviceFile:
+                        created = deviceFile['createTime']
+                        downloadfile = f'cros-logs-{deviceId}-{created}.zip'
+                        downloadfilename = os.path.join(targetFolder,
+                                                        downloadfile)
+                        dl_url = deviceFile['downloadUrl']
+                        _, content = cd._http.request(dl_url)
+                        fileutils.write_file(downloadfilename,
+                                             content,
+                                             mode='wb',
+                                             continue_on_error=True)
+                        print(f'Downloaded: {downloadfilename}')
+                elif downloadfile:
+                    print('ERROR: no files to download.')
+            cpuStatusReports = _filterCreateReportTime(
+                cros.get('cpuStatusReports', []), 'reportTime', startDate,
+                endDate)
+            lenCSR = len(cpuStatusReports)
+            if lenCSR:
+                print('  cpuStatusReports')
+                num_ranges = min(lenCSR, listLimit or lenCSR)
+                for cpuStatusReport in cpuStatusReports[:num_ranges]:
+                    print(f'    reportTime: {cpuStatusReport["reportTime"]}')
+                    print('      cpuTemperatureInfo')
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        temp_label = tempInfo['label'].strip()
+                        temperature = tempInfo['temperature']
+                        print(f'        {temp_label}: {temperature}')
+                    pct_info = cpuStatusReport['cpuUtilizationPercentageInfo']
+                    util = ','.join([str(x) for x in pct_info])
+                    print(f'      cpuUtilizationPercentageInfo: {util}')
+            diskVolumeReports = cros.get('diskVolumeReports', [])
+            lenDVR = len(diskVolumeReports)
+            if lenDVR:
+                print('  diskVolumeReports')
+                print('    volumeInfo')
+                num_ranges = min(lenDVR, listLimit or lenDVR)
+                for diskVolumeReport in diskVolumeReports[:num_ranges]:
+                    volumeInfo = diskVolumeReport['volumeInfo']
+                    for volume in volumeInfo:
+                        vid = volume['volumeId']
+                        vstorage_free = volume['storageFree']
+                        vstorage_total = volume['storageTotal']
+                        print(f'      volumeId: {vid}')
+                        print(f'        storageFree: {vstorage_free}')
+                        print(f'        storageTotal: {vstorage_total}')
+            systemRamFreeReports = _filterCreateReportTime(
+                cros.get('systemRamFreeReports', []), 'reportTime', startDate,
+                endDate)
+            lenSRFR = len(systemRamFreeReports)
+            if lenSRFR:
+                print('  systemRamFreeReports')
+                num_ranges = min(lenSRFR, listLimit or lenSRFR)
+                for systemRamFreeReport in systemRamFreeReports[:num_ranges]:
+                    report_time = systemRamFreeReport['reportTime']
+                    free_info = systemRamFreeReport['systemRamFreeInfo']
+                    free_ram = ','.join(free_info)
+                    print(f'    reportTime: {report_time}')
+                    print(f'      systemRamFreeInfo: {free_ram}')
+
+
+def doPrintCrosActivity():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber',
+        'orgUnitPath'
+    ]
+    csvRows = []
+    fieldsList = [
+        'deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber',
+        'orgUnitPath'
+    ]
+    startDate = endDate = None
+    selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False
+    listLimit = 0
+    delimiter = ','
+    orgUnitPath = None
+    queries = [None]
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectActiveTimeRanges = True
+            i += 1
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectDeviceFiles = True
+            i += 1
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectRecentUsers = True
+            i += 1
+        elif myarg == 'both':
+            selectActiveTimeRanges = selectRecentUsers = True
+            i += 1
+        elif myarg == 'all':
+            selectActiveTimeRanges = selectDeviceFiles = True
+            selectRecentUsers = True
+            i += 1
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=0)
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print crosactivity')
+    if not selectActiveTimeRanges and \
+       not selectDeviceFiles and \
+       not selectRecentUsers:
+        selectActiveTimeRanges = selectRecentUsers = True
+    if selectRecentUsers:
+        fieldsList.append('recentUsers')
+        display.add_titles_to_csv_file([
+            'recentUsers.email',
+        ], titles)
+    if selectActiveTimeRanges:
+        fieldsList.append('activeTimeRanges')
+        titles_to_add = [
+            'activeTimeRanges.date', 'activeTimeRanges.duration',
+            'activeTimeRanges.minutes'
+        ]
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    if selectDeviceFiles:
+        fieldsList.append('deviceFiles')
+        titles_to_add = ['deviceFiles.type', 'deviceFiles.createTime']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    fields = f'nextPageToken,chromeosdevices({",".join(fieldsList)})'
+    for query in queries:
+        gam.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(),
+                                      'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection='FULL',
+                                      fields=fields,
+                                      orgUnitPath=orgUnitPath)
+        for cros in all_cros:
+            row = {}
+            skip_attribs = ['recentUsers', 'activeTimeRanges', 'deviceFiles']
+            for attrib in cros:
+                if attrib not in skip_attribs:
+                    row[attrib] = cros[attrib]
+            if selectActiveTimeRanges:
+                activeTimeRanges = _filterTimeRanges(
+                    cros.get('activeTimeRanges', []), startDate, endDate)
+                lenATR = len(activeTimeRanges)
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['activeTimeRanges.date'] = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    newrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    newrow['activeTimeRanges.minutes'] = \
+                        activeTimeRange['activeTime']//60000
+                    csvRows.append(newrow)
+            if selectRecentUsers:
+                recentUsers = cros.get('recentUsers', [])
+                lenRU = len(recentUsers)
+                num_ranges = min(lenRU, listLimit or lenRU)
+                recent_users = []
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get('email')
+                    if not useremail:
+                        if recentUser['type'] == 'USER_TYPE_UNMANAGED':
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    recent_users.append(useremail)
+                row['recentUsers.email'] = delimiter.join(recent_users)
+                csvRows.append(row)
+            if selectDeviceFiles:
+                deviceFiles = _filterCreateReportTime(
+                    cros.get('deviceFiles', []), 'createTime', startDate,
+                    endDate)
+                lenDF = len(deviceFiles)
+                num_ranges = min(lenDF, listLimit or lenDF)
+                for deviceFile in deviceFiles[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['deviceFiles.type'] = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    newrow['deviceFiles.createTime'] = create_time
+                    csvRows.append(newrow)
+    display.write_csv_file(csvRows, titles, 'CrOS Activity', todrive)
+
+
+def _checkTPMVulnerability(cros):
+    if 'tpmVersionInfo' in cros and \
+       'firmwareVersion' in cros['tpmVersionInfo']:
+        firmware_version = cros['tpmVersionInfo']['firmwareVersion']
+        if firmware_version in CROS_TPM_VULN_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'VULNERABLE'
+        elif firmware_version in CROS_TPM_FIXED_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'UPDATED'
+        else:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'NOT IMPACTED'
+
+
+def doPrintCrosDevices():
+
+    def _getSelectedLists(myarg):
+        if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectedLists['activeTimeRanges'] = True
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectedLists['recentUsers'] = True
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectedLists['deviceFiles'] = True
+        elif myarg in CROS_CPU_STATUS_REPORTS_ARGUMENTS:
+            selectedLists['cpuStatusReports'] = True
+        elif myarg in CROS_DISK_VOLUME_REPORTS_ARGUMENTS:
+            selectedLists['diskVolumeReports'] = True
+        elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS:
+            selectedLists['systemRamFreeReports'] = True
+
+    cd = gapi_directory.build()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    display.add_field_to_csv_file('deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                  fieldsList, fieldsTitles, titles)
+    projection = orderBy = sortOrder = orgUnitPath = None
+    queries = [None]
+    noLists = sortHeaders = False
+    selectedLists = {}
+    startDate = endDate = None
+    listLimit = 0
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'nolists':
+            noLists = True
+            selectedLists = {}
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=0)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i + 1].lower().replace('_', '')
+            validOrderBy = [
+                'location', 'user', 'lastsync', 'notes', 'serialnumber',
+                'status', 'supportenddate'
+            ]
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(validOrderBy),
+                                                   orderBy)
+            if orderBy == 'location':
+                orderBy = 'annotatedLocation'
+            elif orderBy == 'user':
+                orderBy = 'annotatedUser'
+            elif orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'serialnumber':
+                orderBy = 'serialNumber'
+            elif orderBy == 'supportenddate':
+                orderBy = 'supportEndDate'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            sortHeaders = True
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            sortHeaders = True
+            fieldsList = []
+            i += 1
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in CROS_LISTS_ARGUMENTS:
+            _getSelectedLists(myarg)
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_fields_list(myarg,
+                                             CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                             fieldsList)
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i + 1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_LISTS_ARGUMENTS:
+                    _getSelectedLists(field)
+                elif field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    display.add_field_to_fields_list(
+                        field, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+                else:
+                    controlflow.invalid_argument_exit(field,
+                                                      'gam print cros fields')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print cros')
+    if selectedLists:
+        noLists = False
+        projection = 'FULL'
+        for selectList in selectedLists:
+            display.add_field_to_fields_list(selectList,
+                                             CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                             fieldsList)
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = f'nextPageToken,chromeosdevices({",".join(set(fieldsList))})'.replace(
+            '.', '/')
+    else:
+        fields = None
+    for query in queries:
+        gam.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(),
+                                      'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection=projection,
+                                      orgUnitPath=orgUnitPath,
+                                      orderBy=orderBy,
+                                      sortOrder=sortOrder,
+                                      fields=fields)
+        for cros in all_cros:
+            _checkTPMVulnerability(cros)
+        if not noLists and not selectedLists:
+            for cros in all_cros:
+                if 'notes' in cros:
+                    cros['notes'] = cros['notes'].replace('\n', '\\n')
+                if 'autoUpdateExpiration' in cros:
+                    cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                        cros['autoUpdateExpiration'])
+                for cpuStatusReport in cros.get('cpuStatusReports', []):
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        tempInfo['label'] = tempInfo['label'].strip()
+                display.add_row_titles_to_csv_file(
+                    utils.flatten_json(cros, listLimit=listLimit), csvRows,
+                    titles)
+            continue
+        for cros in all_cros:
+            if 'notes' in cros:
+                cros['notes'] = cros['notes'].replace('\n', '\\n')
+            if 'autoUpdateExpiration' in cros:
+                cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                    cros['autoUpdateExpiration'])
+            row = {}
+            for attrib in cros:
+                if attrib not in set([
+                        'kind', 'etag', 'tpmVersionInfo', 'recentUsers',
+                        'activeTimeRanges', 'deviceFiles', 'cpuStatusReports',
+                        'diskVolumeReports', 'systemRamFreeReports'
+                ]):
+                    row[attrib] = cros[attrib]
+            if selectedLists.get('activeTimeRanges'):
+                timergs = cros.get('activeTimeRanges', [])
+            else:
+                timergs = []
+            activeTimeRanges = _filterTimeRanges(timergs, startDate, endDate)
+            if selectedLists.get('recentUsers'):
+                recentUsers = cros.get('recentUsers', [])
+            else:
+                recentUsers = []
+            if selectedLists.get('deviceFiles'):
+                device_files = cros.get('deviceFiles', [])
+            else:
+                device_files = []
+            deviceFiles = _filterCreateReportTime(device_files, 'createTime',
+                                                  startDate, endDate)
+            if selectedLists.get('cpuStatusReports'):
+                cpu_reports = cros.get('cpuStatusReports', [])
+            else:
+                cpu_reports = []
+            cpuStatusReports = _filterCreateReportTime(cpu_reports,
+                                                       'reportTime', startDate,
+                                                       endDate)
+            if selectedLists.get('diskVolumeReports'):
+                diskVolumeReports = cros.get('diskVolumeReports', [])
+            else:
+                diskVolumeReports = []
+            if selectedLists.get('systemRamFreeReports'):
+                ram_reports = cros.get('systemRamFreeReports', [])
+            else:
+                ram_reports = []
+            systemRamFreeReports = _filterCreateReportTime(
+                ram_reports, 'reportTime', startDate, endDate)
+            if noLists or (not activeTimeRanges and \
+                           not recentUsers and \
+                           not deviceFiles and \
+                           not cpuStatusReports and \
+                           not diskVolumeReports and \
+                           not systemRamFreeReports):
+                display.add_row_titles_to_csv_file(row, csvRows, titles)
+                continue
+            lenATR = len(activeTimeRanges)
+            lenRU = len(recentUsers)
+            lenDF = len(deviceFiles)
+            lenCSR = len(cpuStatusReports)
+            lenDVR = len(diskVolumeReports)
+            lenSRFR = len(systemRamFreeReports)
+            max_len = max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR)
+            for i in range(min(max_len, listLimit or max_len)):
+                nrow = row.copy()
+                if i < lenATR:
+                    nrow['activeTimeRanges.date'] = \
+                        activeTimeRanges[i]['date']
+                    nrow['activeTimeRanges.activeTime'] = \
+                        str(activeTimeRanges[i]['activeTime'])
+                    active_time = activeTimeRanges[i]['activeTime']
+                    nrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    nrow['activeTimeRanges.minutes'] = active_time // 60000
+                if i < lenRU:
+                    nrow['recentUsers.type'] = recentUsers[i]['type']
+                    nrow['recentUsers.email'] = recentUsers[i].get('email')
+                    if not nrow['recentUsers.email']:
+                        if nrow['recentUsers.type'] == 'USER_TYPE_UNMANAGED':
+                            nrow['recentUsers.email'] = 'UnmanagedUser'
+                        else:
+                            nrow['recentUsers.email'] = 'Unknown'
+                if i < lenDF:
+                    nrow['deviceFiles.type'] = deviceFiles[i]['type']
+                    nrow['deviceFiles.createTime'] = \
+                        deviceFiles[i]['createTime']
+                if i < lenCSR:
+                    nrow['cpuStatusReports.reportTime'] = \
+                        cpuStatusReports[i]['reportTime']
+                    tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo',
+                                                        [])
+                    for tempInfo in tempInfos:
+                        label = tempInfo['label'].strip()
+                        base = 'cpuStatusReports.cpuTemperatureInfo.'
+                        nrow[f'{base}{label}'] = tempInfo['temperature']
+                    cpu_field = 'cpuUtilizationPercentageInfo'
+                    cpu_reports = cpuStatusReports[i][cpu_field]
+                    cpu_pcts = [str(x) for x in cpu_reports]
+                    nrow[f'cpuStatusReports.{cpu_field}'] = ','.join(cpu_pcts)
+                if i < lenDVR:
+                    volumeInfo = diskVolumeReports[i]['volumeInfo']
+                    j = 0
+                    vfield = 'diskVolumeReports.volumeInfo.'
+                    for volume in volumeInfo:
+                        nrow[f'{vfield}{j}.volumeId'] = \
+                            volume['volumeId']
+                        nrow[f'{vfield}{j}.storageFree'] = \
+                            volume['storageFree']
+                        nrow[f'{vfield}{j}.storageTotal'] = \
+                            volume['storageTotal']
+                        j += 1
+                if i < lenSRFR:
+                    nrow['systemRamFreeReports.reportTime'] = \
+                        systemRamFreeReports[i]['reportTime']
+                    ram_reports = systemRamFreeReports[i]['systemRamFreeInfo']
+                    ram_info = [str(x) for x in ram_reports]
+                    nrow['systenRamFreeReports.systemRamFreeInfo'] = \
+                        ','.join(ram_info)
+                display.add_row_titles_to_csv_file(nrow, csvRows, titles)
+    if sortHeaders:
+        display.sort_csv_titles([
+            'deviceId',
+        ], titles)
+    display.write_csv_file(csvRows, titles, 'CrOS', todrive)
+
+
+def getCrOSDeviceEntity(i, cd):
+    myarg = sys.argv[i].lower()
+    if myarg == 'cros_sn':
+        return i + 2, gam.getUsersToModify('cros_sn', sys.argv[i + 1])
+    if myarg == 'query':
+        return i + 2, gam.getUsersToModify('crosquery', sys.argv[i + 1])
+    if myarg[:6] == 'query:':
+        query = sys.argv[i][6:]
+        if query[:12].lower() == 'orgunitpath:':
+            kwargs = {'orgUnitPath': query[12:]}
+        else:
+            kwargs = {'query': query}
+        fields = 'nextPageToken,chromeosdevices(deviceId)'
+        devices = gapi.get_all_pages(cd.chromeosdevices(),
+                                     'list',
+                                     'chromeosdevices',
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     fields=fields,
+                                     **kwargs)
+        return i + 1, [device['deviceId'] for device in devices]
+    return i + 1, sys.argv[i].replace(',', ' ').split()
+
+
+def _getFilterDate(dateStr):
+    return datetime.datetime.strptime(dateStr, YYYYMMDD_FORMAT)
+
+
+def _filterTimeRanges(activeTimeRanges, startDate, endDate):
+    if startDate is None and endDate is None:
+        return activeTimeRanges
+    filteredTimeRanges = []
+    for timeRange in activeTimeRanges:
+        activityDate = datetime.datetime.strptime(timeRange['date'],
+                                                  YYYYMMDD_FORMAT)
+        if ((startDate is None) or \
+            (activityDate >= startDate)) and \
+           ((endDate is None) or \
+            (activityDate <= endDate)):
+            filteredTimeRanges.append(timeRange)
+    return filteredTimeRanges
+
+
+def _filterCreateReportTime(items, timeField, startTime, endTime):
+    if startTime is None and endTime is None:
+        return items
+    filteredItems = []
+    time_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    for item in items:
+        timeValue = datetime.datetime.strptime(item[timeField], time_format)
+        if ((startTime is None) or \
+            (timeValue >= startTime)) and \
+           ((endTime is None) or \
+            (timeValue <= endTime)):
+            filteredItems.append(item)
+    return filteredItems

src/gam/gapi/directory/customer.py

@@ -0,0 +1,149 @@
+import datetime
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import reports as gapi_reports
+
+
+def doGetCustomerInfo():
+    cd = gapi_directory.build()
+    customer_info = gapi.call(cd.customers(),
+                              'get',
+                              customerKey=GC_Values[GC_CUSTOMER_ID])
+    print(f'Customer ID: {customer_info["id"]}')
+    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    try:
+        result = gapi.call(
+            cd.domains(),
+            'get',
+            customer=customer_info['id'],
+            domainName=customer_info['customerDomain'],
+            fields='verified',
+            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND])
+    except gapi.errors.GapiDomainNotFoundError:
+        result = {'verified': False}
+    print(f'Primary Domain Verified: {result["verified"]}')
+    # If customer has changed primary domain customerCreationTime is date
+    # of current primary being added, not customer create date.
+    # We should also get all domains and use oldest date
+    customer_creation = customer_info['customerCreationTime']
+    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    oldest = datetime.datetime.strptime(customer_creation, date_format)
+    domains = gapi.get_items(cd.domains(),
+                             'list',
+                             'domains',
+                             customer=GC_Values[GC_CUSTOMER_ID],
+                             fields='domains(creationTime)')
+    for domain in domains:
+        creation_timestamp = int(domain['creationTime']) / 1000
+        domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
+        if domain_creation < oldest:
+            oldest = domain_creation
+    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
+    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    print(f'Default Language: {customer_language}')
+    if 'postalAddress' in customer_info:
+        print('Address:')
+        for field in ADDRESS_FIELDS_PRINT_ORDER:
+            if field in customer_info['postalAddress']:
+                print(f' {field}: {customer_info["postalAddress"][field]}')
+    if 'phoneNumber' in customer_info:
+        print(f'Phone: {customer_info["phoneNumber"]}')
+    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    user_counts_map = {
+        'accounts:num_users': 'Total Users',
+        'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',
+        'accounts:gsuite_basic_used_licenses': 'G Suite Basic Users',
+        'accounts:gsuite_enterprise_total_licenses': 'G Suite Enterprise ' \
+        'Licenses',
+        'accounts:gsuite_enterprise_used_licenses': 'G Suite Enterprise ' \
+        'Users',
+        'accounts:gsuite_unlimited_total_licenses': 'G Suite Business ' \
+        'Licenses',
+        'accounts:gsuite_unlimited_used_licenses': 'G Suite Business Users'
+    }
+    parameters = ','.join(list(user_counts_map))
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    rep = gapi_reports.build()
+    usage = None
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.FORBIDDEN
+    ]
+    while True:
+        try:
+            result = gapi.call(rep.customerUsageReports(),
+                               'get',
+                               throw_reasons=throw_reasons,
+                               customerId=customerId,
+                               date=tryDate,
+                               parameters=parameters)
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = gapi_reports._adjust_date(str(e))
+            continue
+        except gapi.errors.GapiForbiddenError:
+            return
+        warnings = result.get('warnings', [])
+        fullDataRequired = ['accounts']
+        usage = result.get('usageReports')
+        has_reports = bool(usage)
+        fullData, tryDate = gapi_reports._check_full_data_available(
+            warnings, tryDate, fullDataRequired, has_reports)
+        if fullData < 0:
+            print('No user report available.')
+            sys.exit(1)
+        if fullData == 0:
+            continue
+        break
+    print(f'User counts as of {tryDate}:')
+    for item in usage[0]['parameters']:
+        api_name = user_counts_map.get(item['name'])
+        api_value = int(item.get('intValue', 0))
+        if api_name and api_value:
+            print(f'  {api_name}: {api_value:,}')
+
+
+def doUpdateCustomer():
+    cd = gapi_directory.build()
+    body = {}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ADDRESS_FIELDS_ARGUMENT_MAP:
+            body.setdefault('postalAddress', {})
+            arg = ADDRESS_FIELDS_ARGUMENT_MAP[myarg]
+            body['postalAddress'][arg] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['adminsecondaryemail', 'alternateemail']:
+            body['alternateEmail'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['phone', 'phonenumber']:
+            body['phoneNumber'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'language':
+            body['language'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam update customer')
+    if not body:
+        controlflow.system_error_exit(
+            2, 'no arguments specified for "gam '
+            'update customer"')
+    gapi.call(cd.customers(),
+              'patch',
+              customerKey=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print('Updated customer')
+
+
+def setTrueCustomerId():
+    if GC_Values[GC_CUSTOMER_ID] == MY_CUSTOMER:
+        cd = gapi_directory.build()
+        GC_Values[GC_CUSTOMER_ID] = gapi.call(cd.customers(), 'get',
+            customerKey=GC_Values[GC_CUSTOMER_ID],
+            fields='id').get('id', GC_Values[GC_CUSTOMER_ID])

src/gam/gapi/directory/domainaliases.py

@@ -0,0 +1,76 @@
+import sys
+
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def create():
+    cd = gapi_directory.build()
+    body = {'domainAliasName': sys.argv[3], 'parentDomainName': sys.argv[4]}
+    print(f'Adding {body["domainAliasName"]} alias for ' \
+       f'{body["parentDomainName"]}')
+    gapi.call(cd.domainAliases(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def delete():
+    cd = gapi_directory.build()
+    domainAliasName = sys.argv[3]
+    print(f'Deleting domain alias {domainAliasName}')
+    gapi.call(cd.domainAliases(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              domainAliasName=domainAliasName)
+
+
+def info():
+    cd = gapi_directory.build()
+    alias = sys.argv[3]
+    result = gapi.call(cd.domainAliases(),
+                       'get',
+                       customer=GC_Values[GC_CUSTOMER_ID],
+                       domainAliasName=alias)
+    if 'creationTime' in result:
+        result['creationTime'] = utils.formatTimestampYMDHMSF(
+            result['creationTime'])
+    display.print_json(result)
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'domainAliasName',
+    ]
+    csvRows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print domainaliases')
+    results = gapi.call(cd.domainAliases(),
+                        'list',
+                        customer=GC_Values[GC_CUSTOMER_ID])
+    for domainAlias in results['domainAliases']:
+        domainAlias_attributes = {}
+        for attr in domainAlias:
+            if attr in ['kind', 'etag']:
+                continue
+            if attr == 'creationTime':
+                domainAlias[attr] = utils.formatTimestampYMDHMSF(
+                    domainAlias[attr])
+            if attr not in titles:
+                titles.append(attr)
+            domainAlias_attributes[attr] = domainAlias[attr]
+        csvRows.append(domainAlias_attributes)
+    display.write_csv_file(csvRows, titles, 'Domains', todrive)

src/gam/gapi/directory/domains.py

@@ -0,0 +1,124 @@
+import sys
+
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi.directory import customer as gapi_directory_customer
+from gam import utils
+
+
+def create():
+    cd = gapi_directory.build()
+    domain_name = sys.argv[3]
+    body = {'domainName': domain_name}
+    gapi.call(cd.domains(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print(f'Added domain {domain_name}')
+
+
+def info():
+    if (len(sys.argv) < 4) or (sys.argv[3] == 'logo'):
+        gapi_directory_customer.doGetCustomerInfo()
+        return
+    cd = gapi_directory.build()
+    domainName = sys.argv[3]
+    result = gapi.call(cd.domains(),
+                       'get',
+                       customer=GC_Values[GC_CUSTOMER_ID],
+                       domainName=domainName)
+    if 'creationTime' in result:
+        result['creationTime'] = utils.formatTimestampYMDHMSF(
+            result['creationTime'])
+    if 'domainAliases' in result:
+        for i in range(0, len(result['domainAliases'])):
+            if 'creationTime' in result['domainAliases'][i]:
+                result['domainAliases'][i][
+                    'creationTime'] = utils.formatTimestampYMDHMSF(
+                        result['domainAliases'][i]['creationTime'])
+    display.print_json(result)
+
+
+def update():
+    cd = gapi_directory.build()
+    domain_name = sys.argv[3]
+    i = 4
+    body = {}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'primary':
+            body['customerDomain'] = domain_name
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update domain')
+    gapi.call(cd.customers(),
+              'update',
+              customerKey=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print(f'{domain_name} is now the primary domain.')
+
+
+def delete():
+    cd = gapi_directory.build()
+    domainName = sys.argv[3]
+    print(f'Deleting domain {domainName}')
+    gapi.call(cd.domains(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              domainName=domainName)
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'domainName',
+    ]
+    csvRows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print domains')
+    results = gapi.call(cd.domains(),
+                        'list',
+                        customer=GC_Values[GC_CUSTOMER_ID])
+    for domain in results.get('domains', []):
+        domain_attributes = {}
+        domain['type'] = ['secondary', 'primary'][domain['isPrimary']]
+        for attr in domain:
+            if attr in ['kind', 'etag', 'domainAliases', 'isPrimary']:
+                continue
+            if attr in [
+                    'creationTime',
+            ]:
+                domain[attr] = utils.formatTimestampYMDHMSF(domain[attr])
+            if attr not in titles:
+                titles.append(attr)
+            domain_attributes[attr] = domain[attr]
+        csvRows.append(domain_attributes)
+        if 'domainAliases' in domain:
+            for aliasdomain in domain['domainAliases']:
+                aliasdomain['domainName'] = aliasdomain['domainAliasName']
+                del aliasdomain['domainAliasName']
+                aliasdomain['type'] = 'alias'
+                aliasdomain_attributes = {}
+                for attr in aliasdomain:
+                    if attr in ['kind', 'etag']:
+                        continue
+                    if attr in [
+                            'creationTime',
+                    ]:
+                        aliasdomain[attr] = utils.formatTimestampYMDHMSF(
+                            aliasdomain[attr])
+                    if attr not in titles:
+                        titles.append(attr)
+                    aliasdomain_attributes[attr] = aliasdomain[attr]
+                csvRows.append(aliasdomain_attributes)
+    display.write_csv_file(csvRows, titles, 'Domains', todrive)

src/gam/gapi/directory/mobiledevices.py

@@ -0,0 +1,239 @@
+import sys
+import uuid
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def delete():
+    cd = gapi_directory.build()
+    resourceId = sys.argv[3]
+    gapi.call(cd.mobiledevices(),
+              'delete',
+              resourceId=resourceId,
+              customerId=GC_Values[GC_CUSTOMER_ID])
+
+
+
+def info():
+    cd = gapi_directory.build()
+    resourceId = sys.argv[3]
+    info = gapi.call(cd.mobiledevices(),
+                     'get',
+                     customerId=GC_Values[GC_CUSTOMER_ID],
+                     resourceId=resourceId)
+    if 'deviceId' in info:
+        info['deviceId'] = info['deviceId'].encode('unicode-escape').decode(
+            UTF8)
+    attrib = 'securityPatchLevel'
+    if attrib in info and int(info[attrib]):
+        info[attrib] = utils.formatTimestampYMDHMS(info[attrib])
+    display.print_json(info)
+
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = []
+    csvRows = []
+    fields = None
+    projection = orderBy = sortOrder = None
+    queries = [None]
+    delimiter = ' '
+    listLimit = 1
+    appsLimit = -1
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['query', 'queries']:
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
+            i += 2
+        elif myarg == 'appslimit':
+            appsLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
+            i += 2
+        elif myarg == 'fields':
+            fields = f'nextPageToken,mobiledevices({sys.argv[i+1]})'
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i + 1].lower()
+            validOrderBy = [
+                'deviceid', 'email', 'lastsync', 'model', 'name', 'os',
+                'status', 'type'
+            ]
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(validOrderBy),
+                                                   orderBy)
+            if orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'deviceid':
+                orderBy = 'deviceId'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print mobile')
+    for query in queries:
+        gam.printGettingAllItems('Mobile Devices', query)
+        page_message = gapi.got_total_items_msg('Mobile Devices', '...\n')
+        all_mobile = gapi.get_all_pages(cd.mobiledevices(),
+                                        'list',
+                                        'mobiledevices',
+                                        page_message=page_message,
+                                        customerId=GC_Values[GC_CUSTOMER_ID],
+                                        query=query,
+                                        projection=projection,
+                                        fields=fields,
+                                        orderBy=orderBy,
+                                        sortOrder=sortOrder)
+        for mobile in all_mobile:
+            row = {}
+            for attrib in mobile:
+                if attrib in ['kind', 'etag']:
+                    continue
+                if attrib in ['name', 'email', 'otherAccountsInfo']:
+                    if attrib not in titles:
+                        titles.append(attrib)
+                    if listLimit > 0:
+                        row[attrib] = delimiter.join(
+                            mobile[attrib][0:listLimit])
+                    elif listLimit == 0:
+                        row[attrib] = delimiter.join(mobile[attrib])
+                elif attrib == 'applications':
+                    if appsLimit >= 0:
+                        if attrib not in titles:
+                            titles.append(attrib)
+                        applications = []
+                        j = 0
+                        for app in mobile[attrib]:
+                            j += 1
+                            if appsLimit and (j > appsLimit):
+                                break
+                            appDetails = []
+                            for field in [
+                                    'displayName', 'packageName', 'versionName'
+                            ]:
+                                appDetails.append(app.get(field, '<None>'))
+                            appDetails.append(
+                                str(app.get('versionCode', '<None>')))
+                            permissions = app.get('permission', [])
+                            if permissions:
+                                appDetails.append('/'.join(permissions))
+                            else:
+                                appDetails.append('<None>')
+                            applications.append('-'.join(appDetails))
+                        row[attrib] = delimiter.join(applications)
+                else:
+                    if attrib not in titles:
+                        titles.append(attrib)
+                    if attrib == 'deviceId':
+                        row[attrib] = mobile[attrib].encode(
+                            'unicode-escape').decode(UTF8)
+                    elif attrib == 'securityPatchLevel' and int(mobile[attrib]):
+                        row[attrib] = utils.formatTimestampYMDHMS(
+                            mobile[attrib])
+                    else:
+                        row[attrib] = mobile[attrib]
+            csvRows.append(row)
+    display.sort_csv_titles(
+        ['resourceId', 'deviceId', 'serialNumber', 'name', 'email', 'status'],
+        titles)
+    display.write_csv_file(csvRows, titles, 'Mobile', todrive)
+
+
+def update():
+    cd = gapi_directory.build
+    resourceIds = sys.argv[3]
+    match_users = None
+    doit = False
+    if resourceIds[:6] == 'query:':
+        query = resourceIds[6:]
+        fields = 'nextPageToken,mobiledevices(resourceId,email)'
+        page_message = gapi.got_total_items_msg('Mobile Devices', '...\n')
+        devices = gapi.get_all_pages(cd.mobiledevices(),
+                                     'list',
+                                     page_message=page_message,
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     items='mobiledevices',
+                                     query=query,
+                                     fields=fields)
+    else:
+        devices = [{'resourceId': resourceIds, 'email': ['not set']}]
+        doit = True
+    i = 4
+    body = {}
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            body['action'] = sys.argv[i + 1].lower()
+            validActions = [
+                'wipe', 'wipeaccount', 'accountwipe', 'wipe_account',
+                'account_wipe', 'approve', 'block',
+                'cancel_remote_wipe_then_activate',
+                'cancel_remote_wipe_then_block'
+            ]
+            if body['action'] not in validActions:
+                controlflow.expected_argument_exit('action',
+                                                   ', '.join(validActions),
+                                                   body['action'])
+            if body['action'] == 'wipe':
+                body['action'] = 'admin_remote_wipe'
+            elif body['action'].replace('_',
+                                        '') in ['accountwipe', 'wipeaccount']:
+                body['action'] = 'admin_account_wipe'
+            i += 2
+        elif myarg in ['ifusers', 'matchusers']:
+            match_users = gam.getUsersToModify(entity_type=sys.argv[i + 1].lower(),
+                                           entity=sys.argv[i + 2])
+            i += 3
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update mobile')
+    if body:
+        if doit:
+            print(f'Updating {len(devices)} devices')
+            describe_as = 'Performing'
+        else:
+            print(
+                f'Showing {len(devices)} changes that would be made, not actually making changes because doit argument not specified'
+            )
+            describe_as = 'Would perform'
+        for device in devices:
+            device_user = device.get('email', [''])[0]
+            if match_users and device_user not in match_users:
+                print(
+                    f'Skipping device for user {device_user} that did not match match_users argument'
+                )
+            else:
+                print(
+                    f'{describe_as} {body["action"]} on user {device_user} device {device["resourceId"]}'
+                )
+                if doit:
+                    gapi.call(cd.mobiledevices(),
+                              'action',
+                              resourceId=device['resourceId'],
+                              body=body,
+                              customerId=GC_Values[GC_CUSTOMER_ID])
+
+

src/gam/gapi/directory/orgunits.py

@@ -0,0 +1,422 @@
+import sys
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import errors as gapi_errors
+from gam import utils
+
+
+def create():
+    cd = gapi_directory.build()
+    name = getOrgUnitItem(sys.argv[3], pathOnly=True, absolutePath=False)
+    parent = ''
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'description':
+            body['description'] = sys.argv[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'parent':
+            parent = getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'noinherit':
+            body['blockInheritance'] = True
+            i += 1
+        elif myarg == 'inherit':
+            body['blockInheritance'] = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create org')
+    if parent.startswith('id:'):
+        parent = gapi.call(cd.orgunits(),
+                           'get',
+                           customerId=GC_Values[GC_CUSTOMER_ID],
+                           orgUnitPath=parent,
+                           fields='orgUnitPath')['orgUnitPath']
+    if parent == '/':
+        orgUnitPath = parent + name
+    else:
+        orgUnitPath = parent + '/' + name
+    if orgUnitPath.count('/') > 1:
+        body['parentOrgUnitPath'], body['name'] = orgUnitPath.rsplit('/', 1)
+    else:
+        body['parentOrgUnitPath'] = '/'
+        body['name'] = orgUnitPath[1:]
+    parent = body['parentOrgUnitPath']
+    gapi.call(cd.orgunits(),
+              'insert',
+              customerId=GC_Values[GC_CUSTOMER_ID],
+              body=body,
+              retry_reasons=[gapi_errors.ErrorReason.DAILY_LIMIT_EXCEEDED])
+    print(f'Created OrgUnit {body["name"]}')
+
+
+def delete():
+    cd = gapi_directory.build()
+    name = getOrgUnitItem(sys.argv[3])
+    print(f'Deleting organization {name}')
+    gapi.call(cd.orgunits(),
+              'delete',
+              customerId=GC_Values[GC_CUSTOMER_ID],
+              orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(name)))
+
+
+def info(name=None, return_attrib=None):
+    cd = gapi_directory.build()
+    checkSuspended = None
+    if not name:
+        name = getOrgUnitItem(sys.argv[3])
+        get_users = True
+        show_children = False
+        i = 4
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower()
+            if myarg == 'nousers':
+                get_users = False
+                i += 1
+            elif myarg in ['children', 'child']:
+                show_children = True
+                i += 1
+            elif myarg in ['suspended', 'notsuspended']:
+                checkSuspended = myarg == 'suspended'
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i], 'gam info org')
+    if name == '/':
+        orgs = gapi.call(cd.orgunits(),
+                         'list',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         type='children',
+                         fields='organizationUnits/parentOrgUnitId')
+        if 'organizationUnits' in orgs and orgs['organizationUnits']:
+            name = orgs['organizationUnits'][0]['parentOrgUnitId']
+        else:
+            topLevelOrgId = getTopLevelOrgId(cd, '/')
+            if topLevelOrgId:
+                name = topLevelOrgId
+    else:
+        name = makeOrgUnitPathRelative(name)
+    result = gapi.call(cd.orgunits(),
+                       'get',
+                       customerId=GC_Values[GC_CUSTOMER_ID],
+                       orgUnitPath=encodeOrgUnitPath(name))
+    if return_attrib:
+        return result[return_attrib]
+    display.print_json(result)
+    if get_users:
+        name = result['orgUnitPath']
+        page_message = gapi.got_total_items_first_last_msg('Users')
+        users = gapi.get_all_pages(
+            cd.users(),
+            'list',
+            'users',
+            page_message=page_message,
+            message_attribute='primaryEmail',
+            customer=GC_Values[GC_CUSTOMER_ID],
+            query=orgUnitPathQuery(name, checkSuspended),
+            fields='users(primaryEmail,orgUnitPath),nextPageToken')
+        if checkSuspended is None:
+            print('Users:')
+        elif not checkSuspended:
+            print('Users (Not suspended):')
+        else:
+            print('Users (Suspended):')
+        for user in users:
+            if show_children or (name.lower() == user['orgUnitPath'].lower()):
+                sys.stdout.write(f' {user["primaryEmail"]}')
+                if name.lower() != user['orgUnitPath'].lower():
+                    print(' (child)')
+                else:
+                    print('')
+
+
+def print_():
+    print_order = [
+        'orgUnitPath', 'orgUnitId', 'name', 'description', 'parentOrgUnitPath',
+        'parentOrgUnitId', 'blockInheritance'
+    ]
+    cd = gapi_directory.build()
+    listType = 'all'
+    orgUnitPath = '/'
+    todrive = False
+    fields = ['orgUnitPath', 'name', 'orgUnitId', 'parentOrgUnitId']
+    titles = []
+    csvRows = []
+    parentOrgIds = []
+    retrievedOrgIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'toplevelonly':
+            listType = 'children'
+            i += 1
+        elif myarg == 'fromparent':
+            orgUnitPath = getOrgUnitItem(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg == 'fields':
+            fields += sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print orgs')
+    gam.printGettingAllItems('Organizational Units', None)
+    if fields:
+        get_fields = ','.join(fields)
+        list_fields = f'organizationUnits({get_fields})'
+    else:
+        list_fields = None
+        get_fields = None
+    orgs = gapi.call(cd.orgunits(),
+                     'list',
+                     customerId=GC_Values[GC_CUSTOMER_ID],
+                     type=listType,
+                     orgUnitPath=orgUnitPath,
+                     fields=list_fields)
+    if not 'organizationUnits' in orgs:
+        topLevelOrgId = getTopLevelOrgId(cd, orgUnitPath)
+        if topLevelOrgId:
+            parentOrgIds.append(topLevelOrgId)
+        orgunits = []
+    else:
+        orgunits = orgs['organizationUnits']
+    for row in orgunits:
+        retrievedOrgIds.append(row['orgUnitId'])
+        if row['parentOrgUnitId'] not in parentOrgIds:
+            parentOrgIds.append(row['parentOrgUnitId'])
+    missing_parents = set(parentOrgIds) - set(retrievedOrgIds)
+    for missing_parent in missing_parents:
+        try:
+            result = gapi.call(cd.orgunits(),
+                               'get',
+                               throw_reasons=['required'],
+                               customerId=GC_Values[GC_CUSTOMER_ID],
+                               orgUnitPath=missing_parent,
+                               fields=get_fields)
+            orgunits.append(result)
+        except:
+            pass
+    for row in orgunits:
+        orgEntity = {}
+        for key, value in list(row.items()):
+            if key in ['kind', 'etag', 'etags']:
+                continue
+            if key not in titles:
+                titles.append(key)
+            orgEntity[key] = value
+        csvRows.append(orgEntity)
+    for title in titles:
+        if title not in print_order:
+            print_order.append(title)
+    titles = sorted(titles, key=print_order.index)
+    # sort results similar to how they list in admin console
+    csvRows.sort(key=lambda x: x['orgUnitPath'].lower(), reverse=False)
+    display.write_csv_file(csvRows, titles, 'Orgs', todrive)
+
+
+def update():
+    cd = gapi_directory.build()
+    orgUnitPath = getOrgUnitItem(sys.argv[3])
+    if sys.argv[4].lower() in ['move', 'add']:
+        entity_type = sys.argv[5].lower()
+        if entity_type in usergroup_types:
+            users = getUsersToModify(entity_type=entity_type,
+                                     entity=sys.argv[6])
+        else:
+            entity_type = 'users'
+            users = getUsersToModify(entity_type=entity_type,
+                                     entity=sys.argv[5])
+        if (entity_type.startswith('cros')) or (
+            (entity_type == 'all') and (sys.argv[6].lower() == 'cros')):
+            for l in range(0, len(users), 50):
+                move_body = {'deviceIds': users[l:l + 50]}
+                print(
+                    f' moving {len(move_body["deviceIds"])} devices to {orgUnitPath}'
+                )
+                gapi.call(cd.chromeosdevices(),
+                          'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath,
+                          body=move_body)
+        else:
+            i = 0
+            count = len(users)
+            for user in users:
+                i += 1
+                sys.stderr.write(
+                    f' moving {user} to {orgUnitPath}{currentCountNL(i, count)}'
+                )
+                try:
+                    gapi.call(cd.users(),
+                              'update',
+                              throw_reasons=[
+                                  gapi_errors.ErrorReason.CONDITION_NOT_MET
+                              ],
+                              userKey=user,
+                              body={'orgUnitPath': orgUnitPath})
+                except gapi_errors.GapiConditionNotMetError:
+                    pass
+    else:
+        body = {}
+        i = 4
+        while i < len(sys.argv):
+            myarg = sys.argv[i].lower()
+            if myarg == 'name':
+                body['name'] = sys.argv[i + 1]
+                i += 2
+            elif myarg == 'description':
+                body['description'] = sys.argv[i + 1].replace('\\n', '\n')
+                i += 2
+            elif myarg == 'parent':
+                parent = getOrgUnitItem(sys.argv[i + 1])
+                if parent.startswith('id:'):
+                    body['parentOrgUnitId'] = parent
+                else:
+                    body['parentOrgUnitPath'] = parent
+                i += 2
+            elif myarg == 'noinherit':
+                body['blockInheritance'] = True
+                i += 1
+            elif myarg == 'inherit':
+                body['blockInheritance'] = False
+                i += 1
+            else:
+                controlflow.invalid_argument_exit(sys.argv[i], 'gam update org')
+        gapi.call(cd.orgunits(),
+                  'update',
+                  customerId=GC_Values[GC_CUSTOMER_ID],
+                  orgUnitPath=encodeOrgUnitPath(
+                      makeOrgUnitPathRelative(orgUnitPath)),
+                  body=body)
+
+
+def orgUnitPathQuery(path, checkSuspended):
+    query = "orgUnitPath='{0}'".format(path.replace(
+        "'", "\\'")) if path != '/' else ''
+    if checkSuspended is not None:
+        query += f' isSuspended={checkSuspended}'
+    return query
+
+
+def makeOrgUnitPathAbsolute(path):
+    if path == '/':
+        return path
+    if path.startswith('/'):
+        return path.rstrip('/')
+    if path.startswith('id:'):
+        return path
+    if path.startswith('uid:'):
+        return path[1:]
+    return '/' + path.rstrip('/')
+
+
+def makeOrgUnitPathRelative(path):
+    if path == '/':
+        return path
+    if path.startswith('/'):
+        return path[1:].rstrip('/')
+    if path.startswith('id:'):
+        return path
+    if path.startswith('uid:'):
+        return path[1:]
+    return path.rstrip('/')
+
+
+def encodeOrgUnitPath(path):
+    if path.find('+') == -1 and path.find('%') == -1:
+        return path
+    encpath = ''
+    for c in path:
+        if c == '+':
+            encpath += '%2B'
+        elif c == '%':
+            encpath += '%25'
+        else:
+            encpath += c
+    return encpath
+
+
+def getOrgUnitItem(orgUnit, pathOnly=False, absolutePath=True):
+    if pathOnly and (orgUnit.startswith('id:') or orgUnit.startswith('uid:')):
+        controlflow.system_error_exit(
+            2, f'{orgUnit} is not valid in this context')
+    if absolutePath:
+        return makeOrgUnitPathAbsolute(orgUnit)
+    return makeOrgUnitPathRelative(orgUnit)
+
+
+def getTopLevelOrgId(cd, orgUnitPath):
+    try:
+        # create a temp org so we can learn what the top level org ID is (sigh)
+        temp_org = gapi.call(cd.orgunits(),
+                             'insert',
+                             customerId=GC_Values[GC_CUSTOMER_ID],
+                             body={
+                                 'name': 'temp-delete-me',
+                                 'parentOrgUnitPath': orgUnitPath
+                             },
+                             fields='parentOrgUnitId,orgUnitId')
+        gapi.call(cd.orgunits(),
+                  'delete',
+                  customerId=GC_Values[GC_CUSTOMER_ID],
+                  orgUnitPath=temp_org['orgUnitId'])
+        return temp_org['parentOrgUnitId']
+    except:
+        pass
+    return None
+
+
+def getOrgUnitId(orgUnit, cd=None):
+    if cd is None:
+        cd = buildGAPIObject('directory')
+    orgUnit = getOrgUnitItem(orgUnit)
+    if orgUnit[:3] == 'id:':
+        return (orgUnit, orgUnit)
+    if orgUnit == '/':
+        result = gapi.call(cd.orgunits(),
+                           'list',
+                           customerId=GC_Values[GC_CUSTOMER_ID],
+                           orgUnitPath='/',
+                           type='children',
+                           fields='organizationUnits(parentOrgUnitId)')
+        if result.get('organizationUnits', []):
+            return (orgUnit, result['organizationUnits'][0]['parentOrgUnitId'])
+        topLevelOrgId = getTopLevelOrgId(cd, '/')
+        if topLevelOrgId:
+            return (orgUnit, topLevelOrgId)
+        return (orgUnit, '/')  #Bogus but should never happen
+    result = gapi.call(cd.orgunits(),
+                       'get',
+                       customerId=GC_Values[GC_CUSTOMER_ID],
+                       orgUnitPath=encodeOrgUnitPath(
+                           makeOrgUnitPathRelative(orgUnit)),
+                       fields='orgUnitId')
+    return (orgUnit, result['orgUnitId'])
+
+
+def buildOrgUnitIdToNameMap():
+    cd = gapi_directory.build()
+    result = gapi.call(cd.orgunits(),
+                       'list',
+                       customerId=GC_Values[GC_CUSTOMER_ID],
+                       fields='organizationUnits(orgUnitPath,orgUnitId)',
+                       type='all')
+    GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME] = {}
+    for orgUnit in result['organizationUnits']:
+        GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][
+            orgUnit['orgUnitId']] = orgUnit['orgUnitPath']
+
+
+def orgunit_from_orgunitid(orgunitid):
+    if not GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME]:
+        buildOrgUnitIdToNameMap()
+    return GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME].get(orgunitid, orgunitid)

src/gam/gapi/directory/privileges.py

@@ -0,0 +1,32 @@
+from gam.var import GC_Values, GC_CUSTOMER_ID
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+
+
+def flatten_privilege_list(privs, parent=None):
+    flat_privs = []
+    for priv in privs:
+        children = []
+        if parent:
+            priv['parent'] = parent
+        if priv.get('childPrivileges'):
+            children = flatten_privilege_list(priv['childPrivileges'],
+                                              parent=priv['privilegeName'])
+            priv['children'] = ' '.join(
+                [child['privilegeName'] for child in children])
+            del (priv['childPrivileges'])
+        flat_privs = flat_privs + children
+        flat_privs.append(priv)
+    return flat_privs
+
+
+def print_(return_only=False):
+    cd = gapi_directory.build()
+    privs = gapi.call(cd.privileges(),
+                      'list',
+                      customer=GC_Values[GC_CUSTOMER_ID])
+    privs = flatten_privilege_list(privs.get('items', []))
+    if return_only:
+        return privs
+    display.print_json(privs)

src/gam/gapi/directory/resource.py

@@ -0,0 +1,534 @@
+import sys
+import uuid
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
+
+
+def printBuildings():
+    to_drive = False
+    cd = gapi_directory.build()
+    titles = []
+    csvRows = []
+    fieldsList = ['buildingId']
+    # buildings.list() currently doesn't support paging
+    # but should soon, attempt to use it now so we
+    # won't break when it's turned on.
+    fields = 'nextPageToken,buildings(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Building']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        # Allows shorter arguments like "name" instead of "buildingname"
+        elif 'building' + myarg in possible_fields:
+            fieldsList.append(possible_fields['building' + myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print buildings')
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    buildings = gapi.get_all_pages(cd.resources().buildings(),
+                                   'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    for building in buildings:
+        building.pop('etags', None)
+        building.pop('etag', None)
+        building.pop('kind', None)
+        if 'buildingId' in building:
+            building['buildingId'] = f'id:{building["buildingId"]}'
+        if 'floorNames' in building:
+            building['floorNames'] = ','.join(building['floorNames'])
+        building = utils.flatten_json(building)
+        for item in building:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(building)
+    display.sort_csv_titles('buildingId', titles)
+    display.write_csv_file(csvRows, titles, 'Buildings', to_drive)
+
+
+def printResourceCalendars():
+    cd = gapi_directory.build()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    query = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'allfields':
+            fieldsList = []
+            fieldsTitles = {}
+            titles = []
+            for field in RESCAL_ALLFIELDS:
+                display.add_field_to_csv_file(field,
+                                              RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                              fieldsList, fieldsTitles, titles)
+            i += 1
+        elif myarg in RESCAL_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_csv_file(myarg,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print resources')
+    if not fieldsList:
+        for field in RESCAL_DFLTFIELDS:
+            display.add_field_to_csv_file(field,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+    fields = f'nextPageToken,items({",".join(set(fieldsList))})'
+    if 'buildingId' in fieldsList:
+        display.add_field_to_csv_file('buildingName',
+                                      {'buildingName': ['buildingName',]},
+                                      fieldsList, fieldsTitles, titles)
+    gam.printGettingAllItems('Resource Calendars', None)
+    page_message = gapi.got_total_items_first_last_msg('Resource Calendars')
+    resources = gapi.get_all_pages(cd.resources().calendars(),
+                                   'list',
+                                   'items',
+                                   page_message=page_message,
+                                   message_attribute='resourceId',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   query=query,
+                                   fields=fields)
+    for resource in resources:
+        if 'featureInstances' in resource:
+            features = [a_feature['feature']['name'] for \
+                a_feature in resource['featureInstances']]
+            resource['featureInstances'] = ','.join(features)
+        if 'buildingId' in resource:
+            resource['buildingName'] = getBuildingNameById(
+                cd, resource['buildingId'])
+            resource['buildingId'] = f'id:{resource["buildingId"]}'
+        resUnit = {}
+        for field in fieldsList:
+            resUnit[fieldsTitles[field]] = resource.get(field, '')
+        csvRows.append(resUnit)
+    display.sort_csv_titles(['resourceId', 'resourceName', 'resourceEmail'],
+                            titles)
+    display.write_csv_file(csvRows, titles, 'Resources', todrive)
+
+
+RESCAL_DFLTFIELDS = [
+    'id',
+    'name',
+    'email',
+]
+RESCAL_ALLFIELDS = [
+    'id',
+    'name',
+    'email',
+    'description',
+    'type',
+    'buildingid',
+    'category',
+    'capacity',
+    'features',
+    'floor',
+    'floorsection',
+    'generatedresourcename',
+    'uservisibledescription',
+]
+
+RESCAL_ARGUMENT_TO_PROPERTY_MAP = {
+    'description': ['resourceDescription'],
+    'building': ['buildingId',],
+    'buildingid': ['buildingId',],
+    'capacity': ['capacity',],
+    'category': ['resourceCategory',],
+    'email': ['resourceEmail'],
+    'feature': ['featureInstances',],
+    'features': ['featureInstances',],
+    'floor': ['floorName',],
+    'floorname': ['floorName',],
+    'floorsection': ['floorSection',],
+    'generatedresourcename': ['generatedResourceName',],
+    'id': ['resourceId'],
+    'name': ['resourceName'],
+    'type': ['resourceType'],
+    'userdescription': ['userVisibleDescription',],
+    'uservisibledescription': ['userVisibleDescription',],
+}
+
+
+def printFeatures():
+    to_drive = False
+    cd = gapi_directory.build()
+    titles = []
+    csvRows = []
+    fieldsList = ['name']
+    fields = 'nextPageToken,features(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Feature']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        elif 'feature' + myarg in possible_fields:
+            fieldsList.append(possible_fields['feature' + myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print features')
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    features = gapi.get_all_pages(cd.resources().features(),
+                                  'list',
+                                  'features',
+                                  customer=GC_Values[GC_CUSTOMER_ID],
+                                  fields=fields)
+    for feature in features:
+        feature.pop('etags', None)
+        feature.pop('etag', None)
+        feature.pop('kind', None)
+        feature = utils.flatten_json(feature)
+        for item in feature:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(feature)
+    display.sort_csv_titles('name', titles)
+    display.write_csv_file(csvRows, titles, 'Features', to_drive)
+
+
+def _getBuildingAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'id':
+            body['buildingId'] = args[i + 1]
+            i += 2
+        elif myarg == 'name':
+            body['buildingName'] = args[i + 1]
+            i += 2
+        elif myarg in ['lat', 'latitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['latitude'] = args[i + 1]
+            i += 2
+        elif myarg in ['long', 'lng', 'longitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['longitude'] = args[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = args[i + 1]
+            i += 2
+        elif myarg == 'floors':
+            body['floorNames'] = args[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg,
+                                              'gam create|update building')
+    return body
+
+
+def createBuilding():
+    cd = gapi_directory.build()
+    body = {
+        'floorNames': ['1'],
+        'buildingId': str(uuid.uuid4()),
+        'buildingName': sys.argv[3]
+    }
+    body = _getBuildingAttributes(sys.argv[4:], body)
+    print(f'Creating building {body["buildingId"]}...')
+    gapi.call(cd.resources().buildings(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def _makeBuildingIdNameMap(cd):
+    fields = 'nextPageToken,buildings(buildingId,buildingName)'
+    buildings = gapi.get_all_pages(cd.resources().buildings(),
+                                   'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {}
+    GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {}
+    for building in buildings:
+        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][
+            building['buildingId']] = building['buildingName']
+        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][
+            building['buildingName']] = building['buildingId']
+
+
+def getBuildingByNameOrId(cd, which_building, minLen=1):
+    if not which_building or \
+       (minLen == 0 and which_building in ['id:', 'uid:']):
+        if minLen == 0:
+            return ''
+        controlflow.system_error_exit(3, 'Building id/name is empty')
+    cg = UID_PATTERN.match(which_building)
+    if cg:
+        return cg.group(1)
+    if GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] is None:
+        _makeBuildingIdNameMap(cd)
+    # Exact name match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID]:
+        return GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][which_building]
+    # No exact name match, check for case insensitive name matches
+    which_building_lower = which_building.lower()
+    ci_matches = []
+    for buildingName, buildingId in GM_Globals[
+            GM_MAP_BUILDING_NAME_TO_ID].items():
+        if buildingName.lower() == which_building_lower:
+            ci_matches.append({
+                'buildingName': buildingName,
+                'buildingId': buildingId
+            })
+    # One match, return ID
+    if len(ci_matches) == 1:
+        return ci_matches[0]['buildingId']
+    # No or multiple name matches, try ID
+    # Exact ID match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        return which_building
+    # No exact ID match, check for case insensitive id match
+    for buildingId in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        # Match, return ID
+        if buildingId.lower() == which_building_lower:
+            return buildingId
+    # Multiple name  matches
+    if len(ci_matches) > 1:
+        message = 'Multiple buildings with same name:\n'
+        for building in ci_matches:
+            message += f'  Name:{building["buildingName"]}  ' \
+                       f'id:{building["buildingId"]}\n'
+        message += '\nPlease specify building name by exact case or by id.'
+        controlflow.system_error_exit(3, message)
+    # No matches
+    else:
+        controlflow.system_error_exit(3, f'No such building {which_building}')
+
+
+def getBuildingNameById(cd, buildingId):
+    if GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] is None:
+        _makeBuildingIdNameMap(cd)
+    return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, 'UNKNOWN')
+
+
+def updateBuilding():
+    cd = gapi_directory.build()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    body = _getBuildingAttributes(sys.argv[4:])
+    print(f'Updating building {buildingId}...')
+    gapi.call(cd.resources().buildings(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              buildingId=buildingId,
+              body=body)
+
+
+def getBuildingInfo():
+    cd = gapi_directory.build()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    building = gapi.call(cd.resources().buildings(),
+                         'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         buildingId=buildingId)
+    if 'buildingId' in building:
+        building['buildingId'] = f'id:{building["buildingId"]}'
+    if 'floorNames' in building:
+        building['floorNames'] = ','.join(building['floorNames'])
+    if 'buildingName' in building:
+        sys.stdout.write(building.pop('buildingName'))
+    display.print_json(building)
+
+
+def deleteBuilding():
+    cd = gapi_directory.build()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    print(f'Deleting building {buildingId}...')
+    gapi.call(cd.resources().buildings(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              buildingId=buildingId)
+
+
+def _getFeatureAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = args[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg,
+                                              'gam create|update feature')
+    return body
+
+
+def createFeature():
+    cd = gapi_directory.build()
+    body = _getFeatureAttributes(sys.argv[3:])
+    print(f'Creating feature {body["name"]}...')
+    gapi.call(cd.resources().features(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def updateFeature():
+    # update does not work for name and name is only field to be updated
+    # if additional writable fields are added to feature in the future
+    # we'll add support for update as well as rename
+    cd = gapi_directory.build()
+    oldName = sys.argv[3]
+    body = {'newName': sys.argv[5:]}
+    print(f'Updating feature {oldName}...')
+    gapi.call(cd.resources().features(),
+              'rename',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              oldName=oldName,
+              body=body)
+
+
+def deleteFeature():
+    cd = gapi_directory.build()
+    featureKey = sys.argv[3]
+    print(f'Deleting feature {featureKey}...')
+    gapi.call(cd.resources().features(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              featureKey=featureKey)
+
+
+def _getResourceCalendarAttributes(cd, args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['resourceName'] = args[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['resourceDescription'] = args[i + 1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'type':
+            body['resourceType'] = args[i + 1]
+            i += 2
+        elif myarg in ['building', 'buildingid']:
+            body['buildingId'] = getBuildingByNameOrId(cd,
+                                                       args[i + 1],
+                                                       minLen=0)
+            i += 2
+        elif myarg in ['capacity']:
+            body['capacity'] = gam.getInteger(args[i + 1], myarg, minVal=0)
+            i += 2
+        elif myarg in ['feature', 'features']:
+            features = args[i + 1].split(',')
+            body['featureInstances'] = []
+            for feature in features:
+                instance = {'feature': {'name': feature}}
+                body['featureInstances'].append(instance)
+            i += 2
+        elif myarg in ['floor', 'floorname']:
+            body['floorName'] = args[i + 1]
+            i += 2
+        elif myarg in ['floorsection']:
+            body['floorSection'] = args[i + 1]
+            i += 2
+        elif myarg in ['category']:
+            body['resourceCategory'] = args[i + 1].upper()
+            if body['resourceCategory'] == 'ROOM':
+                body['resourceCategory'] = 'CONFERENCE_ROOM'
+            i += 2
+        elif myarg in ['uservisibledescription', 'userdescription']:
+            body['userVisibleDescription'] = args[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(args[i],
+                                              'gam create|update resource')
+    return body
+
+
+def createResourceCalendar():
+    cd = gapi_directory.build()
+    body = {'resourceId': sys.argv[3], 'resourceName': sys.argv[4]}
+    body = _getResourceCalendarAttributes(cd, sys.argv[5:], body)
+    print(f'Creating resource {body["resourceId"]}...')
+    gapi.call(cd.resources().calendars(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+
+def updateResourceCalendar():
+    cd = gapi_directory.build()
+    resId = sys.argv[3]
+    body = _getResourceCalendarAttributes(cd, sys.argv[4:])
+    # Use patch since it seems to work better.
+    # update requires name to be set.
+    gapi.call(cd.resources().calendars(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              calendarResourceId=resId,
+              body=body,
+              fields='')
+    print(f'updated resource {resId}')
+
+
+def getResourceCalendarInfo():
+    cd = gapi_directory.build()
+    resId = sys.argv[3]
+    resource = gapi.call(cd.resources().calendars(),
+                         'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         calendarResourceId=resId)
+    if 'featureInstances' in resource:
+        features = []
+        for a_feature in resource.pop('featureInstances'):
+            features.append(a_feature['feature']['name'])
+        resource['features'] = ', '.join(features)
+    if 'buildingId' in resource:
+        resource['buildingName'] = getBuildingNameById(cd,
+                                                       resource['buildingId'])
+        resource['buildingId'] = f'id:{resource["buildingId"]}'
+    display.print_json(resource)
+
+
+def deleteResourceCalendar():
+    resId = sys.argv[3]
+    cd = gapi_directory.build()
+    print(f'Deleting resource calendar {resId}')
+    gapi.call(cd.resources().calendars(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              calendarResourceId=resId)

src/gam/gapi/directory/roles.py

@@ -0,0 +1,124 @@
+import sys
+
+from gam.var import GC_Values, GC_CUSTOMER_ID
+import gam
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi.directory import privileges as gapi_directory_privileges
+
+
+def getPrivileges(body, privs, action):
+    all_privileges = gapi_directory_privileges.print_(return_only=True)
+    if privs == 'ALL':
+        body['rolePrivileges'] = [
+            {'privilegeName': p['privilegeName'], 'serviceId': p['serviceId']} for p in all_privileges
+            ]
+    elif privs == 'ALL_OU':
+        body['rolePrivileges'] = [
+            {'privilegeName': p['privilegeName'], 'serviceId': p['serviceId']} for p in all_privileges if p.get('isOuScopable')
+            ]
+    else:
+      body.setdefault('rolePrivileges', [])
+      for priv in privs.split(','):
+          for p in all_privileges:
+              if priv == p['privilegeName']:
+                body['rolePrivileges'].append({'privilegeName': p['privilegeName'], 'serviceId': p['serviceId']})
+                break
+          else:
+              controlflow.invalid_argument_exit(priv,
+                                                f'gam {action} adminrole privileges')
+
+def create():
+    cd = gapi_directory.build()
+    body = {'roleName': sys.argv[3]}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'privileges':
+            getPrivileges(body, sys.argv[i + 1].upper(), 'create')
+            i += 2
+        elif myarg == 'description':
+            body['roleDescription'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam create adminrole')
+
+    if not body.get('rolePrivileges'):
+        controlflow.missing_argument_exit('privileges',
+                                          'gam create adminrole')
+    print(f'Creating role {body["roleName"]}')
+    gapi.call(cd.roles(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+
+def update():
+    cd = gapi_directory.build()
+    body = {}
+    roleId = gam.getRoleId(sys.argv[3])
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'privileges':
+            getPrivileges(body, sys.argv[i + 1].upper(), 'update')
+            i += 2
+        elif myarg == 'description':
+            body['roleDescription'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'name':
+            body['roleName'] = sys.argv[i + 1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam update adminrole')
+
+    print(f'Updating role {roleId}')
+    gapi.call(cd.roles(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              roleId=roleId,
+              body=body)
+
+
+def delete():
+    cd = gapi_directory.build()
+    roleId = gam.getRoleId(sys.argv[3])
+    print(f'Deleting role {roleId}')
+    gapi.call(cd.roles(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              roleId=roleId)
+
+
+def print_():
+    cd = gapi_directory.build()
+    todrive = False
+    titles = [
+        'roleId', 'roleName', 'roleDescription', 'isSuperAdminRole',
+        'isSystemRole'
+    ]
+    fields = f'nextPageToken,items({",".join(titles)})'
+    csvRows = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print adminroles')
+    roles = gapi.get_all_pages(cd.roles(),
+                               'list',
+                               'items',
+                               customer=GC_Values[GC_CUSTOMER_ID],
+                               fields=fields)
+    for role in roles:
+        role_attrib = {}
+        for key, value in list(role.items()):
+            role_attrib[key] = value
+        csvRows.append(role_attrib)
+    display.write_csv_file(csvRows, titles, 'Admin Roles', todrive)

src/gam/gapi/directory/users.py

@@ -0,0 +1,32 @@
+import gam
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+
+def signout(users):
+    cd = gapi_directory.build()
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user = gam.normalizeEmailAddressOrUID(user)
+        print(f'Signing Out {user}{gam.currentCount(i, count)}')
+        gapi.call(cd.users(),
+                  'signOut',
+                  soft_errors=True,
+                  userKey=user)
+
+
+def turn_off_2sv(users):
+    cd = gapi_directory.build()
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user = gam.normalizeEmailAddressOrUID(user)
+        print(f'Turning Off 2-Step Verification for {user}{gam.currentCount(i, count)}')
+        gapi.call(cd.twoStepVerification(),
+                  'turnOff',
+                  soft_errors=True,
+                  userKey=user)
+
+

src/gam/gapi/errors.py

@@ -0,0 +1,380 @@
+"""GAPI and OAuth Token related errors methods."""
+
+from enum import Enum
+import json
+
+from gam import controlflow
+from gam import display
+from gam.var import UTF8
+
+
+class GapiAbortedError(Exception):
+    pass
+
+
+class GapiAuthErrorError(Exception):
+    pass
+
+
+class GapiBadGatewayError(Exception):
+    pass
+
+
+class GapiBadRequestError(Exception):
+    pass
+
+
+class GapiConditionNotMetError(Exception):
+    pass
+
+
+class GapiCyclicMembershipsNotAllowedError(Exception):
+    pass
+
+
+class GapiDomainCannotUseApisError(Exception):
+    pass
+
+
+class GapiDomainNotFoundError(Exception):
+    pass
+
+
+class GapiDuplicateError(Exception):
+    pass
+
+
+class GapiFailedPreconditionError(Exception):
+    pass
+
+
+class GapiForbiddenError(Exception):
+    pass
+
+
+class GapiGatewayTimeoutError(Exception):
+    pass
+
+
+class GapiGroupNotFoundError(Exception):
+    pass
+
+
+class GapiInvalidError(Exception):
+    pass
+
+
+class GapiInvalidArgumentError(Exception):
+    pass
+
+
+class GapiInvalidMemberError(Exception):
+    pass
+
+
+class GapiMemberNotFoundError(Exception):
+    pass
+
+
+class GapiNotFoundError(Exception):
+    pass
+
+
+class GapiNotImplementedError(Exception):
+    pass
+
+
+class GapiPermissionDeniedError(Exception):
+    pass
+
+
+class GapiResourceNotFoundError(Exception):
+    pass
+
+
+class GapiServiceNotAvailableError(Exception):
+    pass
+
+
+class GapiUserNotFoundError(Exception):
+    pass
+
+
+# GAPI Error Reasons
+class ErrorReason(Enum):
+    """The reason why a non-200 HTTP response was returned from a GAPI."""
+    ABORTED = 'aborted'
+    AUTH_ERROR = 'authError'
+    BACKEND_ERROR = 'backendError'
+    BAD_GATEWAY = 'badGateway'
+    BAD_REQUEST = 'badRequest'
+    CONDITION_NOT_MET = 'conditionNotMet'
+    CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
+    DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
+    DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
+    DOMAIN_NOT_FOUND = 'domainNotFound'
+    DUPLICATE = 'duplicate'
+    FAILED_PRECONDITION = 'failedPrecondition'
+    FORBIDDEN = 'forbidden'
+    FOUR_O_NINE = '409'
+    FOUR_O_O = '400'
+    FOUR_O_THREE = '403'
+    FOUR_TWO_NINE = '429'
+    GATEWAY_TIMEOUT = 'gatewayTimeout'
+    GROUP_NOT_FOUND = 'groupNotFound'
+    INTERNAL_ERROR = 'internalError'
+    INVALID = 'invalid'
+    INVALID_ARGUMENT = 'invalidArgument'
+    INVALID_MEMBER = 'invalidMember'
+    MEMBER_NOT_FOUND = 'memberNotFound'
+    NOT_FOUND = 'notFound'
+    NOT_IMPLEMENTED = 'notImplemented'
+    PERMISSION_DENIED = 'permissionDenied'
+    QUOTA_EXCEEDED = 'quotaExceeded'
+    RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
+    RESOURCE_NOT_FOUND = 'resourceNotFound'
+    SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+    SERVICE_LIMIT = 'serviceLimit'
+    SYSTEM_ERROR = 'systemError'
+    USER_NOT_FOUND = 'userNotFound'
+    USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+
+    def __str__(self):
+        return str(self.value)
+
+
+# Common sets of GAPI error reasons
+DEFAULT_RETRY_REASONS = [
+    ErrorReason.QUOTA_EXCEEDED,
+    ErrorReason.RATE_LIMIT_EXCEEDED,
+    ErrorReason.USER_RATE_LIMIT_EXCEEDED,
+    ErrorReason.BACKEND_ERROR,
+    ErrorReason.BAD_GATEWAY,
+    ErrorReason.GATEWAY_TIMEOUT,
+    ErrorReason.INTERNAL_ERROR,
+    ErrorReason.FOUR_TWO_NINE,
+]
+GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
+GROUP_GET_THROW_REASONS = [
+    ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
+    ErrorReason.BAD_REQUEST
+]
+GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
+MEMBERS_THROW_REASONS = [
+    ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
+    ErrorReason.FORBIDDEN
+]
+MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
+
+# A map of GAPI error reasons to the corresponding GAM Python Exception
+ERROR_REASON_TO_EXCEPTION = {
+    ErrorReason.ABORTED:
+        GapiAbortedError,
+    ErrorReason.AUTH_ERROR:
+        GapiAuthErrorError,
+    ErrorReason.BAD_GATEWAY:
+        GapiBadGatewayError,
+    ErrorReason.BAD_REQUEST:
+        GapiBadRequestError,
+    ErrorReason.CONDITION_NOT_MET:
+        GapiConditionNotMetError,
+    ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
+        GapiCyclicMembershipsNotAllowedError,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS:
+        GapiDomainCannotUseApisError,
+    ErrorReason.DOMAIN_NOT_FOUND:
+        GapiDomainNotFoundError,
+    ErrorReason.DUPLICATE:
+        GapiDuplicateError,
+    ErrorReason.FAILED_PRECONDITION:
+        GapiFailedPreconditionError,
+    ErrorReason.FORBIDDEN:
+        GapiForbiddenError,
+    ErrorReason.GATEWAY_TIMEOUT:
+        GapiGatewayTimeoutError,
+    ErrorReason.GROUP_NOT_FOUND:
+        GapiGroupNotFoundError,
+    ErrorReason.INVALID:
+        GapiInvalidError,
+    ErrorReason.INVALID_ARGUMENT:
+        GapiInvalidArgumentError,
+    ErrorReason.INVALID_MEMBER:
+        GapiInvalidMemberError,
+    ErrorReason.MEMBER_NOT_FOUND:
+        GapiMemberNotFoundError,
+    ErrorReason.NOT_FOUND:
+        GapiNotFoundError,
+    ErrorReason.NOT_IMPLEMENTED:
+        GapiNotImplementedError,
+    ErrorReason.PERMISSION_DENIED:
+        GapiPermissionDeniedError,
+    ErrorReason.RESOURCE_NOT_FOUND:
+        GapiResourceNotFoundError,
+    ErrorReason.SERVICE_NOT_AVAILABLE:
+        GapiServiceNotAvailableError,
+    ErrorReason.USER_NOT_FOUND:
+        GapiUserNotFoundError,
+}
+
+# OAuth Token Errors
+OAUTH2_TOKEN_ERRORS = [
+    'access_denied',
+    'access_denied: Requested client not authorized',
+    'internal_failure: Backend Error',
+    'internal_failure: None',
+    'invalid_grant',
+    'invalid_grant: Bad Request',
+    'invalid_grant: Invalid email or User ID',
+    'invalid_grant: Not a valid email',
+    'invalid_grant: Invalid JWT: No valid verifier found for issuer',
+    'invalid_grant: The account has been deleted',
+    'invalid_grant: reauth related error (invalid_rapt)',
+    'invalid_request: Invalid impersonation prn email address',
+    'invalid_request: Invalid impersonation "sub" field',
+    'unauthorized_client: Client is unauthorized to retrieve access tokens '
+    'using this method',
+    'unauthorized_client: Client is unauthorized to retrieve access tokens '
+    'using this method, or client not authorized for any of the scopes '
+    'requested',
+    'unauthorized_client: Unauthorized client or scope in request',
+]
+
+
+def _create_http_error_dict(status_code, reason, message):
+    """Creates a basic error dict similar to most Google API Errors.
+
+  Args:
+    status_code: Int, the error's HTTP response status code.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    dict
+  """
+    return {
+        'error': {
+            'code': status_code,
+            'errors': [{
+                'reason': str(reason),
+                'message': message,
+            }]
+        }
+    }
+
+
+def get_gapi_error_detail(e,
+                          soft_errors=False,
+                          silent_errors=False,
+                          retry_on_http_error=False):
+    """Extracts error detail from a non-200 GAPI Response.
+
+  Args:
+    e: googleapiclient.HttpError, The HTTP Error received.
+    soft_errors: Boolean, If true, causes error messages to be surpressed,
+      rather than sending them to stderr.
+    silent_errors: Boolean, If true, suppresses and ignores any errors from
+      being displayed
+    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
+      code, indicating that the request can be retried. TODO: Remove this param,
+        as it seems to be outside the scope of this method.
+
+  Returns:
+    A tuple containing the HTTP Response code, GAPI error reason, and error
+        message.
+  """
+    try:
+        error = json.loads(e.content.decode(UTF8))
+    except ValueError:
+        error_content = e.content.decode(UTF8) if isinstance(
+            e.content, bytes) else e.content
+        if (e.resp['status'] == '503') and (
+                error_content == 'Quota exceeded for the current request'):
+            return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value,
+                    error_content)
+        if (e.resp['status'] == '403') and (error_content.startswith(
+                'Request rate higher than configured')):
+            return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value,
+                    error_content)
+        if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
+            return (e.resp['status'], ErrorReason.BAD_GATEWAY.value,
+                    error_content)
+        if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
+            return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value,
+                    error_content)
+        if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
+            error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
+                                            'Domain not found')
+        elif (e.resp['status'] == '400') and (
+                'InvalidSsoSigningKey' in error_content):
+            error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                            'InvalidSsoSigningKey')
+        elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
+            error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                            'UnknownError')
+        elif retry_on_http_error:
+            return (-1, None, None)
+        elif soft_errors:
+            if not silent_errors:
+                display.print_error(error_content)
+            return (0, None, None)
+        else:
+            controlflow.system_error_exit(5, error_content)
+        # END: ValueError catch
+
+    if 'error' in error:
+        http_status = error['error']['code']
+        try:
+            message = error['error']['errors'][0]['message']
+        except KeyError:
+            message = error['error']['message']
+        if http_status == 404:
+            if 'Requested entity was not found' in message or 'does not exist' in message:
+                error = _create_http_error_dict(404, ErrorReason.NOT_FOUND.value,
+                                                message)
+    else:
+        if 'error_description' in error:
+            if error['error_description'] == 'Invalid Value':
+                message = error['error_description']
+                http_status = 400
+                error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                                message)
+            else:
+                controlflow.system_error_exit(4, str(error))
+        else:
+            controlflow.system_error_exit(4, str(error))
+
+    # Extract the error reason
+    try:
+        reason = error['error']['errors'][0]['reason']
+        if reason == 'notFound':
+            if 'userKey' in message:
+                reason = ErrorReason.USER_NOT_FOUND.value
+            elif 'groupKey' in message:
+                reason = ErrorReason.GROUP_NOT_FOUND.value
+            elif 'memberKey' in message:
+                reason = ErrorReason.MEMBER_NOT_FOUND.value
+            elif 'Domain not found' in message:
+                reason = ErrorReason.DOMAIN_NOT_FOUND.value
+            elif 'Resource Not Found' in message:
+                reason = ErrorReason.RESOURCE_NOT_FOUND.value
+        elif reason == 'invalid':
+            if 'userId' in message:
+                reason = ErrorReason.USER_NOT_FOUND.value
+            elif 'memberKey' in message:
+                reason = ErrorReason.INVALID_MEMBER.value
+        elif reason == 'failedPrecondition':
+            if 'Bad Request' in message:
+                reason = ErrorReason.BAD_REQUEST.value
+            elif 'Mail service not enabled' in message:
+                reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
+        elif reason == 'required':
+            if 'memberKey' in message:
+                reason = ErrorReason.MEMBER_NOT_FOUND.value
+        elif reason == 'conditionNotMet':
+            if 'Cyclic memberships not allowed' in message:
+                reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
+    except KeyError:
+        reason = f'{http_status}'
+    return (http_status, reason, message)

src/gam/gapi/errors_test.py

@@ -0,0 +1,210 @@
+"""Python unit tests for gapi.errors"""
+
+import json
+
+import unittest
+from unittest.mock import patch
+
+import googleapiclient.errors
+from gam.gapi import errors
+
+
+def create_simple_http_error(status, reason, message):
+    content = errors._create_http_error_dict(status, reason, message)
+    return create_http_error(status, content)
+
+
+def create_http_error(status, content):
+    response = {
+        'status': status,
+        'content-type': 'application/json',
+    }
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    return googleapiclient.errors.HttpError(response, content_as_bytes)
+
+
+class ErrorsTest(unittest.TestCase):
+
+    def test_get_gapi_error_detail_quota_exceeded(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_invalid_domain(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_invalid_signing_key(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_unknown_error(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_retry_http_error(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_prints_soft_errors(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_exits_on_unrecoverable_errors(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_quota_exceeded_for_current_request(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_quota_exceeded_high_request_rate(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_extracts_user_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: userKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: userKey.')
+
+    def test_get_gapi_error_extracts_group_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: groupKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: groupKey.')
+
+    def test_get_gapi_error_extracts_member_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: memberKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: memberKey.')
+
+    def test_get_gapi_error_extracts_domain_not_found(self):
+        err = create_simple_http_error(404, 'notFound', 'Domain not found.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
+        self.assertEqual(message, 'Domain not found.')
+
+    def test_get_gapi_error_extracts_generic_resource_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: unknownResource.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: unknownResource.')
+
+    def test_get_gapi_error_extracts_invalid_userid(self):
+        err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+        self.assertEqual(message, 'Invalid Input: userId')
+
+    def test_get_gapi_error_extracts_invalid_member(self):
+        err = create_simple_http_error(400, 'invalid',
+                                       'Invalid Input: memberKey')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
+        self.assertEqual(message, 'Invalid Input: memberKey')
+
+    def test_get_gapi_error_extracts_bad_request(self):
+        err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
+        self.assertEqual(message, 'Bad Request')
+
+    def test_get_gapi_error_extracts_service_not_available(self):
+        err = create_simple_http_error(400, 'failedPrecondition',
+                                       'Mail service not enabled')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
+        self.assertEqual(message, 'Mail service not enabled')
+
+    def test_get_gapi_error_extracts_required_member_not_found(self):
+        err = create_simple_http_error(400, 'required',
+                                       'Missing required field: memberKey')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+        self.assertEqual(message, 'Missing required field: memberKey')
+
+    def test_get_gapi_error_extracts_cyclic_memberships_error(self):
+        err = create_simple_http_error(400, 'conditionNotMet',
+                                       'Cyclic memberships not allowed')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(
+            reason, errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
+        self.assertEqual(message, 'Cyclic memberships not allowed')
+
+    def test_get_gapi_error_extracts_single_error_with_message(self):
+        status_code = 999
+        response = {'status': status_code}
+        # This error does not have an "errors" key describing each error.
+        content = {'error': {'code': status_code, 'message': 'unknown error'}}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, status_code)
+        self.assertEqual(reason, str(status_code))
+        self.assertEqual(message, content['error']['message'])
+
+    def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
+        self):
+        status_code = 999
+        response = {'status': status_code}
+        # This error only has an error_description_field and an unknown description.
+        content = {'error_description': 'something errored'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        with self.assertRaises(SystemExit) as context:
+            errors.get_gapi_error_detail(err)
+        self.assertEqual(4, context.exception.code)
+
+    def test_get_gapi_error_exits_on_invalid_error_description(self):
+        status_code = 400
+        response = {'status': status_code}
+        content = {'error_description': 'Invalid Value'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, status_code)
+        self.assertEqual(reason, errors.ErrorReason.INVALID.value)
+        self.assertEqual(message, 'Invalid Value')
+
+    def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
+        status_code = 900
+        response = {'status': status_code}
+        content = {'notErrorContentThatIsExpected': 'foo'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        with self.assertRaises(SystemExit) as context:
+            errors.get_gapi_error_detail(err)
+        self.assertEqual(4, context.exception.code)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/reports.py

@@ -0,0 +1,584 @@
+import calendar
+import datetime
+import re
+import sys
+
+from dateutil.relativedelta import relativedelta
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam import utils
+
+
+def build():
+    return gam.buildGAPIObject('reports')
+
+
+REPORT_CHOICE_MAP = {
+    'access': 'access_transparency',
+    'accesstransparency': 'access_transparency',
+    'calendars': 'calendar',
+    'customers': 'customer',
+    'doc': 'drive',
+    'docs': 'drive',
+    'domain': 'customer',
+    'enterprisegroups': 'groups_enterprise',
+    'google+': 'gplus',
+    'group': 'groups',
+    'groupsenterprise': 'groups_enterprise',
+    'hangoutsmeet': 'meet',
+    'logins': 'login',
+    'oauthtoken': 'token',
+    'tokens': 'token',
+    'usage': 'usage',
+    'usageparameters': 'usageparameters',
+    'users': 'user',
+    'useraccounts': 'user_accounts',
+}
+
+
+def showUsageParameters():
+    rep = build()
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.BAD_REQUEST
+    ]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit('user or customer',
+                                          'report usageparameters')
+    report = sys.argv[3].lower()
+    titles = ['parameter']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = {}
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = {'userKey': gam._get_admin_email()}
+    else:
+        controlflow.expected_argument_exit('usageparameters',
+                                           ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    all_parameters = set()
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam report usageparameters')
+    fullDataRequired = ['all']
+    while True:
+        try:
+            result = gapi.call(endpoint,
+                               'get',
+                               throw_reasons=throw_reasons,
+                               date=tryDate,
+                               customerId=customerId,
+                               fields='warnings,usageReports(parameters(name))',
+                               **kwargs)
+            warnings = result.get('warnings', [])
+            usage = result.get('usageReports')
+            has_reports = bool(usage)
+            fullData, tryDate = _check_full_data_available(
+                warnings, tryDate, fullDataRequired, has_reports)
+            if fullData < 0:
+                print('No usage parameters available.')
+                sys.exit(1)
+            if has_reports:
+                for parameter in usage[0]['parameters']:
+                    name = parameter.get('name')
+                    if name:
+                        all_parameters.add(name)
+            if fullData == 1:
+                break
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = _adjust_date(str(e))
+    csvRows = []
+    for parameter in sorted(all_parameters):
+        csvRows.append({'parameter': parameter})
+    display.write_csv_file(csvRows, titles,
+                           f'{report.capitalize()} Report Usage Parameters',
+                           todrive)
+
+
+REPORTS_PARAMETERS_SIMPLE_TYPES = [
+    'intValue', 'boolValue', 'datetimeValue', 'stringValue'
+]
+
+
+def showUsage():
+    rep = build()
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.BAD_REQUEST
+    ]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit('user or customer', 'report usage')
+    report = sys.argv[3].lower()
+    titles = ['date']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = [{}]
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = [{'userKey': 'all'}]
+        titles.append('user')
+    else:
+        controlflow.expected_argument_exit('usage', ['user', 'customer'],
+                                           report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    parameters = []
+    start_date = end_date = orgUnitId = None
+    skip_day_numbers = []
+    skip_dates = set()
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'startdate':
+            start_date = utils.get_yyyymmdd(sys.argv[i + 1],
+                                            returnDateTime=True)
+            i += 2
+        elif myarg == 'enddate':
+            end_date = utils.get_yyyymmdd(sys.argv[i + 1], returnDateTime=True)
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i + 1].split(',')
+            i += 2
+        elif myarg == 'skipdates':
+            for skip in sys.argv[i + 1].split(','):
+                if skip.find(':') == -1:
+                    skip_dates.add(utils.get_yyyymmdd(skip,
+                                                      returnDateTime=True))
+                else:
+                    skip_start, skip_end = skip.split(':', 1)
+                    skip_start = utils.get_yyyymmdd(skip_start,
+                                                    returnDateTime=True)
+                    skip_end = utils.get_yyyymmdd(skip_end, returnDateTime=True)
+                    while skip_start <= skip_end:
+                        skip_dates.add(skip_start)
+                        skip_start += one_day
+            i += 2
+        elif myarg == 'skipdaysofweek':
+            skipdaynames = sys.argv[i + 1].split(',')
+            dow = [d.lower() for d in calendar.day_abbr]
+            skip_day_numbers = [dow.index(d) for d in skipdaynames if d in dow]
+            i += 2
+        elif report == 'user' and myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = gam.getOrgUnitId(sys.argv[i + 1])
+            i += 2
+        elif report == 'user' and myarg in usergroup_types:
+            users = gam.getUsersToModify(myarg, sys.argv[i + 1])
+            kwargs = [{'userKey': user} for user in users]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              f'gam report usage {report}')
+    if parameters:
+        titles.extend(parameters)
+        parameters = ','.join(parameters)
+    else:
+        parameters = None
+    if not end_date:
+        end_date = datetime.datetime.now()
+    if not start_date:
+        start_date = end_date + relativedelta(months=-1)
+    if orgUnitId:
+        for kw in kwargs:
+            kw['orgUnitID'] = orgUnitId
+    usage_on_date = start_date
+    start_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+    usage_end_date = end_date
+    end_date = end_date.strftime(YYYYMMDD_FORMAT)
+    start_use_date = end_use_date = None
+    csvRows = []
+    while usage_on_date <= usage_end_date:
+        if usage_on_date.weekday() in skip_day_numbers or \
+           usage_on_date in skip_dates:
+            usage_on_date += one_day
+            continue
+        use_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+        usage_on_date += one_day
+        try:
+            for kwarg in kwargs:
+                try:
+                    usage = gapi.get_all_pages(endpoint,
+                                               'get',
+                                               'usageReports',
+                                               throw_reasons=throw_reasons,
+                                               customerId=customerId,
+                                               date=use_date,
+                                               parameters=parameters,
+                                               **kwarg)
+                except gapi.errors.GapiBadRequestError:
+                    continue
+                for entity in usage:
+                    row = {'date': use_date}
+                    if 'userEmail' in entity['entity']:
+                        row['user'] = entity['entity']['userEmail']
+                    for item in entity.get('parameters', []):
+                        if 'name' not in item:
+                            continue
+                        name = item['name']
+                        if name == 'cros:device_version_distribution':
+                            for cros_ver in item['msgValue']:
+                                v = cros_ver['version_number']
+                                column_name = f'cros:num_devices_chrome_{v}'
+                                if column_name not in titles:
+                                    titles.append(column_name)
+                                row[column_name] = cros_ver['num_devices']
+                        else:
+                            if not name in titles:
+                                titles.append(name)
+                            for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                                if ptype in item:
+                                    row[name] = item[ptype]
+                                    break
+                            else:
+                                row[name] = ''
+                    if not start_use_date:
+                        start_use_date = use_date
+                    end_use_date = use_date
+                    csvRows.append(row)
+        except gapi.errors.GapiInvalidError as e:
+            display.print_warning(str(e))
+            break
+    if start_use_date:
+        report_name = f'{report.capitalize()} Usage Report - {start_use_date}:{end_use_date}'
+    else:
+        report_name = f'{report.capitalize()} Usage Report - {start_date}:{end_date} - No Data'
+    display.write_csv_file(csvRows, titles, report_name, todrive)
+
+
+def showReport():
+    rep = build()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    report = sys.argv[2].lower()
+    report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    if report == 'usage':
+        showUsage()
+        return
+    if report == 'usageparameters':
+        showUsageParameters()
+        return
+    valid_apps = gapi.get_enum_values_minus_unspecified(
+        rep._rootDesc['resources']['activities']['methods']['list']
+        ['parameters']['applicationName']['enum']) + ['customer', 'user']
+    if report not in valid_apps:
+        controlflow.expected_argument_exit('report',
+                                           ', '.join(sorted(valid_apps)),
+                                           report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    filters = parameters = actorIpAddress = startTime = endTime = eventName = orgUnitId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    to_drive = False
+    userKey = 'all'
+    fullDataRequired = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'date':
+            tryDate = utils.get_yyyymmdd(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = gam.getOrgUnitId(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'fulldatarequired':
+            fullDataRequired = []
+            fdr = sys.argv[i + 1].lower()
+            if fdr and fdr == 'all':
+                fullDataRequired = 'all'
+            else:
+                fullDataRequired = fdr.replace(',', ' ').split()
+            i += 2
+        elif myarg == 'start':
+            startTime = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'end':
+            endTime = utils.get_time_or_delta_from_now(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'event':
+            eventName = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'user':
+            userKey = sys.argv[i + 1].lower()
+            if userKey != 'all':
+                userKey = gam.normalizeEmailAddressOrUID(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['filter', 'filters']:
+            filters = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'ip':
+            actorIpAddress = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'todrive':
+            to_drive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam report')
+    if report == 'user':
+        while True:
+            try:
+                one_page = gapi.call(rep.userUsageReport(),
+                                     'get',
+                                     throw_reasons=throw_reasons,
+                                     date=tryDate,
+                                     userKey=userKey,
+                                     customerId=customerId,
+                                     orgUnitID=orgUnitId,
+                                     fields='warnings,usageReports',
+                                     maxResults=1)
+                warnings = one_page.get('warnings', [])
+                has_reports = bool(one_page.get('usageReports'))
+                fullData, tryDate = _check_full_data_available(
+                    warnings, tryDate, fullDataRequired, has_reports)
+                if fullData < 0:
+                    print('No user report available.')
+                    sys.exit(1)
+                if fullData == 0:
+                    continue
+                page_message = gapi.got_total_items_msg('Users', '...\n')
+                usage = gapi.get_all_pages(rep.userUsageReport(),
+                                           'get',
+                                           'usageReports',
+                                           page_message=page_message,
+                                           throw_reasons=throw_reasons,
+                                           date=tryDate,
+                                           userKey=userKey,
+                                           customerId=customerId,
+                                           orgUnitID=orgUnitId,
+                                           filters=filters,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No user report available.')
+            sys.exit(1)
+        titles = ['email', 'date']
+        csvRows = []
+        for user_report in usage:
+            if 'entity' not in user_report:
+                continue
+            row = {'email': user_report['entity']['userEmail'], 'date': tryDate}
+            for item in user_report.get('parameters', []):
+                if 'name' not in item:
+                    continue
+                name = item['name']
+                if not name in titles:
+                    titles.append(name)
+                for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                    if ptype in item:
+                        row[name] = item[ptype]
+                        break
+                else:
+                    row[name] = ''
+            csvRows.append(row)
+        display.write_csv_file(csvRows, titles, f'User Reports - {tryDate}',
+                               to_drive)
+    elif report == 'customer':
+        while True:
+            try:
+                first_page = gapi.call(rep.customerUsageReports(),
+                                       'get',
+                                       throw_reasons=throw_reasons,
+                                       customerId=customerId,
+                                       date=tryDate,
+                                       fields='warnings,usageReports')
+                warnings = first_page.get('warnings', [])
+                has_reports = bool(first_page.get('usageReports'))
+                fullData, tryDate = _check_full_data_available(
+                    warnings, tryDate, fullDataRequired, has_reports)
+                if fullData < 0:
+                    print('No customer report available.')
+                    sys.exit(1)
+                if fullData == 0:
+                    continue
+                usage = gapi.get_all_pages(rep.customerUsageReports(),
+                                           'get',
+                                           'usageReports',
+                                           throw_reasons=throw_reasons,
+                                           customerId=customerId,
+                                           date=tryDate,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No customer report available.')
+            sys.exit(1)
+        titles = ['name', 'value', 'client_id']
+        csvRows = []
+        auth_apps = list()
+        for item in usage[0]['parameters']:
+            if 'name' not in item:
+                continue
+            name = item['name']
+            if 'intValue' in item:
+                value = item['intValue']
+            elif 'msgValue' in item:
+                if name == 'accounts:authorized_apps':
+                    for subitem in item['msgValue']:
+                        app = {}
+                        for an_item in subitem:
+                            if an_item == 'client_name':
+                                app['name'] = 'App: ' + \
+                                    subitem[an_item].replace('\n', '\\n')
+                            elif an_item == 'num_users':
+                                app['value'] = f'{subitem[an_item]} users'
+                            elif an_item == 'client_id':
+                                app['client_id'] = subitem[an_item]
+                        auth_apps.append(app)
+                    continue
+                values = []
+                for subitem in item['msgValue']:
+                    if 'count' in subitem:
+                        mycount = myvalue = None
+                        for key, value in list(subitem.items()):
+                            if key == 'count':
+                                mycount = value
+                            else:
+                                myvalue = value
+                            if mycount and myvalue:
+                                values.append(f'{myvalue}:{mycount}')
+                        value = ' '.join(values)
+                    elif 'version_number' in subitem \
+                         and 'num_devices' in subitem:
+                        values.append(f'{subitem["version_number"]}:'
+                                      f'{subitem["num_devices"]}')
+                    else:
+                        continue
+                    value = ' '.join(sorted(values, reverse=True))
+            csvRows.append({'name': name, 'value': value})
+        for app in auth_apps:  # put apps at bottom
+            csvRows.append(app)
+        display.write_csv_file(csvRows,
+                               titles,
+                               f'Customer Report - {tryDate}',
+                               todrive=to_drive)
+    else:
+        page_message = gapi.got_total_items_msg('Activities', '...\n')
+        activities = gapi.get_all_pages(rep.activities(),
+                                        'list',
+                                        'items',
+                                        page_message=page_message,
+                                        applicationName=report,
+                                        userKey=userKey,
+                                        customerId=customerId,
+                                        actorIpAddress=actorIpAddress,
+                                        startTime=startTime,
+                                        endTime=endTime,
+                                        eventName=eventName,
+                                        filters=filters,
+                                        orgUnitID=orgUnitId)
+        if activities:
+            titles = ['name']
+            csvRows = []
+            for activity in activities:
+                events = activity['events']
+                del activity['events']
+                activity_row = utils.flatten_json(activity)
+                purge_parameters = True
+                for event in events:
+                    for item in event.get('parameters', []):
+                        if set(item) == set(['value', 'name']):
+                            event[item['name']] = item['value']
+                        elif set(item) == set(['intValue', 'name']):
+                            if item['name'] in ['start_time', 'end_time']:
+                                val = item.get('intValue')
+                                if val is not None:
+                                    val = int(val)
+                                    if val >= 62135683200:
+                                        event[item['name']] = \
+                                            datetime.datetime.fromtimestamp(
+                                                val-62135683200).isoformat()
+                            else:
+                                event[item['name']] = item['intValue']
+                        elif set(item) == set(['boolValue', 'name']):
+                            event[item['name']] = item['boolValue']
+                        elif set(item) == set(['multiValue', 'name']):
+                            event[item['name']] = ' '.join(item['multiValue'])
+                        elif item['name'] == 'scope_data':
+                            parts = {}
+                            for message in item['multiMessageValue']:
+                                for mess in message['parameter']:
+                                    value = mess.get(
+                                        'value',
+                                        ' '.join(mess.get('multiValue', [])))
+                                    parts[mess['name']] = parts.get(
+                                        mess['name'], []) + [value]
+                            for part, v in parts.items():
+                                if part == 'scope_name':
+                                    part = 'scope'
+                                event[part] = ' '.join(v)
+                        else:
+                            purge_parameters = False
+                    if purge_parameters:
+                        event.pop('parameters', None)
+                    row = utils.flatten_json(event)
+                    row.update(activity_row)
+                    for item in row:
+                        if item not in titles:
+                            titles.append(item)
+                    csvRows.append(row)
+            display.sort_csv_titles([
+                'name',
+            ], titles)
+            display.write_csv_file(csvRows, titles,
+                                   f'{report.capitalize()} Activity Report',
+                                   to_drive)
+
+
+def _adjust_date(errMsg):
+    match_date = re.match(
+        'Data for dates later than (.*) is not yet '
+        'available. Please check back later', errMsg)
+    if not match_date:
+        match_date = re.match('Start date can not be later than (.*)', errMsg)
+    if not match_date:
+        controlflow.system_error_exit(4, errMsg)
+    return str(match_date.group(1))
+
+
+def _check_full_data_available(warnings, tryDate, fullDataRequired,
+                               has_reports):
+    one_day = datetime.timedelta(days=1)
+    tryDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
+    # move to day before if we don't have at least one usageReport
+    if not has_reports:
+        tryDateTime -= one_day
+        return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+    for warning in warnings:
+        if warning['code'] == 'PARTIAL_DATA_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   fullDataRequired is not None and \
+                   (fullDataRequired == 'all' or app['value'] in fullDataRequired):
+                    tryDateTime -= one_day
+                    return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+        elif warning['code'] == 'DATA_NOT_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    return (-1, tryDate)
+    return (1, tryDate)

src/gam/gapi/siteverification.py

@@ -0,0 +1,188 @@
+import json
+import sys
+from urllib.parse import urlencode
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import errors as gapi_errors
+from gam.gapi.directory import customer as gapi_directory_customer
+from gam import transport
+from gam import utils
+
+import gam
+
+
+def build():
+    return gam.buildGAPIObject('siteVerification')
+
+
+def create():
+    verif = build()
+    a_domain = sys.argv[3]
+    txt_record = gapi.call(verif.webResource(),
+                           'getToken',
+                           body={
+                               'site': {
+                                   'type': 'INET_DOMAIN',
+                                   'identifier': a_domain
+                               },
+                               'verificationMethod': 'DNS_TXT'
+                           })
+    print(f'TXT Record Name:   {a_domain}')
+    print(f'TXT Record Value:  {txt_record["token"]}')
+    print()
+    cname_record = gapi.call(verif.webResource(),
+                             'getToken',
+                             body={
+                                 'site': {
+                                     'type': 'INET_DOMAIN',
+                                     'identifier': a_domain
+                                 },
+                                 'verificationMethod': 'DNS_CNAME'
+                             })
+    cname_token = cname_record['token']
+    cname_list = cname_token.split(' ')
+    cname_subdomain = cname_list[0]
+    cname_value = cname_list[1]
+    print(f'CNAME Record Name:   {cname_subdomain}.{a_domain}')
+    print(f'CNAME Record Value:  {cname_value}')
+    print('')
+    webserver_file_record = gapi.call(
+        verif.webResource(),
+        'getToken',
+        body={
+            'site': {
+                'type': 'SITE',
+                'identifier': f'http://{a_domain}/'
+            },
+            'verificationMethod': 'FILE'
+        })
+    webserver_file_token = webserver_file_record['token']
+    print(f'Saving web server verification file to: {webserver_file_token}')
+    fileutils.write_file(webserver_file_token,
+                         f'google-site-verification: {webserver_file_token}',
+                         continue_on_error=True)
+    print(f'Verification File URL: http://{a_domain}/{webserver_file_token}')
+    print()
+    webserver_meta_record = gapi.call(
+        verif.webResource(),
+        'getToken',
+        body={
+            'site': {
+                'type': 'SITE',
+                'identifier': f'http://{a_domain}/'
+            },
+            'verificationMethod': 'META'
+        })
+    print(f'Meta URL:               http://{a_domain}/')
+    print(f'Meta HTML Header Data:  {webserver_meta_record["token"]}')
+    print()
+
+
+def info():
+    verif = build()
+    sites = gapi.get_items(verif.webResource(), 'list', 'items')
+    if sites:
+        for site in sites:
+            print(f'Site: {site["site"]["identifier"]}')
+            print(f'Type: {site["site"]["type"]}')
+            print('Owners:')
+            for owner in site['owners']:
+                print(f' {owner}')
+            print()
+    else:
+        print('No Sites Verified.')
+
+
+def update():
+    verif = build()
+    a_domain = sys.argv[3]
+    verificationMethod = sys.argv[4].upper()
+    if verificationMethod == 'CNAME':
+        verificationMethod = 'DNS_CNAME'
+    elif verificationMethod in ['TXT', 'TEXT']:
+        verificationMethod = 'DNS_TXT'
+    if verificationMethod in ['DNS_TXT', 'DNS_CNAME']:
+        verify_type = 'INET_DOMAIN'
+        identifier = a_domain
+    else:
+        verify_type = 'SITE'
+        identifier = f'http://{a_domain}/'
+    body = {
+        'site': {
+            'type': verify_type,
+            'identifier': identifier
+        },
+        'verificationMethod': verificationMethod
+    }
+    try:
+        verify_result = gapi.call(
+            verif.webResource(),
+            'insert',
+            throw_reasons=[gapi_errors.ErrorReason.BAD_REQUEST],
+            verificationMethod=verificationMethod,
+            body=body)
+    except gapi_errors.GapiBadRequestError as e:
+        print(f'ERROR: {str(e)}')
+        verify_data = gapi.call(verif.webResource(), 'getToken', body=body)
+        print(f'Method:  {verify_data["method"]}')
+        print(f'Expected Token:      {verify_data["token"]}')
+        if verify_data['method'] in ['DNS_CNAME', 'DNS_TXT']:
+            simplehttp = transport.create_http()
+            base_url = 'https://dns.google/resolve?'
+            query_params = {}
+            if verify_data['method'] == 'DNS_CNAME':
+                cname_token = verify_data['token']
+                cname_list = cname_token.split(' ')
+                cname_subdomain = cname_list[0]
+                query_params['name'] = f'{cname_subdomain}.{a_domain}'
+                query_params['type'] = 'cname'
+            else:
+                query_params['name'] = a_domain
+                query_params['type'] = 'txt'
+            full_url = base_url + urlencode(query_params)
+            (_, c) = simplehttp.request(full_url, 'GET')
+            result = json.loads(c)
+            status = result['Status']
+            if status == 0 and 'Answer' in result:
+                answers = result['Answer']
+                if verify_data['method'] == 'DNS_CNAME':
+                    answer = answers[0]['data']
+                else:
+                    answer = 'no matching record found'
+                    for possible_answer in answers:
+                        possible_answer['data'] = possible_answer['data'].strip(
+                            '"')
+                        if possible_answer['data'].startswith(
+                                'google-site-verification'):
+                            answer = possible_answer['data']
+                            break
+                        print(
+                            f'Unrelated TXT record: {possible_answer["data"]}')
+                print(f'Found DNS Record: {answer}')
+            elif status == 0:
+                controlflow.system_error_exit(1, 'DNS record not found')
+            else:
+                controlflow.system_error_exit(
+                    status,
+                    DNS_ERROR_CODES_MAP.get(status, f'Unknown error {status}'))
+        return
+    print('SUCCESS!')
+    print(f'Verified:  {verify_result["site"]["identifier"]}')
+    print(f'ID:  {verify_result["id"]}')
+    print(f'Type: {verify_result["site"]["type"]}')
+    print('All Owners:')
+    try:
+        for owner in verify_result['owners']:
+            print(f' {owner}')
+    except KeyError:
+        pass
+    print()
+    print(
+        f'You can now add {a_domain} or it\'s subdomains as secondary or domain aliases of the {GC_Values[GC_DOMAIN]} G Suite Account.'
+    )

src/gam/gapi/storage.py

@@ -0,0 +1,81 @@
+import base64
+import os
+import re
+import sys
+
+import googleapiclient
+
+import gam
+from gam.var import *
+from gam import fileutils
+from gam import gapi
+from gam import utils
+
+
+def build_gapi():
+    return gam.buildGAPIObject('storage')
+
+
+def get_cloud_storage_object(s,
+                             bucket,
+                             object_,
+                             local_file=None,
+                             expectedMd5=None):
+    if not local_file:
+        local_file = object_
+    if os.path.exists(local_file):
+        sys.stdout.write(' File already exists. ')
+        sys.stdout.flush()
+        if expectedMd5:
+            sys.stdout.write(f'Verifying {expectedMd5} hash...')
+            sys.stdout.flush()
+            if utils.md5_matches_file(local_file, expectedMd5, False):
+                print('VERIFIED')
+                return
+            print('not verified. Downloading again and over-writing...')
+        else:
+            return  # nothing to verify, just assume we're good.
+    print(f'saving to {local_file}')
+    request = s.objects().get_media(bucket=bucket, object=object_)
+    file_path = os.path.dirname(local_file)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    f = fileutils.open_file(local_file, 'wb')
+    downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+    done = False
+    while not done:
+        status, done = downloader.next_chunk()
+        sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
+        sys.stdout.flush()
+    sys.stdout.write('\n Download complete. Flushing to disk...\n')
+    fileutils.close_file(f, True)
+    if expectedMd5:
+        f = fileutils.open_file(local_file, 'rb')
+        sys.stdout.write(f' Verifying file hash is {expectedMd5}...')
+        sys.stdout.flush()
+        utils.md5_matches_file(local_file, expectedMd5, True)
+        print('VERIFIED')
+        fileutils.close_file(f)
+
+
+def download_bucket():
+    bucket = sys.argv[3]
+    s = build_gapi()
+    page_message = gapi.got_total_items_msg('Files', '...')
+    fields = 'nextPageToken,items(name,id,md5Hash)'
+    objects = gapi.get_all_pages(s.objects(),
+                                 'list',
+                                 'items',
+                                 page_message=page_message,
+                                 bucket=bucket,
+                                 projection='noAcl',
+                                 fields=fields)
+    i = 1
+    for object_ in objects:
+        print(f'{i}/{len(objects)}')
+        expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
+        get_cloud_storage_object(s,
+                                 bucket,
+                                 object_['name'],
+                                 expectedMd5=expectedMd5)
+        i += 1

src/gam/gapi/vault.py

@@ -0,0 +1,844 @@
+import datetime
+import json
+import sys
+
+import googleapiclient.http
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import storage as gapi_storage
+from gam import utils
+
+
+def buildGAPIObject():
+    return gam.buildGAPIObject('vault')
+
+
+def validateCollaborators(collaboratorList, cd):
+    collaborators = []
+    for collaborator in collaboratorList.split(','):
+        collaborator_id = gam.convertEmailAddressToUID(collaborator, cd)
+        if not collaborator_id:
+            controlflow.system_error_exit(
+                4, f'failed to get a UID for '
+                f'{collaborator}. Please make '
+                f'sure this is a real user.')
+        collaborators.append({'email': collaborator, 'id': collaborator_id})
+    return collaborators
+
+
+def createMatter():
+    v = buildGAPIObject()
+    matter_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    body = {'name': f'New Matter - {matter_time}'}
+    collaborators = []
+    cd = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['collaborator', 'collaborators']:
+            if not cd:
+                cd = gam.buildGAPIObject('directory')
+            collaborators.extend(validateCollaborators(sys.argv[i + 1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create matter')
+    matterId = gapi.call(v.matters(), 'create', body=body,
+                         fields='matterId')['matterId']
+    print(f'Created matter {matterId}')
+    for collaborator in collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {
+            'matterPermission': {
+                'role': 'COLLABORATOR',
+                'accountId': collaborator['id']
+            }
+        }
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+
+
+VAULT_SEARCH_METHODS_MAP = {
+    'account': 'ACCOUNT',
+    'accounts': 'ACCOUNT',
+    'entireorg': 'ENTIRE_ORG',
+    'everyone': 'ENTIRE_ORG',
+    'orgunit': 'ORG_UNIT',
+    'ou': 'ORG_UNIT',
+    'room': 'ROOM',
+    'rooms': 'ROOM',
+    'shareddrive': 'SHARED_DRIVE',
+    'shareddrives': 'SHARED_DRIVE',
+    'teamdrive': 'SHARED_DRIVE',
+    'teamdrives': 'SHARED_DRIVE',
+}
+VAULT_SEARCH_METHODS_LIST = [
+    'accounts', 'orgunit', 'shareddrives', 'rooms', 'everyone'
+]
+
+
+def createExport():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['corpus']['enum'])
+    allowed_scopes = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['dataScope']['enum'])
+    allowed_formats = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['MailExportOptions']['properties']
+        ['exportFormat']['enum'])
+    export_format = 'MBOX'
+    showConfidentialModeContent = None  # default to not even set
+    matterId = None
+    body = {'query': {'dataScope': 'ALL_DATA'}, 'exportOptions': {}}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            body['matterId'] = matterId
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'corpus':
+            body['query']['corpus'] = sys.argv[i + 1].upper()
+            if body['query']['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit('corpus',
+                                                   ', '.join(allowed_corpuses),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg in VAULT_SEARCH_METHODS_MAP:
+            if body['query'].get('searchMethod'):
+                message = f'Multiple search methods ' \
+                          f'({", ".join(VAULT_SEARCH_METHODS_LIST)})' \
+                          f'specified, only one is allowed'
+                controlflow.system_error_exit(3, message)
+            searchMethod = VAULT_SEARCH_METHODS_MAP[myarg]
+            body['query']['searchMethod'] = searchMethod
+            if searchMethod == 'ACCOUNT':
+                body['query']['accountInfo'] = {
+                    'emails': sys.argv[i + 1].split(',')
+                }
+                i += 2
+            elif searchMethod == 'ORG_UNIT':
+                body['query']['orgUnitInfo'] = {
+                    'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+                }
+                i += 2
+            elif searchMethod == 'SHARED_DRIVE':
+                body['query']['sharedDriveInfo'] = {
+                    'sharedDriveIds': sys.argv[i + 1].split(',')
+                }
+                i += 2
+            elif searchMethod == 'ROOM':
+                body['query']['hangoutsChatInfo'] = {
+                    'roomId': sys.argv[i + 1].split(',')
+                }
+                i += 2
+            else:
+                i += 1
+        elif myarg == 'scope':
+            body['query']['dataScope'] = sys.argv[i + 1].upper()
+            if body['query']['dataScope'] not in allowed_scopes:
+                controlflow.expected_argument_exit('scope',
+                                                   ', '.join(allowed_scopes),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg in ['terms']:
+            body['query']['terms'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            body['query']['startTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i + 1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            body['query']['endTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i + 1])
+            i += 2
+        elif myarg in ['timezone']:
+            body['query']['timeZone'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['excludedrafts']:
+            body['query']['mailOptions'] = {
+                'excludeDrafts': gam.getBoolean(sys.argv[i + 1], myarg)
+            }
+            i += 2
+        elif myarg in ['driveversiondate']:
+            body['query'].setdefault('driveOptions', {})['versionDate'] = \
+                utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['includeshareddrives', 'includeteamdrives']:
+            body['query'].setdefault(
+                'driveOptions', {})['includeSharedDrives'] = gam.getBoolean(
+                    sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg in ['includerooms']:
+            body['query']['hangoutsChatOptions'] = {
+                'includeRooms': gam.getBoolean(sys.argv[i + 1], myarg)
+            }
+            i += 2
+        elif myarg in ['format']:
+            export_format = sys.argv[i + 1].upper()
+            if export_format not in allowed_formats:
+                controlflow.expected_argument_exit('export format',
+                                                   ', '.join(allowed_formats),
+                                                   export_format)
+            i += 2
+        elif myarg in ['showconfidentialmodecontent']:
+            showConfidentialModeContent = gam.getBoolean(sys.argv[i + 1], myarg)
+            i += 2
+        elif myarg in ['region']:
+            allowed_regions = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['ExportOptions']['properties']['region']
+                ['enum'])
+            body['exportOptions']['region'] = sys.argv[i + 1].upper()
+            if body['exportOptions']['region'] not in allowed_regions:
+                controlflow.expected_argument_exit(
+                    'region', ', '.join(allowed_regions),
+                    body['exportOptions']['region'])
+            i += 2
+        elif myarg in ['includeaccessinfo']:
+            body['exportOptions'].setdefault(
+                'driveOptions', {})['includeAccessInfo'] = gam.getBoolean(
+                    sys.argv[i + 1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create export')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new export.')
+    if 'corpus' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a corpus for the ' \
+          f'new export. Choose one of {", ".join(allowed_corpuses)}')
+    if 'searchMethod' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a search method ' \
+          'for the new export. Choose one of ' \
+          f'{", ".join(VAULT_SEARCH_METHODS_LIST)}')
+    if 'name' not in body:
+        corpus_name = body['query']['corpus']
+        corpus_date = datetime.datetime.now()
+        body['name'] = f'GAM {corpus_name} export - {corpus_date}'
+    options_field = None
+    if body['query']['corpus'] == 'MAIL':
+        options_field = 'mailOptions'
+    elif body['query']['corpus'] == 'GROUPS':
+        options_field = 'groupsOptions'
+    elif body['query']['corpus'] == 'HANGOUTS_CHAT':
+        options_field = 'hangoutsChatOptions'
+    if options_field:
+        body['exportOptions'].pop('driveOptions', None)
+        body['exportOptions'][options_field] = {'exportFormat': export_format}
+        if showConfidentialModeContent is not None:
+            body['exportOptions'][options_field][
+                'showConfidentialModeContent'] = showConfidentialModeContent
+    results = gapi.call(v.matters().exports(),
+                        'create',
+                        matterId=matterId,
+                        body=body)
+    print(f'Created export {results["id"]}')
+    display.print_json(results)
+
+
+def deleteExport():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    print(f'Deleting export {sys.argv[4]} / {exportId}')
+    gapi.call(v.matters().exports(),
+              'delete',
+              matterId=matterId,
+              exportId=exportId)
+
+
+def getExportInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
+    display.print_json(export)
+
+
+def createHold():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Hold']['properties']['corpus']['enum'])
+    body = {'query': {}}
+    i = 3
+    query = None
+    start_time = None
+    end_time = None
+    matterId = None
+    accounts = []
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'corpus':
+            body['corpus'] = sys.argv[i + 1].upper()
+            if body['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit('corpus',
+                                                   ', '.join(allowed_corpuses),
+                                                   sys.argv[i + 1])
+            i += 2
+        elif myarg in ['accounts', 'users', 'groups']:
+            accounts = sys.argv[i + 1].split(',')
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+            }
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new hold.')
+    if not body.get('name'):
+        controlflow.system_error_exit(
+            3, 'you must specify a name for the new hold.')
+    if not body.get('corpus'):
+        controlflow.system_error_exit(3, f'you must specify a corpus for ' \
+          f'the new hold. Choose one of {", ".join(allowed_corpuses)}')
+    if body['corpus'] == 'HANGOUTS_CHAT':
+        query_type = 'hangoutsChatQuery'
+    else:
+        query_type = f'{body["corpus"].lower()}Query'
+    body['query'][query_type] = {}
+    if body['corpus'] == 'DRIVE':
+        if query:
+            try:
+                body['query'][query_type] = json.loads(query)
+            except ValueError as e:
+                controlflow.system_error_exit(3, f'{str(e)}, query: {query}')
+    elif body['corpus'] in ['GROUPS', 'MAIL']:
+        if query:
+            body['query'][query_type] = {'terms': query}
+        if start_time:
+            body['query'][query_type]['startTime'] = start_time
+        if end_time:
+            body['query'][query_type]['endTime'] = end_time
+    if accounts:
+        body['accounts'] = []
+        cd = gam.buildGAPIObject('directory')
+        account_type = 'group' if body['corpus'] == 'GROUPS' else 'user'
+        for account in accounts:
+            body['accounts'].append({
+                'accountId':
+                    gam.convertEmailAddressToUID(account, cd, account_type)
+            })
+    holdId = gapi.call(v.matters().holds(),
+                       'create',
+                       matterId=matterId,
+                       body=body,
+                       fields='holdId')
+    print(f'Created hold {holdId["holdId"]}')
+
+
+def deleteHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam delete hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    print(f'Deleting hold {hold} / {holdId}')
+    gapi.call(v.matters().holds(), 'delete', matterId=matterId, holdId=holdId)
+
+
+def getHoldInfo():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam info hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    results = gapi.call(v.matters().holds(),
+                        'get',
+                        matterId=matterId,
+                        holdId=holdId)
+    cd = gam.buildGAPIObject('directory')
+    if 'accounts' in results:
+        account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
+        for i in range(0, len(results['accounts'])):
+            uid = f'uid:{results["accounts"][i]["accountId"]}'
+            acct_email = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
+            results['accounts'][i]['email'] = acct_email
+    if 'orgUnit' in results:
+        results['orgUnit']['orgUnitPath'] = gam.doGetOrgInfo(
+            results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
+    display.print_json(results)
+
+
+def convertExportNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'exports(id,name),nextPageToken'
+    exports = gapi.get_all_pages(v.matters().exports(),
+                                 'list',
+                                 'exports',
+                                 matterId=matterId,
+                                 fields=fields)
+    for export in exports:
+        if export['name'].lower() == nameOrID:
+            return export['id']
+    controlflow.system_error_exit(
+        4, f'could not find export name {nameOrID} '
+        f'in matter {matterId}')
+
+
+def convertHoldNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'holds(holdId,name),nextPageToken'
+    holds = gapi.get_all_pages(v.matters().holds(),
+                               'list',
+                               'holds',
+                               matterId=matterId,
+                               fields=fields)
+    for hold in holds:
+        if hold['name'].lower() == nameOrID:
+            return hold['holdId']
+    controlflow.system_error_exit(
+        4, f'could not find hold name {nameOrID} '
+        f'in matter {matterId}')
+
+
+def convertMatterNameToID(v, nameOrID):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'matters(matterId,name),nextPageToken'
+    matters = gapi.get_all_pages(v.matters(),
+                                 'list',
+                                 'matters',
+                                 view='BASIC',
+                                 fields=fields)
+    for matter in matters:
+        if matter['name'].lower() == nameOrID:
+            return matter['matterId']
+    return None
+
+
+def getMatterItem(v, nameOrID):
+    matterId = convertMatterNameToID(v, nameOrID)
+    if not matterId:
+        controlflow.system_error_exit(4, f'could not find matter {nameOrID}')
+    return matterId
+
+
+def updateHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    body = {}
+    query = None
+    add_accounts = []
+    del_accounts = []
+    start_time = None
+    end_time = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i + 1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+            }
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
+            i += 2
+        elif myarg in ['addusers', 'addaccounts', 'addgroups']:
+            add_accounts = sys.argv[i + 1].split(',')
+            i += 2
+        elif myarg in ['removeusers', 'removeaccounts', 'removegroups']:
+            del_accounts = sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam update hold')
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    if query or start_time or end_time or body.get('orgUnit'):
+        fields = 'corpus,query,orgUnit'
+        old_body = gapi.call(v.matters().holds(),
+                             'get',
+                             matterId=matterId,
+                             holdId=holdId,
+                             fields=fields)
+        body['query'] = old_body['query']
+        body['corpus'] = old_body['corpus']
+        if 'orgUnit' in old_body and 'orgUnit' not in body:
+            # bah, API requires this to be sent
+            # on update even when it's not changing
+            body['orgUnit'] = old_body['orgUnit']
+        query_type = f'{body["corpus"].lower()}Query'
+        if body['corpus'] == 'DRIVE':
+            if query:
+                try:
+                    body['query'][query_type] = json.loads(query)
+                except ValueError as e:
+                    message = f'{str(e)}, query: {query}'
+                    controlflow.system_error_exit(3, message)
+        elif body['corpus'] in ['GROUPS', 'MAIL']:
+            if query:
+                body['query'][query_type]['terms'] = query
+            if start_time:
+                body['query'][query_type]['startTime'] = start_time
+            if end_time:
+                body['query'][query_type]['endTime'] = end_time
+    if body:
+        print(f'Updating hold {hold} / {holdId}')
+        gapi.call(v.matters().holds(),
+                  'update',
+                  matterId=matterId,
+                  holdId=holdId,
+                  body=body)
+    if add_accounts or del_accounts:
+        cd = gam.buildGAPIObject('directory')
+        for account in add_accounts:
+            print(f'adding {account} to hold.')
+            add_body = {'accountId': gam.convertEmailAddressToUID(account, cd)}
+            gapi.call(v.matters().holds().accounts(),
+                      'create',
+                      matterId=matterId,
+                      holdId=holdId,
+                      body=add_body)
+        for account in del_accounts:
+            print(f'removing {account} from hold.')
+            accountId = gam.convertEmailAddressToUID(account, cd)
+            gapi.call(v.matters().holds().accounts(),
+                      'delete',
+                      matterId=matterId,
+                      holdId=holdId,
+                      accountId=accountId)
+
+
+def updateMatter(action=None):
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    body = {}
+    action_kwargs = {'body': {}}
+    add_collaborators = []
+    remove_collaborators = []
+    cd = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            action = sys.argv[i + 1].lower()
+            if action not in VAULT_MATTER_ACTIONS:
+                controlflow.system_error_exit(3, f'allowed actions are ' \
+                    f'{", ".join(VAULT_MATTER_ACTIONS)}, got {action}')
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i + 1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i + 1]
+            i += 2
+        elif myarg in ['addcollaborator', 'addcollaborators']:
+            if not cd:
+                cd = gam.buildGAPIObject('directory')
+            add_collaborators.extend(validateCollaborators(sys.argv[i + 1], cd))
+            i += 2
+        elif myarg in ['removecollaborator', 'removecollaborators']:
+            if not cd:
+                cd = gam.buildGAPIObject('directory')
+            remove_collaborators.extend(
+                validateCollaborators(sys.argv[i + 1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update matter')
+    if action == 'delete':
+        action_kwargs = {}
+    if body:
+        print(f'Updating matter {sys.argv[3]}...')
+        if 'name' not in body or 'description' not in body:
+            # bah, API requires name/description to be sent
+            # on update even when it's not changing
+            result = gapi.call(v.matters(),
+                               'get',
+                               matterId=matterId,
+                               view='BASIC')
+            body.setdefault('name', result['name'])
+            body.setdefault('description', result.get('description'))
+        gapi.call(v.matters(), 'update', body=body, matterId=matterId)
+    if action:
+        print(f'Performing {action} on matter {sys.argv[3]}')
+        gapi.call(v.matters(), action, matterId=matterId, **action_kwargs)
+    for collaborator in add_collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {
+            'matterPermission': {
+                'role': 'COLLABORATOR',
+                'accountId': collaborator['id']
+            }
+        }
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+    for collaborator in remove_collaborators:
+        print(f' removing collaborator {collaborator["email"]}')
+        gapi.call(v.matters(),
+                  'removePermissions',
+                  matterId=matterId,
+                  body={'accountId': collaborator['id']})
+
+
+def getMatterInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    result = gapi.call(v.matters(), 'get', matterId=matterId, view='FULL')
+    if 'matterPermissions' in result:
+        cd = gam.buildGAPIObject('directory')
+        for i in range(0, len(result['matterPermissions'])):
+            uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
+            user_email = gam.convertUIDtoEmailAddress(uid, cd)
+            result['matterPermissions'][i]['email'] = user_email
+    display.print_json(result)
+
+
+def downloadExport():
+    verifyFiles = True
+    extractFiles = True
+    v = buildGAPIObject()
+    s = gapi_storage.build_gapi()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i + 1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        elif myarg == 'noverify':
+            verifyFiles = False
+            i += 1
+        elif myarg == 'noextract':
+            extractFiles = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam download export')
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
+    for s_file in export['cloudStorageSink']['files']:
+        bucket = s_file['bucketName']
+        s_object = s_file['objectName']
+        filename = os.path.join(targetFolder, s_object.replace('/', '-'))
+        print(f'saving to {filename}')
+        request = s.objects().get_media(bucket=bucket, object=s_object)
+        f = fileutils.open_file(filename, 'wb')
+        downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+        done = False
+        while not done:
+            status, done = downloader.next_chunk()
+            sys.stdout.write(' Downloaded: {0:>7.2%}\r'.format(
+                status.progress()))
+            sys.stdout.flush()
+        sys.stdout.write('\n Download complete. Flushing to disk...\n')
+        fileutils.close_file(f, True)
+        if verifyFiles:
+            expected_hash = s_file['md5Hash']
+            sys.stdout.write(f' Verifying file hash is {expected_hash}...')
+            sys.stdout.flush()
+            utils.md5_matches_file(filename, expected_hash, True)
+            print('VERIFIED')
+        if extractFiles and re.search(r'\.zip$', filename):
+            gam.extract_nested_zip(filename, targetFolder)
+
+
+def printMatters():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'name', 'description', 'state']
+    titles = initialTitles[:]
+    view = 'FULL'
+    state = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            view = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg == 'matterstate':
+            valid_states = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['Matter']['properties']['state']['enum'])
+            state = sys.argv[i + 1].upper()
+            if state not in valid_states:
+                controlflow.expected_argument_exit('state',
+                                                   ', '.join(valid_states),
+                                                   state)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print matters')
+    gam.printGettingAllItems('Vault Matters', None)
+    page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
+    matters = gapi.get_all_pages(v.matters(),
+                                 'list',
+                                 'matters',
+                                 page_message=page_message,
+                                 view=view,
+                                 state=state)
+    for matter in matters:
+        display.add_row_titles_to_csv_file(utils.flatten_json(matter), csvRows,
+                                           titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Matters', todrive)
+
+
+def printExports():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'id', 'name', 'createTime', 'status']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print exports')
+    if not matters:
+        fields = 'matters(matterId),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(),
+                                             'list',
+                                             'matters',
+                                             view='BASIC',
+                                             state='OPEN',
+                                             fields=fields)
+        for matter in matters_results:
+            matterIds.append(matter['matterId'])
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving exports for matter {matterId}\n')
+        exports = gapi.get_all_pages(v.matters().exports(),
+                                     'list',
+                                     'exports',
+                                     matterId=matterId)
+        for export in exports:
+            display.add_row_titles_to_csv_file(
+                utils.flatten_json(export, flattened={'matterId': matterId}),
+                csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Exports', todrive)
+
+
+def printHolds():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'holdId', 'name', 'corpus', 'updateTime']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i + 1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print holds')
+    if not matters:
+        fields = 'matters(matterId),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(),
+                                             'list',
+                                             'matters',
+                                             view='BASIC',
+                                             state='OPEN',
+                                             fields=fields)
+        for matter in matters_results:
+            matterIds.append(matter['matterId'])
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving holds for matter {matterId}\n')
+        holds = gapi.get_all_pages(v.matters().holds(),
+                                   'list',
+                                   'holds',
+                                   matterId=matterId)
+        for hold in holds:
+            display.add_row_titles_to_csv_file(
+                utils.flatten_json(hold, flattened={'matterId': matterId}),
+                csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Holds', todrive)

src/gam/transport.py

@@ -0,0 +1,102 @@
+"""Methods related to network transport."""
+
+import google_auth_httplib2
+import httplib2
+
+from gam.var import GAM_INFO
+from gam.var import GC_CA_FILE
+from gam.var import GC_TLS_MAX_VERSION
+from gam.var import GC_TLS_MIN_VERSION
+from gam.var import GC_Values
+
+
+def create_http(cache=None,
+                timeout=None,
+                override_min_tls=None,
+                override_max_tls=None):
+    """Creates a uniform HTTP transport object.
+
+  Args:
+    cache: The HTTP cache to use.
+    timeout: The cache timeout, in seconds.
+    override_min_tls: The minimum TLS version to require. If not provided, the
+      default is used.
+    override_max_tls: The maximum TLS version to require. If not provided, the
+      default is used.
+
+  Returns:
+    httplib2.Http with the specified options.
+  """
+    tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
+        GC_TLS_MIN_VERSION)
+    tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
+        GC_TLS_MAX_VERSION)
+    httpObj = httplib2.Http(ca_certs=GC_Values.get(GC_CA_FILE),
+                            tls_maximum_version=tls_maximum_version,
+                            tls_minimum_version=tls_minimum_version,
+                            cache=cache,
+                            timeout=timeout)
+    httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
+    return httpObj
+
+
+def create_request(http=None):
+    """Creates a uniform Request object with a default http, if not provided.
+
+  Args:
+    http: Optional httplib2.Http compatible object to be used with the request.
+      If not provided, a default HTTP will be used.
+
+  Returns:
+    Request: A google_auth_httplib2.Request compatible Request.
+  """
+    if not http:
+        http = create_http()
+    return Request(http)
+
+
+GAM_USER_AGENT = GAM_INFO
+
+
+def _force_user_agent(user_agent):
+    """Creates a decorator which can force a user agent in HTTP headers."""
+
+    def decorator(request_method):
+        """Wraps a request method to insert a user-agent in HTTP headers."""
+
+        def wrapped_request_method(*args, **kwargs):
+            """Modifies HTTP headers to include a specified user-agent."""
+            if kwargs.get('headers') is not None:
+                if kwargs['headers'].get('user-agent'):
+                    if user_agent not in kwargs['headers']['user-agent']:
+                        # Save the existing user-agent header and tack on our own.
+                        kwargs['headers']['user-agent'] = (
+                            f'{user_agent} '
+                            f'{kwargs["headers"]["user-agent"]}')
+                else:
+                    kwargs['headers']['user-agent'] = user_agent
+            else:
+                kwargs['headers'] = {'user-agent': user_agent}
+            return request_method(*args, **kwargs)
+
+        return wrapped_request_method
+
+    return decorator
+
+
+class Request(google_auth_httplib2.Request):
+    """A Request which forces a user agent."""
+
+    @_force_user_agent(GAM_USER_AGENT)
+    def __call__(self, *args, **kwargs):
+        """Inserts the GAM user-agent header in requests."""
+        return super(Request, self).__call__(*args, **kwargs)
+
+
+class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
+    """An AuthorizedHttp which forces a user agent during requests."""
+
+    @_force_user_agent(GAM_USER_AGENT)
+    def request(self, *args, **kwargs):
+        """Inserts the GAM user-agent header in requests."""
+        return super(AuthorizedHttp, self).request(*args, **kwargs)

src/gam/transport_test.py

@@ -0,0 +1,185 @@
+"""Tests for transport."""
+
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import google_auth_httplib2
+import httplib2
+
+from gam import transport
+
+
+class CreateHttpTest(unittest.TestCase):
+
+    def setUp(self):
+        SetGlobalVariables()
+        super(CreateHttpTest, self).setUp()
+
+    def test_create_http_sets_default_values_on_http(self):
+        http = transport.create_http()
+        self.assertIsNone(http.cache)
+        self.assertIsNone(http.timeout)
+        self.assertEqual(http.tls_minimum_version,
+                         transport.GC_Values[transport.GC_TLS_MIN_VERSION])
+        self.assertEqual(http.tls_maximum_version,
+                         transport.GC_Values[transport.GC_TLS_MAX_VERSION])
+        self.assertEqual(http.ca_certs,
+                         transport.GC_Values[transport.GC_CA_FILE])
+
+    def test_create_http_sets_tls_min_version(self):
+        http = transport.create_http(override_min_tls='TLSv1_1')
+        self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
+
+    def test_create_http_sets_tls_max_version(self):
+        http = transport.create_http(override_max_tls='TLSv1_3')
+        self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
+
+    def test_create_http_sets_cache(self):
+        fake_cache = {}
+        http = transport.create_http(cache=fake_cache)
+        self.assertEqual(http.cache, fake_cache)
+
+    def test_create_http_sets_cache_timeout(self):
+        http = transport.create_http(timeout=1234)
+        self.assertEqual(http.timeout, 1234)
+
+
+class TransportTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mock_http = MagicMock(spec=httplib2.Http)
+        self.mock_response = MagicMock(spec=httplib2.Response)
+        self.mock_content = MagicMock()
+        self.mock_http.request.return_value = (self.mock_response,
+                                               self.mock_content)
+        self.mock_credentials = MagicMock()
+        self.test_uri = 'http://example.com'
+        super(TransportTest, self).setUp()
+
+    @patch.object(transport, 'create_http')
+    def test_create_request_uses_default_http(self, mock_create_http):
+        request = transport.create_request()
+        self.assertEqual(request.http, mock_create_http.return_value)
+
+    def test_create_request_uses_provided_http(self):
+        request = transport.create_request(http=self.mock_http)
+        self.assertEqual(request.http, self.mock_http)
+
+    def test_create_request_returns_request_with_forced_user_agent(self):
+        request = transport.create_request()
+        self.assertIsInstance(request, transport.Request)
+
+    def test_request_is_google_auth_httplib2_compatible(self):
+        request = transport.create_request()
+        self.assertIsInstance(request, google_auth_httplib2.Request)
+
+    def test_request_call_returns_response_content(self):
+        request = transport.Request(self.mock_http)
+        response = request(self.test_uri)
+        self.assertEqual(self.mock_response.status, response.status)
+        self.assertEqual(self.mock_content, response.data)
+
+    def test_request_call_forces_user_agent_no_provided_headers(self):
+        request = transport.Request(self.mock_http)
+
+        request(self.test_uri)
+        headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', headers)
+        self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+    def test_request_call_forces_user_agent_no_agent_in_headers(self):
+        request = transport.Request(self.mock_http)
+        fake_request_headers = {
+            'some-header-thats-not-a-user-agent': 'someData'
+        }
+
+        request(self.test_uri, headers=fake_request_headers)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+        self.assertEqual('someData',
+                         final_headers['some-header-thats-not-a-user-agent'])
+
+    def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
+        request = transport.Request(self.mock_http)
+        headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+        request(self.test_uri, headers=headers_with_user_agent)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn('existing-user-agent', final_headers['user-agent'])
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+    def test_request_call_same_user_agent_already_in_headers(self):
+        request = transport.Request(self.mock_http)
+        same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+        request(self.test_uri, headers=same_user_agent_header)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        # Make sure the header wasn't duplicated
+        self.assertEqual(len(transport.GAM_USER_AGENT),
+                         len(final_headers['user-agent']))
+
+    def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
+        http = transport.AuthorizedHttp(self.mock_credentials)
+        self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
+
+    def test_authorizedhttp_request_returns_response_content(self):
+        http = transport.AuthorizedHttp(self.mock_credentials,
+                                        http=self.mock_http)
+        response, content = http.request(self.test_uri)
+        self.assertEqual(self.mock_response, response)
+        self.assertEqual(self.mock_content, content)
+
+    def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        authorized_http.request(self.test_uri)
+        headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', headers)
+        self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+    def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        fake_request_headers = {
+            'some-header-thats-not-a-user-agent': 'someData'
+        }
+
+        authorized_http.request(self.test_uri, headers=fake_request_headers)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+        self.assertEqual('someData',
+                         final_headers['some-header-thats-not-a-user-agent'])
+
+    def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
+        self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+        authorized_http.request(self.test_uri, headers=headers_with_user_agent)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn('existing-user-agent', final_headers['user-agent'])
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+    def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+        authorized_http.request(self.test_uri, headers=same_user_agent_header)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        # Make sure the header wasn't duplicated
+        self.assertEqual(len(transport.GAM_USER_AGENT),
+                         len(final_headers['user-agent']))

src/gam/utils.py

@@ -0,0 +1,343 @@
+import datetime
+import re
+import sys
+import time
+from hashlib import md5
+from html.entities import name2codepoint
+from html.parser import HTMLParser
+import json
+import dateutil.parser
+
+from gam import controlflow
+from gam import fileutils
+from gam import transport
+from gam.var import *
+
+
+class _DeHTMLParser(HTMLParser):
+
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.__text = []
+
+    def handle_data(self, data):
+        self.__text.append(data)
+
+    def handle_charref(self, name):
+        self.__text.append(
+            chr(int(name[1:], 16)) if name.startswith('x') else chr(int(name)))
+
+    def handle_entityref(self, name):
+        cp = name2codepoint.get(name)
+        if cp:
+            self.__text.append(chr(cp))
+        else:
+            self.__text.append('&' + name)
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'p':
+            self.__text.append('\n\n')
+        elif tag == 'br':
+            self.__text.append('\n')
+        elif tag == 'a':
+            for attr in attrs:
+                if attr[0] == 'href':
+                    self.__text.append(f'({attr[1]}) ')
+                    break
+        elif tag == 'div':
+            if not attrs:
+                self.__text.append('\n')
+        elif tag in {'http:', 'https'}:
+            self.__text.append(f' ({tag}//{attrs[0][0]}) ')
+
+    def handle_startendtag(self, tag, attrs):
+        if tag == 'br':
+            self.__text.append('\n\n')
+
+    def text(self):
+        return re.sub(r'\n{2}\n+', '\n\n',
+                      re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
+
+
+def dehtml(text):
+    try:
+        parser = _DeHTMLParser()
+        parser.feed(str(text))
+        parser.close()
+        return parser.text()
+    except:
+        from traceback import print_exc
+        print_exc(file=sys.stderr)
+        return text
+
+
+def indentMultiLineText(message, n=0):
+    return message.replace('\n', '\n{0}'.format(' ' * n)).rstrip()
+
+
+def flatten_json(structure, key='', path='', flattened=None, listLimit=None):
+    if flattened is None:
+        flattened = {}
+    if not isinstance(structure, (dict, list)):
+        flattened[((path + '.') if path else '') + key] = structure
+    elif isinstance(structure, list):
+        for i, item in enumerate(structure):
+            if listLimit and (i >= listLimit):
+                break
+            flatten_json(item,
+                         f'{i}',
+                         '.'.join([item for item in [path, key] if item]),
+                         flattened=flattened,
+                         listLimit=listLimit)
+    else:
+        for new_key, value in list(structure.items()):
+            if new_key in ['kind', 'etag', '@type']:
+                continue
+            if value == NEVER_TIME:
+                value = 'Never'
+            flatten_json(value,
+                         new_key,
+                         '.'.join([item for item in [path, key] if item]),
+                         flattened=flattened,
+                         listLimit=listLimit)
+    return flattened
+
+
+def formatTimestampYMD(timestamp):
+    return datetime.datetime.fromtimestamp(int(timestamp) /
+                                           1000).strftime('%Y-%m-%d')
+
+
+def formatTimestampYMDHMS(timestamp):
+    return datetime.datetime.fromtimestamp(int(timestamp) /
+                                           1000).strftime('%Y-%m-%d %H:%M:%S')
+
+
+def formatTimestampYMDHMSF(timestamp):
+    return str(datetime.datetime.fromtimestamp(int(timestamp) / 1000))
+
+
+def formatFileSize(fileSize):
+    if fileSize == 0:
+        return '0kb'
+    if fileSize < ONE_KILO_BYTES:
+        return '1kb'
+    if fileSize < ONE_MEGA_BYTES:
+        return f'{fileSize // ONE_KILO_BYTES}kb'
+    if fileSize < ONE_GIGA_BYTES:
+        return f'{fileSize // ONE_MEGA_BYTES}mb'
+    return f'{fileSize // ONE_GIGA_BYTES}gb'
+
+
+def formatMilliSeconds(millis):
+    seconds, millis = divmod(millis, 1000)
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
+
+
+def integerLimits(minVal, maxVal, item='integer'):
+    if (minVal is not None) and (maxVal is not None):
+        return f'{item} {minVal}<=x<={maxVal}'
+    if minVal is not None:
+        return f'{item} x>={minVal}'
+    if maxVal is not None:
+        return f'{item} x<={maxVal}'
+    return f'{item} x'
+
+
+def get_string(i, item, optional=False, minLen=1, maxLen=None):
+    if i < len(sys.argv):
+        argstr = sys.argv[i]
+        if argstr:
+            if (len(argstr) >= minLen) and ((maxLen is None) or
+                                            (len(argstr) <= maxLen)):
+                return argstr
+            controlflow.system_error_exit(
+                2,
+                f'expected <{integerLimits(minLen, maxLen, "string length")} for {item}>'
+            )
+        if optional or (minLen == 0):
+            return ''
+        controlflow.system_error_exit(2, f'expected a Non-empty <{item}>')
+    elif optional:
+        return ''
+    controlflow.system_error_exit(2, f'expected a <{item}>')
+
+
+def get_delta(argstr, pattern):
+    tg = pattern.match(argstr.lower())
+    if tg is None:
+        return None
+    sign = tg.group(1)
+    delta = int(tg.group(2))
+    unit = tg.group(3)
+    if unit == 'y':
+        deltaTime = datetime.timedelta(days=delta * 365)
+    elif unit == 'w':
+        deltaTime = datetime.timedelta(weeks=delta)
+    elif unit == 'd':
+        deltaTime = datetime.timedelta(days=delta)
+    elif unit == 'h':
+        deltaTime = datetime.timedelta(hours=delta)
+    elif unit == 'm':
+        deltaTime = datetime.timedelta(minutes=delta)
+    if sign == '-':
+        return -deltaTime
+    return deltaTime
+
+
+def get_delta_date(argstr):
+    deltaDate = get_delta(argstr, DELTA_DATE_PATTERN)
+    if deltaDate is None:
+        controlflow.system_error_exit(
+            2, f'expected a <{DELTA_DATE_FORMAT_REQUIRED}>; got {argstr}')
+    return deltaDate
+
+
+def get_delta_time(argstr):
+    deltaTime = get_delta(argstr, DELTA_TIME_PATTERN)
+    if deltaTime is None:
+        controlflow.system_error_exit(
+            2, f'expected a <{DELTA_TIME_FORMAT_REQUIRED}>; got {argstr}')
+    return deltaTime
+
+
+def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
+    argstr = argstr.strip()
+    if argstr:
+        if argstr[0] in ['+', '-']:
+            today = datetime.date.today()
+            argstr = (datetime.datetime(today.year, today.month, today.day) +
+                      get_delta_date(argstr)).strftime(YYYYMMDD_FORMAT)
+        try:
+            dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
+            if returnTimeStamp:
+                return time.mktime(dateTime.timetuple()) * 1000
+            if returnDateTime:
+                return dateTime
+            return argstr
+        except ValueError:
+            controlflow.system_error_exit(
+                2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>; got {argstr}')
+    elif minLen == 0:
+        return ''
+    controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>')
+
+
+def get_time_or_delta_from_now(time_string):
+    """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+    time_string = time_string.strip().upper()
+    if time_string:
+        if time_string[0] not in ['+', '-']:
+            return time_string
+        return (datetime.datetime.utcnow() +
+                get_delta_time(time_string)).isoformat() + 'Z'
+    controlflow.system_error_exit(
+        2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+
+def get_row_filter_date_or_delta_from_now(date_string):
+    """Get an ISO 8601 date or a positive/negative delta applied to now.
+  Args:
+    date_string (string): The time or delta (e.g. '2017-09-01' or '-4y')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+    date_string = date_string.strip().upper()
+    if date_string:
+        if date_string[0] in ['+', '-']:
+            deltaDate = get_delta(date_string, DELTA_DATE_PATTERN)
+            if deltaDate is None:
+                return (False, DELTA_DATE_FORMAT_REQUIRED)
+            today = datetime.date.today()
+            return (True,
+                    (datetime.datetime(today.year, today.month, today.day) +
+                     deltaDate).isoformat() + 'Z')
+        try:
+            deltaDate = dateutil.parser.parse(date_string, ignoretz=True)
+            return (True,
+                    datetime.datetime(deltaDate.year, deltaDate.month,
+                                      deltaDate.day).isoformat() + 'Z')
+        except ValueError:
+            pass
+    return (False, YYYYMMDD_FORMAT_REQUIRED)
+
+
+def get_row_filter_time_or_delta_from_now(time_string):
+    """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  Exits:
+    2: Not a valid delta.
+  """
+    time_string = time_string.strip().upper()
+    if time_string:
+        if time_string[0] in ['+', '-']:
+            deltaTime = get_delta(time_string, DELTA_TIME_PATTERN)
+            if deltaTime is None:
+                return (False, DELTA_TIME_FORMAT_REQUIRED)
+            return (True,
+                    (datetime.datetime.utcnow() + deltaTime).isoformat() + 'Z')
+        try:
+            deltaTime = dateutil.parser.parse(time_string, ignoretz=True)
+            return (True, deltaTime.isoformat() + 'Z')
+        except ValueError:
+            pass
+    return (False, YYYYMMDDTHHMMSS_FORMAT_REQUIRED)
+
+
+def get_date_zero_time_or_full_time(time_string):
+    time_string = time_string.strip()
+    if time_string:
+        if YYYYMMDD_PATTERN.match(time_string):
+            return get_yyyymmdd(time_string) + 'T00:00:00.000Z'
+        return get_time_or_delta_from_now(time_string)
+    controlflow.system_error_exit(
+        2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+
+def md5_matches_file(local_file, expected_md5, exitOnError):
+    f = fileutils.open_file(local_file, 'rb')
+    hash_md5 = md5()
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+    actual_hash = hash_md5.hexdigest()
+    if exitOnError and actual_hash != expected_md5:
+        controlflow.system_error_exit(
+            6, f'actual hash was {actual_hash}. Exiting on corrupt file.')
+    return actual_hash == expected_md5
+
+
+URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+
+def shorten_url(long_url, httpc=None):
+    if GC_Defaults[GC_NO_SHORT_URLS]:
+        return long_url
+    if not httpc:
+        httpc = transport.create_http(timeout=10)
+    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
+    try:
+        payload = json.dumps({'long_url': long_url})
+        resp, content = httpc.request(URL_SHORTENER_ENDPOINT,
+                                      'POST',
+                                      payload,
+                                      headers=headers)
+    except:
+        return long_url
+    if resp.status != 200:
+        return long_url
+    try:
+        if isinstance(content, bytes):
+            content = content.decode()
+        return json.loads(content).get('short_url', long_url)
+    except:
+        return long_url

src/google/__init__.py

@@ -1,22 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google namespace package."""
-
-try:
-    import pkg_resources
-    pkg_resources.declare_namespace(__name__)
-except ImportError:
-    import pkgutil
-    __path__ = pkgutil.extend_path(__path__, __name__)

src/google/auth/__init__.py

@@ -1,28 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google Auth Library for Python."""
-
-import logging
-
-from google.auth._default import default
-
-
-__all__ = [
-    'default',
-]
-
-
-# Set default logging handler to avoid "No handler found" warnings.
-logging.getLogger(__name__).addHandler(logging.NullHandler())

src/google/auth/_cloud_sdk.py

@@ -1,126 +0,0 @@
-# Copyright 2015 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helpers for reading the Google Cloud SDK's configuration."""
-
-import json
-import os
-import subprocess
-
-from google.auth import environment_vars
-import google.oauth2.credentials
-
-
-# The ~/.config subdirectory containing gcloud credentials.
-_CONFIG_DIRECTORY = 'gcloud'
-# Windows systems store config at %APPDATA%\gcloud
-_WINDOWS_CONFIG_ROOT_ENV_VAR = 'APPDATA'
-# The name of the file in the Cloud SDK config that contains default
-# credentials.
-_CREDENTIALS_FILENAME = 'application_default_credentials.json'
-# The name of the Cloud SDK shell script
-_CLOUD_SDK_POSIX_COMMAND = 'gcloud'
-_CLOUD_SDK_WINDOWS_COMMAND = 'gcloud.cmd'
-# The command to get the Cloud SDK configuration
-_CLOUD_SDK_CONFIG_COMMAND = ('config', 'config-helper', '--format', 'json')
-# Cloud SDK's application-default client ID
-CLOUD_SDK_CLIENT_ID = (
-    '764086051850-6qr4p6gpi6hn506pt8ejuq83di341hur.apps.googleusercontent.com')
-
-
-def get_config_path():
-    """Returns the absolute path the the Cloud SDK's configuration directory.
-
-    Returns:
-        str: The Cloud SDK config path.
-    """
-    # If the path is explicitly set, return that.
-    try:
-        return os.environ[environment_vars.CLOUD_SDK_CONFIG_DIR]
-    except KeyError:
-        pass
-
-    # Non-windows systems store this at ~/.config/gcloud
-    if os.name != 'nt':
-        return os.path.join(
-            os.path.expanduser('~'), '.config', _CONFIG_DIRECTORY)
-    # Windows systems store config at %APPDATA%\gcloud
-    else:
-        try:
-            return os.path.join(
-                os.environ[_WINDOWS_CONFIG_ROOT_ENV_VAR],
-                _CONFIG_DIRECTORY)
-        except KeyError:
-            # This should never happen unless someone is really
-            # messing with things, but we'll cover the case anyway.
-            drive = os.environ.get('SystemDrive', 'C:')
-            return os.path.join(
-                drive, '\\', _CONFIG_DIRECTORY)
-
-
-def get_application_default_credentials_path():
-    """Gets the path to the application default credentials file.
-
-    The path may or may not exist.
-
-    Returns:
-        str: The full path to application default credentials.
-    """
-    config_path = get_config_path()
-    return os.path.join(config_path, _CREDENTIALS_FILENAME)
-
-
-def load_authorized_user_credentials(info):
-    """Loads an authorized user credential.
-
-    Args:
-        info (Mapping[str, str]): The loaded file's data.
-
-    Returns:
-        google.oauth2.credentials.Credentials: The constructed credentials.
-
-    Raises:
-        ValueError: if the info is in the wrong format or missing data.
-    """
-    return google.oauth2.credentials.Credentials.from_authorized_user_info(
-        info)
-
-
-def get_project_id():
-    """Gets the project ID from the Cloud SDK.
-
-    Returns:
-        Optional[str]: The project ID.
-    """
-    if os.name == 'nt':
-        command = _CLOUD_SDK_WINDOWS_COMMAND
-    else:
-        command = _CLOUD_SDK_POSIX_COMMAND
-
-    try:
-        output = subprocess.check_output(
-            (command,) + _CLOUD_SDK_CONFIG_COMMAND,
-            stderr=subprocess.STDOUT)
-    except (subprocess.CalledProcessError, OSError, IOError):
-        return None
-
-    try:
-        configuration = json.loads(output.decode('utf-8'))
-    except ValueError:
-        return None
-
-    try:
-        return configuration['configuration']['properties']['core']['project']
-    except KeyError:
-        return None

src/google/auth/_default.py

@@ -1,306 +0,0 @@
-# Copyright 2015 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Application default credentials.
-
-Implements application default credentials and project ID detection.
-"""
-
-import io
-import json
-import logging
-import os
-import warnings
-
-import six
-
-from google.auth import environment_vars
-from google.auth import exceptions
-import google.auth.transport._http_client
-
-_LOGGER = logging.getLogger(__name__)
-
-# Valid types accepted for file-based credentials.
-_AUTHORIZED_USER_TYPE = 'authorized_user'
-_SERVICE_ACCOUNT_TYPE = 'service_account'
-_VALID_TYPES = (_AUTHORIZED_USER_TYPE, _SERVICE_ACCOUNT_TYPE)
-
-# Help message when no credentials can be found.
-_HELP_MESSAGE = """\
-Could not automatically determine credentials. Please set {env} or \
-explicitly create credentials and re-run the application. For more \
-information, please see \
-https://developers.google.com/accounts/docs/application-default-credentials.
-""".format(env=environment_vars.CREDENTIALS).strip()
-
-# Warning when using Cloud SDK user credentials
-_CLOUD_SDK_CREDENTIALS_WARNING = """\
-Your application has authenticated using end user credentials from Google \
-Cloud SDK. We recommend that most server applications use service accounts \
-instead. If your application continues to use end user credentials from Cloud \
-SDK, you might receive a "quota exceeded" or "API not enabled" error. For \
-more information about service accounts, see \
-https://cloud.google.com/docs/authentication/."""
-
-
-def _warn_about_problematic_credentials(credentials):
-    """Determines if the credentials are problematic.
-
-    Credentials from the Cloud SDK that are associated with Cloud SDK's project
-    are problematic because they may not have APIs enabled and have limited
-    quota. If this is the case, warn about it.
-    """
-    from google.auth import _cloud_sdk
-    if credentials.client_id == _cloud_sdk.CLOUD_SDK_CLIENT_ID:
-        warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)
-
-
-def _load_credentials_from_file(filename):
-    """Loads credentials from a file.
-
-    The credentials file must be a service account key or stored authorized
-    user credentials.
-
-    Args:
-        filename (str): The full path to the credentials file.
-
-    Returns:
-        Tuple[google.auth.credentials.Credentials, Optional[str]]: Loaded
-            credentials and the project ID. Authorized user credentials do not
-            have the project ID information.
-
-    Raises:
-        google.auth.exceptions.DefaultCredentialsError: if the file is in the
-            wrong format or is missing.
-    """
-    if not os.path.exists(filename):
-        raise exceptions.DefaultCredentialsError(
-            'File {} was not found.'.format(filename))
-
-    with io.open(filename, 'r') as file_obj:
-        try:
-            info = json.load(file_obj)
-        except ValueError as caught_exc:
-            new_exc = exceptions.DefaultCredentialsError(
-                'File {} is not a valid json file.'.format(filename),
-                caught_exc)
-            six.raise_from(new_exc, caught_exc)
-
-    # The type key should indicate that the file is either a service account
-    # credentials file or an authorized user credentials file.
-    credential_type = info.get('type')
-
-    if credential_type == _AUTHORIZED_USER_TYPE:
-        from google.auth import _cloud_sdk
-
-        try:
-            credentials = _cloud_sdk.load_authorized_user_credentials(info)
-        except ValueError as caught_exc:
-            msg = 'Failed to load authorized user credentials from {}'.format(
-                filename)
-            new_exc = exceptions.DefaultCredentialsError(msg, caught_exc)
-            six.raise_from(new_exc, caught_exc)
-        # Authorized user credentials do not contain the project ID.
-        _warn_about_problematic_credentials(credentials)
-        return credentials, None
-
-    elif credential_type == _SERVICE_ACCOUNT_TYPE:
-        from google.oauth2 import service_account
-
-        try:
-            credentials = (
-                service_account.Credentials.from_service_account_info(info))
-        except ValueError as caught_exc:
-            msg = 'Failed to load service account credentials from {}'.format(
-                filename)
-            new_exc = exceptions.DefaultCredentialsError(msg, caught_exc)
-            six.raise_from(new_exc, caught_exc)
-        return credentials, info.get('project_id')
-
-    else:
-        raise exceptions.DefaultCredentialsError(
-            'The file {file} does not have a valid type. '
-            'Type is {type}, expected one of {valid_types}.'.format(
-                file=filename, type=credential_type, valid_types=_VALID_TYPES))
-
-
-def _get_gcloud_sdk_credentials():
-    """Gets the credentials and project ID from the Cloud SDK."""
-    from google.auth import _cloud_sdk
-
-    # Check if application default credentials exist.
-    credentials_filename = (
-        _cloud_sdk.get_application_default_credentials_path())
-
-    if not os.path.isfile(credentials_filename):
-        return None, None
-
-    credentials, project_id = _load_credentials_from_file(
-        credentials_filename)
-
-    if not project_id:
-        project_id = _cloud_sdk.get_project_id()
-
-    return credentials, project_id
-
-
-def _get_explicit_environ_credentials():
-    """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment
-    variable."""
-    explicit_file = os.environ.get(environment_vars.CREDENTIALS)
-
-    if explicit_file is not None:
-        credentials, project_id = _load_credentials_from_file(
-            os.environ[environment_vars.CREDENTIALS])
-
-        return credentials, project_id
-
-    else:
-        return None, None
-
-
-def _get_gae_credentials():
-    """Gets Google App Engine App Identity credentials and project ID."""
-    from google.auth import app_engine
-
-    try:
-        credentials = app_engine.Credentials()
-        project_id = app_engine.get_project_id()
-        return credentials, project_id
-    except EnvironmentError:
-        return None, None
-
-
-def _get_gce_credentials(request=None):
-    """Gets credentials and project ID from the GCE Metadata Service."""
-    # Ping requires a transport, but we want application default credentials
-    # to require no arguments. So, we'll use the _http_client transport which
-    # uses http.client. This is only acceptable because the metadata server
-    # doesn't do SSL and never requires proxies.
-    from google.auth import compute_engine
-    from google.auth.compute_engine import _metadata
-
-    if request is None:
-        request = google.auth.transport._http_client.Request()
-
-    if _metadata.ping(request=request):
-        # Get the project ID.
-        try:
-            project_id = _metadata.get_project_id(request=request)
-        except exceptions.TransportError:
-            project_id = None
-
-        return compute_engine.Credentials(), project_id
-    else:
-        return None, None
-
-
-def default(scopes=None, request=None):
-    """Gets the default credentials for the current environment.
-
-    `Application Default Credentials`_ provides an easy way to obtain
-    credentials to call Google APIs for server-to-server or local applications.
-    This function acquires credentials from the environment in the following
-    order:
-
-    1. If the environment variable ``GOOGLE_APPLICATION_CREDENTIALS`` is set
-       to the path of a valid service account JSON private key file, then it is
-       loaded and returned. The project ID returned is the project ID defined
-       in the service account file if available (some older files do not
-       contain project ID information).
-    2. If the `Google Cloud SDK`_ is installed and has application default
-       credentials set they are loaded and returned.
-
-       To enable application default credentials with the Cloud SDK run::
-
-            gcloud auth application-default login
-
-       If the Cloud SDK has an active project, the project ID is returned. The
-       active project can be set using::
-
-            gcloud config set project
-
-    3. If the application is running in the `App Engine standard environment`_
-       then the credentials and project ID from the `App Identity Service`_
-       are used.
-    4. If the application is running in `Compute Engine`_ or the
-       `App Engine flexible environment`_ then the credentials and project ID
-       are obtained from the `Metadata Service`_.
-    5. If no credentials are found,
-       :class:`~google.auth.exceptions.DefaultCredentialsError` will be raised.
-
-    .. _Application Default Credentials: https://developers.google.com\
-            /identity/protocols/application-default-credentials
-    .. _Google Cloud SDK: https://cloud.google.com/sdk
-    .. _App Engine standard environment: https://cloud.google.com/appengine
-    .. _App Identity Service: https://cloud.google.com/appengine/docs/python\
-            /appidentity/
-    .. _Compute Engine: https://cloud.google.com/compute
-    .. _App Engine flexible environment: https://cloud.google.com\
-            /appengine/flexible
-    .. _Metadata Service: https://cloud.google.com/compute/docs\
-            /storing-retrieving-metadata
-
-    Example::
-
-        import google.auth
-
-        credentials, project_id = google.auth.default()
-
-    Args:
-        scopes (Sequence[str]): The list of scopes for the credentials. If
-            specified, the credentials will automatically be scoped if
-            necessary.
-        request (google.auth.transport.Request): An object used to make
-            HTTP requests. This is used to detect whether the application
-            is running on Compute Engine. If not specified, then it will
-            use the standard library http client to make requests.
-
-    Returns:
-        Tuple[~google.auth.credentials.Credentials, Optional[str]]:
-            the current environment's credentials and project ID. Project ID
-            may be None, which indicates that the Project ID could not be
-            ascertained from the environment.
-
-    Raises:
-        ~google.auth.exceptions.DefaultCredentialsError:
-            If no credentials were found, or if the credentials found were
-            invalid.
-    """
-    from google.auth.credentials import with_scopes_if_required
-
-    explicit_project_id = os.environ.get(
-        environment_vars.PROJECT,
-        os.environ.get(environment_vars.LEGACY_PROJECT))
-
-    checkers = (
-        _get_explicit_environ_credentials,
-        _get_gcloud_sdk_credentials,
-        _get_gae_credentials,
-        lambda: _get_gce_credentials(request))
-
-    for checker in checkers:
-        credentials, project_id = checker()
-        if credentials is not None:
-            credentials = with_scopes_if_required(credentials, scopes)
-            effective_project_id = explicit_project_id or project_id
-            if not effective_project_id:
-                _LOGGER.warning(
-                    'No project ID could be determined. Consider running '
-                    '`gcloud config set project` or setting the %s '
-                    'environment variable',
-                    environment_vars.PROJECT)
-            return credentials, effective_project_id
-
-    raise exceptions.DefaultCredentialsError(_HELP_MESSAGE)

src/google/auth/_helpers.py

@@ -1,217 +0,0 @@
-# Copyright 2015 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helper functions for commonly used utilities."""
-
-import base64
-import calendar
-import datetime
-
-import six
-from six.moves import urllib
-
-
-CLOCK_SKEW_SECS = 300  # 5 minutes in seconds
-CLOCK_SKEW = datetime.timedelta(seconds=CLOCK_SKEW_SECS)
-
-
-def copy_docstring(source_class):
-    """Decorator that copies a method's docstring from another class.
-
-    Args:
-        source_class (type): The class that has the documented method.
-
-    Returns:
-        Callable: A decorator that will copy the docstring of the same
-            named method in the source class to the decorated method.
-    """
-    def decorator(method):
-        """Decorator implementation.
-
-        Args:
-            method (Callable): The method to copy the docstring to.
-
-        Returns:
-            Callable: the same method passed in with an updated docstring.
-
-        Raises:
-            ValueError: if the method already has a docstring.
-        """
-        if method.__doc__:
-            raise ValueError('Method already has a docstring.')
-
-        source_method = getattr(source_class, method.__name__)
-        method.__doc__ = source_method.__doc__
-
-        return method
-    return decorator
-
-
-def utcnow():
-    """Returns the current UTC datetime.
-
-    Returns:
-        datetime: The current time in UTC.
-    """
-    return datetime.datetime.utcnow()
-
-
-def datetime_to_secs(value):
-    """Convert a datetime object to the number of seconds since the UNIX epoch.
-
-    Args:
-        value (datetime): The datetime to convert.
-
-    Returns:
-        int: The number of seconds since the UNIX epoch.
-    """
-    return calendar.timegm(value.utctimetuple())
-
-
-def to_bytes(value, encoding='utf-8'):
-    """Converts a string value to bytes, if necessary.
-
-    Unfortunately, ``six.b`` is insufficient for this task since in
-    Python 2 because it does not modify ``unicode`` objects.
-
-    Args:
-        value (Union[str, bytes]): The value to be converted.
-        encoding (str): The encoding to use to convert unicode to bytes.
-            Defaults to "utf-8".
-
-    Returns:
-        bytes: The original value converted to bytes (if unicode) or as
-            passed in if it started out as bytes.
-
-    Raises:
-        ValueError: If the value could not be converted to bytes.
-    """
-    result = (value.encode(encoding)
-              if isinstance(value, six.text_type) else value)
-    if isinstance(result, six.binary_type):
-        return result
-    else:
-        raise ValueError('{0!r} could not be converted to bytes'.format(value))
-
-
-def from_bytes(value):
-    """Converts bytes to a string value, if necessary.
-
-    Args:
-        value (Union[str, bytes]): The value to be converted.
-
-    Returns:
-        str: The original value converted to unicode (if bytes) or as passed in
-            if it started out as unicode.
-
-    Raises:
-        ValueError: If the value could not be converted to unicode.
-    """
-    result = (value.decode('utf-8')
-              if isinstance(value, six.binary_type) else value)
-    if isinstance(result, six.text_type):
-        return result
-    else:
-        raise ValueError(
-            '{0!r} could not be converted to unicode'.format(value))
-
-
-def update_query(url, params, remove=None):
-    """Updates a URL's query parameters.
-
-    Replaces any current values if they are already present in the URL.
-
-    Args:
-        url (str): The URL to update.
-        params (Mapping[str, str]): A mapping of query parameter
-            keys to values.
-        remove (Sequence[str]): Parameters to remove from the query string.
-
-    Returns:
-        str: The URL with updated query parameters.
-
-    Examples:
-
-        >>> url = 'http://example.com?a=1'
-        >>> update_query(url, {'a': '2'})
-        http://example.com?a=2
-        >>> update_query(url, {'b': '3'})
-        http://example.com?a=1&b=3
-        >> update_query(url, {'b': '3'}, remove=['a'])
-        http://example.com?b=3
-
-    """
-    if remove is None:
-        remove = []
-
-    # Split the URL into parts.
-    parts = urllib.parse.urlparse(url)
-    # Parse the query string.
-    query_params = urllib.parse.parse_qs(parts.query)
-    # Update the query parameters with the new parameters.
-    query_params.update(params)
-    # Remove any values specified in remove.
-    query_params = {
-        key: value for key, value
-        in six.iteritems(query_params)
-        if key not in remove}
-    # Re-encoded the query string.
-    new_query = urllib.parse.urlencode(query_params, doseq=True)
-    # Unsplit the url.
-    new_parts = parts._replace(query=new_query)
-    return urllib.parse.urlunparse(new_parts)
-
-
-def scopes_to_string(scopes):
-    """Converts scope value to a string suitable for sending to OAuth 2.0
-    authorization servers.
-
-    Args:
-        scopes (Sequence[str]): The sequence of scopes to convert.
-
-    Returns:
-        str: The scopes formatted as a single string.
-    """
-    return ' '.join(scopes)
-
-
-def string_to_scopes(scopes):
-    """Converts stringifed scopes value to a list.
-
-    Args:
-        scopes (Union[Sequence, str]): The string of space-separated scopes
-            to convert.
-    Returns:
-        Sequence(str): The separated scopes.
-    """
-    if not scopes:
-        return []
-
-    return scopes.split(' ')
-
-
-def padded_urlsafe_b64decode(value):
-    """Decodes base64 strings lacking padding characters.
-
-    Google infrastructure tends to omit the base64 padding characters.
-
-    Args:
-        value (Union[str, bytes]): The encoded value.
-
-    Returns:
-        bytes: The decoded value
-    """
-    b64string = to_bytes(value)
-    padded = b64string + b'=' * (-len(b64string) % 4)
-    return base64.urlsafe_b64decode(padded)

src/google/auth/_oauth2client.py

@@ -1,170 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helpers for transitioning from oauth2client to google-auth.
-
-.. warning::
-    This module is private as it is intended to assist first-party downstream
-    clients with the transition from oauth2client to google-auth.
-"""
-
-from __future__ import absolute_import
-
-import six
-
-from google.auth import _helpers
-import google.auth.app_engine
-import google.oauth2.credentials
-import google.oauth2.service_account
-
-try:
-    import oauth2client.client
-    import oauth2client.contrib.gce
-    import oauth2client.service_account
-except ImportError as caught_exc:
-    six.raise_from(
-        ImportError('oauth2client is not installed.'), caught_exc)
-
-try:
-    import oauth2client.contrib.appengine
-    _HAS_APPENGINE = True
-except ImportError:
-    _HAS_APPENGINE = False
-
-
-_CONVERT_ERROR_TMPL = (
-    'Unable to convert {} to a google-auth credentials class.')
-
-
-def _convert_oauth2_credentials(credentials):
-    """Converts to :class:`google.oauth2.credentials.Credentials`.
-
-    Args:
-        credentials (Union[oauth2client.client.OAuth2Credentials,
-            oauth2client.client.GoogleCredentials]): The credentials to
-            convert.
-
-    Returns:
-        google.oauth2.credentials.Credentials: The converted credentials.
-    """
-    new_credentials = google.oauth2.credentials.Credentials(
-        token=credentials.access_token,
-        refresh_token=credentials.refresh_token,
-        token_uri=credentials.token_uri,
-        client_id=credentials.client_id,
-        client_secret=credentials.client_secret,
-        scopes=credentials.scopes)
-
-    new_credentials._expires = credentials.token_expiry
-
-    return new_credentials
-
-
-def _convert_service_account_credentials(credentials):
-    """Converts to :class:`google.oauth2.service_account.Credentials`.
-
-    Args:
-        credentials (Union[
-            oauth2client.service_account.ServiceAccountCredentials,
-            oauth2client.service_account._JWTAccessCredentials]): The
-            credentials to convert.
-
-    Returns:
-        google.oauth2.service_account.Credentials: The converted credentials.
-    """
-    info = credentials.serialization_data.copy()
-    info['token_uri'] = credentials.token_uri
-    return google.oauth2.service_account.Credentials.from_service_account_info(
-        info)
-
-
-def _convert_gce_app_assertion_credentials(credentials):
-    """Converts to :class:`google.auth.compute_engine.Credentials`.
-
-    Args:
-        credentials (oauth2client.contrib.gce.AppAssertionCredentials): The
-            credentials to convert.
-
-    Returns:
-        google.oauth2.service_account.Credentials: The converted credentials.
-    """
-    return google.auth.compute_engine.Credentials(
-        service_account_email=credentials.service_account_email)
-
-
-def _convert_appengine_app_assertion_credentials(credentials):
-    """Converts to :class:`google.auth.app_engine.Credentials`.
-
-    Args:
-        credentials (oauth2client.contrib.app_engine.AppAssertionCredentials):
-            The credentials to convert.
-
-    Returns:
-        google.oauth2.service_account.Credentials: The converted credentials.
-    """
-    # pylint: disable=invalid-name
-    return google.auth.app_engine.Credentials(
-        scopes=_helpers.string_to_scopes(credentials.scope),
-        service_account_id=credentials.service_account_id)
-
-
-_CLASS_CONVERSION_MAP = {
-    oauth2client.client.OAuth2Credentials: _convert_oauth2_credentials,
-    oauth2client.client.GoogleCredentials: _convert_oauth2_credentials,
-    oauth2client.service_account.ServiceAccountCredentials:
-        _convert_service_account_credentials,
-    oauth2client.service_account._JWTAccessCredentials:
-        _convert_service_account_credentials,
-    oauth2client.contrib.gce.AppAssertionCredentials:
-        _convert_gce_app_assertion_credentials,
-}
-
-if _HAS_APPENGINE:
-    _CLASS_CONVERSION_MAP[
-        oauth2client.contrib.appengine.AppAssertionCredentials] = (
-            _convert_appengine_app_assertion_credentials)
-
-
-def convert(credentials):
-    """Convert oauth2client credentials to google-auth credentials.
-
-    This class converts:
-
-    - :class:`oauth2client.client.OAuth2Credentials` to
-      :class:`google.oauth2.credentials.Credentials`.
-    - :class:`oauth2client.client.GoogleCredentials` to
-      :class:`google.oauth2.credentials.Credentials`.
-    - :class:`oauth2client.service_account.ServiceAccountCredentials` to
-      :class:`google.oauth2.service_account.Credentials`.
-    - :class:`oauth2client.service_account._JWTAccessCredentials` to
-      :class:`google.oauth2.service_account.Credentials`.
-    - :class:`oauth2client.contrib.gce.AppAssertionCredentials` to
-      :class:`google.auth.compute_engine.Credentials`.
-    - :class:`oauth2client.contrib.appengine.AppAssertionCredentials` to
-      :class:`google.auth.app_engine.Credentials`.
-
-    Returns:
-        google.auth.credentials.Credentials: The converted credentials.
-
-    Raises:
-        ValueError: If the credentials could not be converted.
-    """
-
-    credentials_class = type(credentials)
-
-    try:
-        return _CLASS_CONVERSION_MAP[credentials_class](credentials)
-    except KeyError as caught_exc:
-        new_exc = ValueError(_CONVERT_ERROR_TMPL.format(credentials_class))
-        six.raise_from(new_exc, caught_exc)

src/google/auth/_service_account_info.py

@@ -1,73 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helper functions for loading data from a Google service account file."""
-
-import io
-import json
-
-import six
-
-from google.auth import crypt
-
-
-def from_dict(data, require=None):
-    """Validates a dictionary containing Google service account data.
-
-    Creates and returns a :class:`google.auth.crypt.Signer` instance from the
-    private key specified in the data.
-
-    Args:
-        data (Mapping[str, str]): The service account data
-        require (Sequence[str]): List of keys required to be present in the
-            info.
-
-    Returns:
-        google.auth.crypt.Signer: A signer created from the private key in the
-            service account file.
-
-    Raises:
-        ValueError: if the data was in the wrong format, or if one of the
-            required keys is missing.
-    """
-    keys_needed = set(require if require is not None else [])
-
-    missing = keys_needed.difference(six.iterkeys(data))
-
-    if missing:
-        raise ValueError(
-            'Service account info was not in the expected format, missing '
-            'fields {}.'.format(', '.join(missing)))
-
-    # Create a signer.
-    signer = crypt.RSASigner.from_service_account_info(data)
-
-    return signer
-
-
-def from_filename(filename, require=None):
-    """Reads a Google service account JSON file and returns its parsed info.
-
-    Args:
-        filename (str): The path to the service account .json file.
-        require (Sequence[str]): List of keys required to be present in the
-            info.
-
-    Returns:
-        Tuple[ Mapping[str, str], google.auth.crypt.Signer ]: The verified
-            info and a signer instance.
-    """
-    with io.open(filename, 'r', encoding='utf-8') as json_file:
-        data = json.load(json_file)
-        return data, from_dict(data, require=require)

src/google/auth/app_engine.py

@@ -1,154 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google App Engine standard environment support.
-
-This module provides authentication and signing for applications running on App
-Engine in the standard environment using the `App Identity API`_.
-
-
-.. _App Identity API:
-    https://cloud.google.com/appengine/docs/python/appidentity/
-"""
-
-import datetime
-
-from google.auth import _helpers
-from google.auth import credentials
-from google.auth import crypt
-
-try:
-    from google.appengine.api import app_identity
-except ImportError:
-    app_identity = None
-
-
-class Signer(crypt.Signer):
-    """Signs messages using the App Engine App Identity service.
-
-    This can be used in place of :class:`google.auth.crypt.Signer` when
-    running in the App Engine standard environment.
-    """
-
-    @property
-    def key_id(self):
-        """Optional[str]: The key ID used to identify this private key.
-
-        .. warning::
-           This is always ``None``. The key ID used by App Engine can not
-           be reliably determined ahead of time.
-        """
-        return None
-
-    @_helpers.copy_docstring(crypt.Signer)
-    def sign(self, message):
-        message = _helpers.to_bytes(message)
-        _, signature = app_identity.sign_blob(message)
-        return signature
-
-
-def get_project_id():
-    """Gets the project ID for the current App Engine application.
-
-    Returns:
-        str: The project ID
-
-    Raises:
-        EnvironmentError: If the App Engine APIs are unavailable.
-    """
-    # pylint: disable=missing-raises-doc
-    # Pylint rightfully thinks EnvironmentError is OSError, but doesn't
-    # realize it's a valid alias.
-    if app_identity is None:
-        raise EnvironmentError(
-            'The App Engine APIs are not available.')
-    return app_identity.get_application_id()
-
-
-class Credentials(credentials.Scoped, credentials.Signing,
-                  credentials.Credentials):
-    """App Engine standard environment credentials.
-
-    These credentials use the App Engine App Identity API to obtain access
-    tokens.
-    """
-
-    def __init__(self, scopes=None, service_account_id=None):
-        """
-        Args:
-            scopes (Sequence[str]): Scopes to request from the App Identity
-                API.
-            service_account_id (str): The service account ID passed into
-                :func:`google.appengine.api.app_identity.get_access_token`.
-                If not specified, the default application service account
-                ID will be used.
-
-        Raises:
-            EnvironmentError: If the App Engine APIs are unavailable.
-        """
-        # pylint: disable=missing-raises-doc
-        # Pylint rightfully thinks EnvironmentError is OSError, but doesn't
-        # realize it's a valid alias.
-        if app_identity is None:
-            raise EnvironmentError(
-                'The App Engine APIs are not available.')
-
-        super(Credentials, self).__init__()
-        self._scopes = scopes
-        self._service_account_id = service_account_id
-        self._signer = Signer()
-
-    @_helpers.copy_docstring(credentials.Credentials)
-    def refresh(self, request):
-        # pylint: disable=unused-argument
-        token, ttl = app_identity.get_access_token(
-            self._scopes, self._service_account_id)
-        expiry = datetime.datetime.utcfromtimestamp(ttl)
-
-        self.token, self.expiry = token, expiry
-
-    @property
-    def service_account_email(self):
-        """The service account email."""
-        if self._service_account_id is None:
-            self._service_account_id = app_identity.get_service_account_name()
-        return self._service_account_id
-
-    @property
-    def requires_scopes(self):
-        """Checks if the credentials requires scopes.
-
-        Returns:
-            bool: True if there are no scopes set otherwise False.
-        """
-        return not self._scopes
-
-    @_helpers.copy_docstring(credentials.Scoped)
-    def with_scopes(self, scopes):
-        return self.__class__(
-            scopes=scopes, service_account_id=self._service_account_id)
-
-    @_helpers.copy_docstring(credentials.Signing)
-    def sign_bytes(self, message):
-        return self._signer.sign(message)
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer_email(self):
-        return self.service_account_email
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer(self):
-        return self._signer

src/google/auth/compute_engine/__init__.py

@@ -1,24 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google Compute Engine authentication."""
-
-from google.auth.compute_engine.credentials import Credentials
-from google.auth.compute_engine.credentials import IDTokenCredentials
-
-
-__all__ = [
-    'Credentials',
-    'IDTokenCredentials',
-]

src/google/auth/compute_engine/_metadata.py

@@ -1,204 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Provides helper methods for talking to the Compute Engine metadata server.
-
-See https://cloud.google.com/compute/docs/metadata for more details.
-"""
-
-import datetime
-import json
-import logging
-import os
-
-import six
-from six.moves import http_client
-from six.moves.urllib import parse as urlparse
-
-from google.auth import _helpers
-from google.auth import environment_vars
-from google.auth import exceptions
-
-_LOGGER = logging.getLogger(__name__)
-
-_METADATA_ROOT = 'http://{}/computeMetadata/v1/'.format(
-    os.getenv(environment_vars.GCE_METADATA_ROOT, 'metadata.google.internal'))
-
-# This is used to ping the metadata server, it avoids the cost of a DNS
-# lookup.
-_METADATA_IP_ROOT = 'http://{}'.format(
-    os.getenv(environment_vars.GCE_METADATA_IP, '169.254.169.254'))
-_METADATA_FLAVOR_HEADER = 'metadata-flavor'
-_METADATA_FLAVOR_VALUE = 'Google'
-_METADATA_HEADERS = {_METADATA_FLAVOR_HEADER: _METADATA_FLAVOR_VALUE}
-
-# Timeout in seconds to wait for the GCE metadata server when detecting the
-# GCE environment.
-try:
-    _METADATA_DEFAULT_TIMEOUT = int(os.getenv('GCE_METADATA_TIMEOUT', 3))
-except ValueError:  # pragma: NO COVER
-    _METADATA_DEFAULT_TIMEOUT = 3
-
-
-def ping(request, timeout=_METADATA_DEFAULT_TIMEOUT):
-    """Checks to see if the metadata server is available.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        timeout (int): How long to wait for the metadata server to respond.
-
-    Returns:
-        bool: True if the metadata server is reachable, False otherwise.
-    """
-    # NOTE: The explicit ``timeout`` is a workaround. The underlying
-    #       issue is that resolving an unknown host on some networks will take
-    #       20-30 seconds; making this timeout short fixes the issue, but
-    #       could lead to false negatives in the event that we are on GCE, but
-    #       the metadata resolution was particularly slow. The latter case is
-    #       "unlikely".
-    try:
-        response = request(
-            url=_METADATA_IP_ROOT, method='GET', headers=_METADATA_HEADERS,
-            timeout=timeout)
-
-        metadata_flavor = response.headers.get(_METADATA_FLAVOR_HEADER)
-        return (response.status == http_client.OK and
-                metadata_flavor == _METADATA_FLAVOR_VALUE)
-
-    except exceptions.TransportError:
-        _LOGGER.info('Compute Engine Metadata server unavailable.')
-        return False
-
-
-def get(request, path, root=_METADATA_ROOT, recursive=False):
-    """Fetch a resource from the metadata server.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        path (str): The resource to retrieve. For example,
-            ``'instance/service-accounts/default'``.
-        root (str): The full path to the metadata server root.
-        recursive (bool): Whether to do a recursive query of metadata. See
-            https://cloud.google.com/compute/docs/metadata#aggcontents for more
-            details.
-
-    Returns:
-        Union[Mapping, str]: If the metadata server returns JSON, a mapping of
-            the decoded JSON is return. Otherwise, the response content is
-            returned as a string.
-
-    Raises:
-        google.auth.exceptions.TransportError: if an error occurred while
-            retrieving metadata.
-    """
-    base_url = urlparse.urljoin(root, path)
-    query_params = {}
-
-    if recursive:
-        query_params['recursive'] = 'true'
-
-    url = _helpers.update_query(base_url, query_params)
-
-    response = request(url=url, method='GET', headers=_METADATA_HEADERS)
-
-    if response.status == http_client.OK:
-        content = _helpers.from_bytes(response.data)
-        if response.headers['content-type'] == 'application/json':
-            try:
-                return json.loads(content)
-            except ValueError as caught_exc:
-                new_exc = exceptions.TransportError(
-                    'Received invalid JSON from the Google Compute Engine'
-                    'metadata service: {:.20}'.format(content))
-                six.raise_from(new_exc, caught_exc)
-        else:
-            return content
-    else:
-        raise exceptions.TransportError(
-            'Failed to retrieve {} from the Google Compute Engine'
-            'metadata service. Status: {} Response:\n{}'.format(
-                url, response.status, response.data), response)
-
-
-def get_project_id(request):
-    """Get the Google Cloud Project ID from the metadata server.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-
-    Returns:
-        str: The project ID
-
-    Raises:
-        google.auth.exceptions.TransportError: if an error occurred while
-            retrieving metadata.
-    """
-    return get(request, 'project/project-id')
-
-
-def get_service_account_info(request, service_account='default'):
-    """Get information about a service account from the metadata server.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        service_account (str): The string 'default' or a service account email
-            address. The determines which service account for which to acquire
-            information.
-
-    Returns:
-        Mapping: The service account's information, for example::
-
-            {
-                'email': '...',
-                'scopes': ['scope', ...],
-                'aliases': ['default', '...']
-            }
-
-    Raises:
-        google.auth.exceptions.TransportError: if an error occurred while
-            retrieving metadata.
-    """
-    return get(
-        request,
-        'instance/service-accounts/{0}/'.format(service_account),
-        recursive=True)
-
-
-def get_service_account_token(request, service_account='default'):
-    """Get the OAuth 2.0 access token for a service account.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        service_account (str): The string 'default' or a service account email
-            address. The determines which service account for which to acquire
-            an access token.
-
-    Returns:
-        Union[str, datetime]: The access token and its expiration.
-
-    Raises:
-        google.auth.exceptions.TransportError: if an error occurred while
-            retrieving metadata.
-    """
-    token_json = get(
-        request,
-        'instance/service-accounts/{0}/token'.format(service_account))
-    token_expiry = _helpers.utcnow() + datetime.timedelta(
-        seconds=token_json['expires_in'])
-    return token_json['access_token'], token_expiry

src/google/auth/compute_engine/credentials.py

@@ -1,239 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google Compute Engine credentials.
-
-This module provides authentication for application running on Google Compute
-Engine using the Compute Engine metadata server.
-
-"""
-
-import datetime
-
-import six
-
-from google.auth import _helpers
-from google.auth import credentials
-from google.auth import exceptions
-from google.auth import iam
-from google.auth import jwt
-from google.auth.compute_engine import _metadata
-from google.oauth2 import _client
-
-
-class Credentials(credentials.ReadOnlyScoped, credentials.Credentials):
-    """Compute Engine Credentials.
-
-    These credentials use the Google Compute Engine metadata server to obtain
-    OAuth 2.0 access tokens associated with the instance's service account.
-
-    For more information about Compute Engine authentication, including how
-    to configure scopes, see the `Compute Engine authentication
-    documentation`_.
-
-    .. note:: Compute Engine instances can be created with scopes and therefore
-        these credentials are considered to be 'scoped'. However, you can
-        not use :meth:`~google.auth.credentials.ScopedCredentials.with_scopes`
-        because it is not possible to change the scopes that the instance
-        has. Also note that
-        :meth:`~google.auth.credentials.ScopedCredentials.has_scopes` will not
-        work until the credentials have been refreshed.
-
-    .. _Compute Engine authentication documentation:
-        https://cloud.google.com/compute/docs/authentication#using
-    """
-
-    def __init__(self, service_account_email='default'):
-        """
-        Args:
-            service_account_email (str): The service account email to use, or
-                'default'. A Compute Engine instance may have multiple service
-                accounts.
-        """
-        super(Credentials, self).__init__()
-        self._service_account_email = service_account_email
-
-    def _retrieve_info(self, request):
-        """Retrieve information about the service account.
-
-        Updates the scopes and retrieves the full service account email.
-
-        Args:
-            request (google.auth.transport.Request): The object used to make
-                HTTP requests.
-        """
-        info = _metadata.get_service_account_info(
-            request,
-            service_account=self._service_account_email)
-
-        self._service_account_email = info['email']
-        self._scopes = info['scopes']
-
-    def refresh(self, request):
-        """Refresh the access token and scopes.
-
-        Args:
-            request (google.auth.transport.Request): The object used to make
-                HTTP requests.
-
-        Raises:
-            google.auth.exceptions.RefreshError: If the Compute Engine metadata
-                service can't be reached if if the instance has not
-                credentials.
-        """
-        try:
-            self._retrieve_info(request)
-            self.token, self.expiry = _metadata.get_service_account_token(
-                request,
-                service_account=self._service_account_email)
-        except exceptions.TransportError as caught_exc:
-            new_exc = exceptions.RefreshError(caught_exc)
-            six.raise_from(new_exc, caught_exc)
-
-    @property
-    def service_account_email(self):
-        """The service account email.
-
-        .. note: This is not guaranteed to be set until :meth`refresh` has been
-            called.
-        """
-        return self._service_account_email
-
-    @property
-    def requires_scopes(self):
-        """False: Compute Engine credentials can not be scoped."""
-        return False
-
-
-_DEFAULT_TOKEN_LIFETIME_SECS = 3600  # 1 hour in seconds
-_DEFAULT_TOKEN_URI = 'https://www.googleapis.com/oauth2/v4/token'
-
-
-class IDTokenCredentials(credentials.Credentials, credentials.Signing):
-    """Open ID Connect ID Token-based service account credentials.
-
-    These credentials relies on the default service account of a GCE instance.
-
-    In order for this to work, the GCE instance must have been started with
-    a service account that has access to the IAM Cloud API.
-    """
-    def __init__(self, request, target_audience,
-                 token_uri=_DEFAULT_TOKEN_URI,
-                 additional_claims=None,
-                 service_account_email=None):
-        """
-        Args:
-            request (google.auth.transport.Request): The object used to make
-                HTTP requests.
-            target_audience (str): The intended audience for these credentials,
-                used when requesting the ID Token. The ID Token's ``aud`` claim
-                will be set to this string.
-            token_uri (str): The OAuth 2.0 Token URI.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT assertion used in the authorization grant.
-            service_account_email (str): Optional explicit service account to
-                use to sign JWT tokens.
-                By default, this is the default GCE service account.
-        """
-        super(IDTokenCredentials, self).__init__()
-
-        if service_account_email is None:
-            sa_info = _metadata.get_service_account_info(request)
-            service_account_email = sa_info['email']
-        self._service_account_email = service_account_email
-
-        self._signer = iam.Signer(
-            request=request,
-            credentials=Credentials(),
-            service_account_email=service_account_email)
-
-        self._token_uri = token_uri
-        self._target_audience = target_audience
-
-        if additional_claims is not None:
-            self._additional_claims = additional_claims
-        else:
-            self._additional_claims = {}
-
-    def with_target_audience(self, target_audience):
-        """Create a copy of these credentials with the specified target
-        audience.
-        Args:
-            target_audience (str): The intended audience for these credentials,
-            used when requesting the ID Token.
-        Returns:
-            google.auth.service_account.IDTokenCredentials: A new credentials
-                instance.
-        """
-        return self.__class__(
-            self._signer,
-            service_account_email=self._service_account_email,
-            token_uri=self._token_uri,
-            target_audience=target_audience,
-            additional_claims=self._additional_claims.copy())
-
-    def _make_authorization_grant_assertion(self):
-        """Create the OAuth 2.0 assertion.
-        This assertion is used during the OAuth 2.0 grant to acquire an
-        ID token.
-        Returns:
-            bytes: The authorization grant assertion.
-        """
-        now = _helpers.utcnow()
-        lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS)
-        expiry = now + lifetime
-
-        payload = {
-            'iat': _helpers.datetime_to_secs(now),
-            'exp': _helpers.datetime_to_secs(expiry),
-            # The issuer must be the service account email.
-            'iss': self.service_account_email,
-            # The audience must be the auth token endpoint's URI
-            'aud': self._token_uri,
-            # The target audience specifies which service the ID token is
-            # intended for.
-            'target_audience': self._target_audience
-        }
-
-        payload.update(self._additional_claims)
-
-        token = jwt.encode(self._signer, payload)
-
-        return token
-
-    @_helpers.copy_docstring(credentials.Credentials)
-    def refresh(self, request):
-        assertion = self._make_authorization_grant_assertion()
-        access_token, expiry, _ = _client.id_token_jwt_grant(
-            request, self._token_uri, assertion)
-        self.token = access_token
-        self.expiry = expiry
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer(self):
-        return self._signer
-
-    @_helpers.copy_docstring(credentials.Signing)
-    def sign_bytes(self, message):
-        return self._signer.sign(message)
-
-    @property
-    def service_account_email(self):
-        """The service account email."""
-        return self._service_account_email
-
-    @property
-    def signer_email(self):
-        return self._service_account_email

src/google/auth/credentials.py

@@ -1,322 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-"""Interfaces for credentials."""
-
-import abc
-
-import six
-
-from google.auth import _helpers
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Credentials(object):
-    """Base class for all credentials.
-
-    All credentials have a :attr:`token` that is used for authentication and
-    may also optionally set an :attr:`expiry` to indicate when the token will
-    no longer be valid.
-
-    Most credentials will be :attr:`invalid` until :meth:`refresh` is called.
-    Credentials can do this automatically before the first HTTP request in
-    :meth:`before_request`.
-
-    Although the token and expiration will change as the credentials are
-    :meth:`refreshed <refresh>` and used, credentials should be considered
-    immutable. Various credentials will accept configuration such as private
-    keys, scopes, and other options. These options are not changeable after
-    construction. Some classes will provide mechanisms to copy the credentials
-    with modifications such as :meth:`ScopedCredentials.with_scopes`.
-    """
-    def __init__(self):
-        self.token = None
-        """str: The bearer token that can be used in HTTP headers to make
-        authenticated requests."""
-        self.expiry = None
-        """Optional[datetime]: When the token expires and is no longer valid.
-        If this is None, the token is assumed to never expire."""
-
-    @property
-    def expired(self):
-        """Checks if the credentials are expired.
-
-        Note that credentials can be invalid but not expired because
-        Credentials with :attr:`expiry` set to None is considered to never
-        expire.
-        """
-        if not self.expiry:
-            return False
-
-        # Remove 5 minutes from expiry to err on the side of reporting
-        # expiration early so that we avoid the 401-refresh-retry loop.
-        skewed_expiry = self.expiry - _helpers.CLOCK_SKEW
-        return _helpers.utcnow() >= skewed_expiry
-
-    @property
-    def valid(self):
-        """Checks the validity of the credentials.
-
-        This is True if the credentials have a :attr:`token` and the token
-        is not :attr:`expired`.
-        """
-        return self.token is not None and not self.expired
-
-    @abc.abstractmethod
-    def refresh(self, request):
-        """Refreshes the access token.
-
-        Args:
-            request (google.auth.transport.Request): The object used to make
-                HTTP requests.
-
-        Raises:
-            google.auth.exceptions.RefreshError: If the credentials could
-                not be refreshed.
-        """
-        # pylint: disable=missing-raises-doc
-        # (pylint doesn't recognize that this is abstract)
-        raise NotImplementedError('Refresh must be implemented')
-
-    def apply(self, headers, token=None):
-        """Apply the token to the authentication header.
-
-        Args:
-            headers (Mapping): The HTTP request headers.
-            token (Optional[str]): If specified, overrides the current access
-                token.
-        """
-        headers['authorization'] = 'Bearer {}'.format(
-            _helpers.from_bytes(token or self.token))
-
-    def before_request(self, request, method, url, headers):
-        """Performs credential-specific before request logic.
-
-        Refreshes the credentials if necessary, then calls :meth:`apply` to
-        apply the token to the authentication header.
-
-        Args:
-            request (google.auth.transport.Request): The object used to make
-                HTTP requests.
-            method (str): The request's HTTP method or the RPC method being
-                invoked.
-            url (str): The request's URI or the RPC service's URI.
-            headers (Mapping): The request's headers.
-        """
-        # pylint: disable=unused-argument
-        # (Subclasses may use these arguments to ascertain information about
-        # the http request.)
-        if not self.valid:
-            self.refresh(request)
-        self.apply(headers)
-
-
-class AnonymousCredentials(Credentials):
-    """Credentials that do not provide any authentication information.
-
-    These are useful in the case of services that support anonymous access or
-    local service emulators that do not use credentials.
-    """
-
-    @property
-    def expired(self):
-        """Returns `False`, anonymous credentials never expire."""
-        return False
-
-    @property
-    def valid(self):
-        """Returns `True`, anonymous credentials are always valid."""
-        return True
-
-    def refresh(self, request):
-        """Raises :class:`ValueError``, anonymous credentials cannot be
-        refreshed."""
-        raise ValueError("Anonymous credentials cannot be refreshed.")
-
-    def apply(self, headers, token=None):
-        """Anonymous credentials do nothing to the request.
-
-        The optional ``token`` argument is not supported.
-
-        Raises:
-            ValueError: If a token was specified.
-        """
-        if token is not None:
-            raise ValueError("Anonymous credentials don't support tokens.")
-
-    def before_request(self, request, method, url, headers):
-        """Anonymous credentials do nothing to the request."""
-
-
-@six.add_metaclass(abc.ABCMeta)
-class ReadOnlyScoped(object):
-    """Interface for credentials whose scopes can be queried.
-
-    OAuth 2.0-based credentials allow limiting access using scopes as described
-    in `RFC6749 Section 3.3`_.
-    If a credential class implements this interface then the credentials either
-    use scopes in their implementation.
-
-    Some credentials require scopes in order to obtain a token. You can check
-    if scoping is necessary with :attr:`requires_scopes`::
-
-        if credentials.requires_scopes:
-            # Scoping is required.
-            credentials = credentials.with_scopes(scopes=['one', 'two'])
-
-    Credentials that require scopes must either be constructed with scopes::
-
-        credentials = SomeScopedCredentials(scopes=['one', 'two'])
-
-    Or must copy an existing instance using :meth:`with_scopes`::
-
-        scoped_credentials = credentials.with_scopes(scopes=['one', 'two'])
-
-    Some credentials have scopes but do not allow or require scopes to be set,
-    these credentials can be used as-is.
-
-    .. _RFC6749 Section 3.3: https://tools.ietf.org/html/rfc6749#section-3.3
-    """
-    def __init__(self):
-        super(ReadOnlyScoped, self).__init__()
-        self._scopes = None
-
-    @property
-    def scopes(self):
-        """Sequence[str]: the credentials' current set of scopes."""
-        return self._scopes
-
-    @abc.abstractproperty
-    def requires_scopes(self):
-        """True if these credentials require scopes to obtain an access token.
-        """
-        return False
-
-    def has_scopes(self, scopes):
-        """Checks if the credentials have the given scopes.
-
-        .. warning: This method is not guaranteed to be accurate if the
-            credentials are :attr:`~Credentials.invalid`.
-
-        Args:
-            scopes (Sequence[str]): The list of scopes to check.
-
-        Returns:
-            bool: True if the credentials have the given scopes.
-        """
-        return set(scopes).issubset(set(self._scopes or []))
-
-
-class Scoped(ReadOnlyScoped):
-    """Interface for credentials whose scopes can be replaced while copying.
-
-    OAuth 2.0-based credentials allow limiting access using scopes as described
-    in `RFC6749 Section 3.3`_.
-    If a credential class implements this interface then the credentials either
-    use scopes in their implementation.
-
-    Some credentials require scopes in order to obtain a token. You can check
-    if scoping is necessary with :attr:`requires_scopes`::
-
-        if credentials.requires_scopes:
-            # Scoping is required.
-            credentials = credentials.create_scoped(['one', 'two'])
-
-    Credentials that require scopes must either be constructed with scopes::
-
-        credentials = SomeScopedCredentials(scopes=['one', 'two'])
-
-    Or must copy an existing instance using :meth:`with_scopes`::
-
-        scoped_credentials = credentials.with_scopes(scopes=['one', 'two'])
-
-    Some credentials have scopes but do not allow or require scopes to be set,
-    these credentials can be used as-is.
-
-    .. _RFC6749 Section 3.3: https://tools.ietf.org/html/rfc6749#section-3.3
-    """
-    @abc.abstractmethod
-    def with_scopes(self, scopes):
-        """Create a copy of these credentials with the specified scopes.
-
-        Args:
-            scopes (Sequence[str]): The list of scopes to attach to the
-                current credentials.
-
-        Raises:
-            NotImplementedError: If the credentials' scopes can not be changed.
-                This can be avoided by checking :attr:`requires_scopes` before
-                calling this method.
-        """
-        raise NotImplementedError('This class does not require scoping.')
-
-
-def with_scopes_if_required(credentials, scopes):
-    """Creates a copy of the credentials with scopes if scoping is required.
-
-    This helper function is useful when you do not know (or care to know) the
-    specific type of credentials you are using (such as when you use
-    :func:`google.auth.default`). This function will call
-    :meth:`Scoped.with_scopes` if the credentials are scoped credentials and if
-    the credentials require scoping. Otherwise, it will return the credentials
-    as-is.
-
-    Args:
-        credentials (google.auth.credentials.Credentials): The credentials to
-            scope if necessary.
-        scopes (Sequence[str]): The list of scopes to use.
-
-    Returns:
-        google.auth.credentials.Credentials: Either a new set of scoped
-            credentials, or the passed in credentials instance if no scoping
-            was required.
-    """
-    if isinstance(credentials, Scoped) and credentials.requires_scopes:
-        return credentials.with_scopes(scopes)
-    else:
-        return credentials
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Signing(object):
-    """Interface for credentials that can cryptographically sign messages."""
-
-    @abc.abstractmethod
-    def sign_bytes(self, message):
-        """Signs the given message.
-
-        Args:
-            message (bytes): The message to sign.
-
-        Returns:
-            bytes: The message's cryptographic signature.
-        """
-        # pylint: disable=missing-raises-doc,redundant-returns-doc
-        # (pylint doesn't recognize that this is abstract)
-        raise NotImplementedError('Sign bytes must be implemented.')
-
-    @abc.abstractproperty
-    def signer_email(self):
-        """Optional[str]: An email address that identifies the signer."""
-        # pylint: disable=missing-raises-doc
-        # (pylint doesn't recognize that this is abstract)
-        raise NotImplementedError('Signer email must be implemented.')
-
-    @abc.abstractproperty
-    def signer(self):
-        """google.auth.crypt.Signer: The signer used to sign bytes."""
-        # pylint: disable=missing-raises-doc
-        # (pylint doesn't recognize that this is abstract)
-        raise NotImplementedError('Signer must be implemented.')

src/google/auth/crypt/__init__.py

@@ -1,79 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Cryptography helpers for verifying and signing messages.
-
-The simplest way to verify signatures is using :func:`verify_signature`::
-
-    cert = open('certs.pem').read()
-    valid = crypt.verify_signature(message, signature, cert)
-
-If you're going to verify many messages with the same certificate, you can use
-:class:`RSAVerifier`::
-
-    cert = open('certs.pem').read()
-    verifier = crypt.RSAVerifier.from_string(cert)
-    valid = verifier.verify(message, signature)
-
-To sign messages use :class:`RSASigner` with a private key::
-
-    private_key = open('private_key.pem').read()
-    signer = crypt.RSASigner.from_string(private_key)
-    signature = signer.sign(message)
-"""
-
-import six
-
-from google.auth.crypt import base
-from google.auth.crypt import rsa
-
-
-__all__ = [
-    'RSASigner',
-    'RSAVerifier',
-    'Signer',
-    'Verifier',
-]
-
-# Aliases to maintain the v1.0.0 interface, as the crypt module was split
-# into submodules.
-Signer = base.Signer
-Verifier = base.Verifier
-RSASigner = rsa.RSASigner
-RSAVerifier = rsa.RSAVerifier
-
-
-def verify_signature(message, signature, certs):
-    """Verify an RSA cryptographic signature.
-
-    Checks that the provided ``signature`` was generated from ``bytes`` using
-    the private key associated with the ``cert``.
-
-    Args:
-        message (Union[str, bytes]): The plaintext message.
-        signature (Union[str, bytes]): The cryptographic signature to check.
-        certs (Union[Sequence, str, bytes]): The certificate or certificates
-            to use to check the signature.
-
-    Returns:
-        bool: True if the signature is valid, otherwise False.
-    """
-    if isinstance(certs, (six.text_type, six.binary_type)):
-        certs = [certs]
-
-    for cert in certs:
-        verifier = rsa.RSAVerifier.from_string(cert)
-        if verifier.verify(message, signature):
-            return True
-    return False

src/google/auth/crypt/_cryptography_rsa.py

@@ -1,149 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""RSA verifier and signer that use the ``cryptography`` library.
-
-This is a much faster implementation than the default (in
-``google.auth.crypt._python_rsa``), which depends on the pure-Python
-``rsa`` library.
-"""
-
-import cryptography.exceptions
-from cryptography.hazmat import backends
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric import padding
-import cryptography.x509
-import pkg_resources
-
-from google.auth import _helpers
-from google.auth.crypt import base
-
-_IMPORT_ERROR_MSG = (
-    'cryptography>=1.4.0 is required to use cryptography-based RSA '
-    'implementation.')
-
-try:  # pragma: NO COVER
-    release = pkg_resources.get_distribution('cryptography').parsed_version
-    if release < pkg_resources.parse_version('1.4.0'):
-        raise ImportError(_IMPORT_ERROR_MSG)
-except pkg_resources.DistributionNotFound:  # pragma: NO COVER
-    raise ImportError(_IMPORT_ERROR_MSG)
-
-
-_CERTIFICATE_MARKER = b'-----BEGIN CERTIFICATE-----'
-_BACKEND = backends.default_backend()
-_PADDING = padding.PKCS1v15()
-_SHA256 = hashes.SHA256()
-
-
-class RSAVerifier(base.Verifier):
-    """Verifies RSA cryptographic signatures using public keys.
-
-    Args:
-        public_key (
-                cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey):
-            The public key used to verify signatures.
-    """
-
-    def __init__(self, public_key):
-        self._pubkey = public_key
-
-    @_helpers.copy_docstring(base.Verifier)
-    def verify(self, message, signature):
-        message = _helpers.to_bytes(message)
-        try:
-            self._pubkey.verify(signature, message, _PADDING, _SHA256)
-            return True
-        except (ValueError, cryptography.exceptions.InvalidSignature):
-            return False
-
-    @classmethod
-    def from_string(cls, public_key):
-        """Construct an Verifier instance from a public key or public
-        certificate string.
-
-        Args:
-            public_key (Union[str, bytes]): The public key in PEM format or the
-                x509 public key certificate.
-
-        Returns:
-            Verifier: The constructed verifier.
-
-        Raises:
-            ValueError: If the public key can't be parsed.
-        """
-        public_key_data = _helpers.to_bytes(public_key)
-
-        if _CERTIFICATE_MARKER in public_key_data:
-            cert = cryptography.x509.load_pem_x509_certificate(
-                public_key_data, _BACKEND)
-            pubkey = cert.public_key()
-
-        else:
-            pubkey = serialization.load_pem_public_key(
-                public_key_data, _BACKEND)
-
-        return cls(pubkey)
-
-
-class RSASigner(base.Signer, base.FromServiceAccountMixin):
-    """Signs messages with an RSA private key.
-
-    Args:
-        private_key (
-                cryptography.hazmat.primitives.asymmetric.rsa.RSAPrivateKey):
-            The private key to sign with.
-        key_id (str): Optional key ID used to identify this private key. This
-            can be useful to associate the private key with its associated
-            public key or certificate.
-    """
-
-    def __init__(self, private_key, key_id=None):
-        self._key = private_key
-        self._key_id = key_id
-
-    @property
-    @_helpers.copy_docstring(base.Signer)
-    def key_id(self):
-        return self._key_id
-
-    @_helpers.copy_docstring(base.Signer)
-    def sign(self, message):
-        message = _helpers.to_bytes(message)
-        return self._key.sign(
-            message, _PADDING, _SHA256)
-
-    @classmethod
-    def from_string(cls, key, key_id=None):
-        """Construct a RSASigner from a private key in PEM format.
-
-        Args:
-            key (Union[bytes, str]): Private key in PEM format.
-            key_id (str): An optional key id used to identify the private key.
-
-        Returns:
-            google.auth.crypt._cryptography_rsa.RSASigner: The
-            constructed signer.
-
-        Raises:
-            ValueError: If ``key`` is not ``bytes`` or ``str`` (unicode).
-            UnicodeDecodeError: If ``key`` is ``bytes`` but cannot be decoded
-                into a UTF-8 ``str``.
-            ValueError: If ``cryptography`` "Could not deserialize key data."
-        """
-        key = _helpers.to_bytes(key)
-        private_key = serialization.load_pem_private_key(
-            key, password=None, backend=_BACKEND)
-        return cls(private_key, key_id=key_id)

src/google/auth/crypt/_python_rsa.py

@@ -1,176 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Pure-Python RSA cryptography implementation.
-
-Uses the ``rsa``, ``pyasn1`` and ``pyasn1_modules`` packages
-to parse PEM files storing PKCS#1 or PKCS#8 keys as well as
-certificates. There is no support for p12 files.
-"""
-
-from __future__ import absolute_import
-
-from pyasn1.codec.der import decoder
-from pyasn1_modules import pem
-from pyasn1_modules.rfc2459 import Certificate
-from pyasn1_modules.rfc5208 import PrivateKeyInfo
-import rsa
-import six
-
-from google.auth import _helpers
-from google.auth.crypt import base
-
-_POW2 = (128, 64, 32, 16, 8, 4, 2, 1)
-_CERTIFICATE_MARKER = b'-----BEGIN CERTIFICATE-----'
-_PKCS1_MARKER = ('-----BEGIN RSA PRIVATE KEY-----',
-                 '-----END RSA PRIVATE KEY-----')
-_PKCS8_MARKER = ('-----BEGIN PRIVATE KEY-----',
-                 '-----END PRIVATE KEY-----')
-_PKCS8_SPEC = PrivateKeyInfo()
-
-
-def _bit_list_to_bytes(bit_list):
-    """Converts an iterable of 1s and 0s to bytes.
-
-    Combines the list 8 at a time, treating each group of 8 bits
-    as a single byte.
-
-    Args:
-        bit_list (Sequence): Sequence of 1s and 0s.
-
-    Returns:
-        bytes: The decoded bytes.
-    """
-    num_bits = len(bit_list)
-    byte_vals = bytearray()
-    for start in six.moves.xrange(0, num_bits, 8):
-        curr_bits = bit_list[start:start + 8]
-        char_val = sum(
-            val * digit for val, digit in six.moves.zip(_POW2, curr_bits))
-        byte_vals.append(char_val)
-    return bytes(byte_vals)
-
-
-class RSAVerifier(base.Verifier):
-    """Verifies RSA cryptographic signatures using public keys.
-
-    Args:
-        public_key (rsa.key.PublicKey): The public key used to verify
-            signatures.
-    """
-
-    def __init__(self, public_key):
-        self._pubkey = public_key
-
-    @_helpers.copy_docstring(base.Verifier)
-    def verify(self, message, signature):
-        message = _helpers.to_bytes(message)
-        try:
-            return rsa.pkcs1.verify(message, signature, self._pubkey)
-        except (ValueError, rsa.pkcs1.VerificationError):
-            return False
-
-    @classmethod
-    def from_string(cls, public_key):
-        """Construct an Verifier instance from a public key or public
-        certificate string.
-
-        Args:
-            public_key (Union[str, bytes]): The public key in PEM format or the
-                x509 public key certificate.
-
-        Returns:
-            Verifier: The constructed verifier.
-
-        Raises:
-            ValueError: If the public_key can't be parsed.
-        """
-        public_key = _helpers.to_bytes(public_key)
-        is_x509_cert = _CERTIFICATE_MARKER in public_key
-
-        # If this is a certificate, extract the public key info.
-        if is_x509_cert:
-            der = rsa.pem.load_pem(public_key, 'CERTIFICATE')
-            asn1_cert, remaining = decoder.decode(der, asn1Spec=Certificate())
-            if remaining != b'':
-                raise ValueError('Unused bytes', remaining)
-
-            cert_info = asn1_cert['tbsCertificate']['subjectPublicKeyInfo']
-            key_bytes = _bit_list_to_bytes(cert_info['subjectPublicKey'])
-            pubkey = rsa.PublicKey.load_pkcs1(key_bytes, 'DER')
-        else:
-            pubkey = rsa.PublicKey.load_pkcs1(public_key, 'PEM')
-        return cls(pubkey)
-
-
-class RSASigner(base.Signer, base.FromServiceAccountMixin):
-    """Signs messages with an RSA private key.
-
-    Args:
-        private_key (rsa.key.PrivateKey): The private key to sign with.
-        key_id (str): Optional key ID used to identify this private key. This
-            can be useful to associate the private key with its associated
-            public key or certificate.
-    """
-
-    def __init__(self, private_key, key_id=None):
-        self._key = private_key
-        self._key_id = key_id
-
-    @property
-    @_helpers.copy_docstring(base.Signer)
-    def key_id(self):
-        return self._key_id
-
-    @_helpers.copy_docstring(base.Signer)
-    def sign(self, message):
-        message = _helpers.to_bytes(message)
-        return rsa.pkcs1.sign(message, self._key, 'SHA-256')
-
-    @classmethod
-    def from_string(cls, key, key_id=None):
-        """Construct an Signer instance from a private key in PEM format.
-
-        Args:
-            key (str): Private key in PEM format.
-            key_id (str): An optional key id used to identify the private key.
-
-        Returns:
-            google.auth.crypt.Signer: The constructed signer.
-
-        Raises:
-            ValueError: If the key cannot be parsed as PKCS#1 or PKCS#8 in
-                PEM format.
-        """
-        key = _helpers.from_bytes(key)  # PEM expects str in Python 3
-        marker_id, key_bytes = pem.readPemBlocksFromFile(
-            six.StringIO(key), _PKCS1_MARKER, _PKCS8_MARKER)
-
-        # Key is in pkcs1 format.
-        if marker_id == 0:
-            private_key = rsa.key.PrivateKey.load_pkcs1(
-                key_bytes, format='DER')
-        # Key is in pkcs8.
-        elif marker_id == 1:
-            key_info, remaining = decoder.decode(
-                key_bytes, asn1Spec=_PKCS8_SPEC)
-            if remaining != b'':
-                raise ValueError('Unused bytes', remaining)
-            private_key_info = key_info.getComponentByName('privateKey')
-            private_key = rsa.key.PrivateKey.load_pkcs1(
-                private_key_info.asOctets(), format='DER')
-        else:
-            raise ValueError('No key could be detected.')
-
-        return cls(private_key, key_id=key_id)

src/google/auth/crypt/base.py

@@ -1,131 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Base classes for cryptographic signers and verifiers."""
-
-import abc
-import io
-import json
-
-import six
-
-
-_JSON_FILE_PRIVATE_KEY = 'private_key'
-_JSON_FILE_PRIVATE_KEY_ID = 'private_key_id'
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Verifier(object):
-    """Abstract base class for crytographic signature verifiers."""
-
-    @abc.abstractmethod
-    def verify(self, message, signature):
-        """Verifies a message against a cryptographic signature.
-
-        Args:
-            message (Union[str, bytes]): The message to verify.
-            signature (Union[str, bytes]): The cryptography signature to check.
-
-        Returns:
-            bool: True if message was signed by the private key associated
-            with the public key that this object was constructed with.
-        """
-        # pylint: disable=missing-raises-doc,redundant-returns-doc
-        # (pylint doesn't recognize that this is abstract)
-        raise NotImplementedError('Verify must be implemented')
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Signer(object):
-    """Abstract base class for cryptographic signers."""
-
-    @abc.abstractproperty
-    def key_id(self):
-        """Optional[str]: The key ID used to identify this private key."""
-        raise NotImplementedError('Key id must be implemented')
-
-    @abc.abstractmethod
-    def sign(self, message):
-        """Signs a message.
-
-        Args:
-            message (Union[str, bytes]): The message to be signed.
-
-        Returns:
-            bytes: The signature of the message.
-        """
-        # pylint: disable=missing-raises-doc,redundant-returns-doc
-        # (pylint doesn't recognize that this is abstract)
-        raise NotImplementedError('Sign must be implemented')
-
-
-@six.add_metaclass(abc.ABCMeta)
-class FromServiceAccountMixin(object):
-    """Mix-in to enable factory constructors for a Signer."""
-
-    @abc.abstractmethod
-    def from_string(cls, key, key_id=None):
-        """Construct an Signer instance from a private key string.
-
-        Args:
-            key (str): Private key as a string.
-            key_id (str): An optional key id used to identify the private key.
-
-        Returns:
-            google.auth.crypt.Signer: The constructed signer.
-
-        Raises:
-            ValueError: If the key cannot be parsed.
-        """
-        raise NotImplementedError('from_string must be implemented')
-
-    @classmethod
-    def from_service_account_info(cls, info):
-        """Creates a Signer instance instance from a dictionary containing
-        service account info in Google format.
-
-        Args:
-            info (Mapping[str, str]): The service account info in Google
-                format.
-
-        Returns:
-            google.auth.crypt.Signer: The constructed signer.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        if _JSON_FILE_PRIVATE_KEY not in info:
-            raise ValueError(
-                'The private_key field was not found in the service account '
-                'info.')
-
-        return cls.from_string(
-            info[_JSON_FILE_PRIVATE_KEY],
-            info.get(_JSON_FILE_PRIVATE_KEY_ID))
-
-    @classmethod
-    def from_service_account_file(cls, filename):
-        """Creates a Signer instance from a service account .json file
-        in Google format.
-
-        Args:
-            filename (str): The path to the service account .json file.
-
-        Returns:
-            google.auth.crypt.Signer: The constructed signer.
-        """
-        with io.open(filename, 'r', encoding='utf-8') as json_file:
-            data = json.load(json_file)
-
-        return cls.from_service_account_info(data)

src/google/auth/crypt/rsa.py

@@ -1,30 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""RSA cryptography signer and verifier."""
-
-
-try:
-    # Prefer cryptograph-based RSA implementation.
-    from google.auth.crypt import _cryptography_rsa
-
-    RSASigner = _cryptography_rsa.RSASigner
-    RSAVerifier = _cryptography_rsa.RSAVerifier
-except ImportError:  # pragma: NO COVER
-    # Fallback to pure-python RSA implementation if cryptography is
-    # unavailable.
-    from google.auth.crypt import _python_rsa
-
-    RSASigner = _python_rsa.RSASigner
-    RSAVerifier = _python_rsa.RSAVerifier

src/google/auth/environment_vars.py

@@ -1,49 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Environment variables used by :mod:`google.auth`."""
-
-
-PROJECT = 'GOOGLE_CLOUD_PROJECT'
-"""Environment variable defining default project.
-
-This used by :func:`google.auth.default` to explicitly set a project ID. This
-environment variable is also used by the Google Cloud Python Library.
-"""
-
-LEGACY_PROJECT = 'GCLOUD_PROJECT'
-"""Previously used environment variable defining the default project.
-
-This environment variable is used instead of the current one in some
-situations (such as Google App Engine).
-"""
-
-CREDENTIALS = 'GOOGLE_APPLICATION_CREDENTIALS'
-"""Environment variable defining the location of Google application default
-credentials."""
-
-# The environment variable name which can replace ~/.config if set.
-CLOUD_SDK_CONFIG_DIR = 'CLOUDSDK_CONFIG'
-"""Environment variable defines the location of Google Cloud SDK's config
-files."""
-
-# These two variables allow for customization of the addresses used when
-# contacting the GCE metadata service.
-GCE_METADATA_ROOT = 'GCE_METADATA_ROOT'
-"""Environment variable providing an alternate hostname or host:port to be
-used for GCE metadata requests."""
-
-GCE_METADATA_IP = 'GCE_METADATA_IP'
-"""Environment variable providing an alternate ip:port to be used for ip-only
-GCE metadata requests."""

src/google/auth/exceptions.py

@@ -1,32 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Exceptions used in the google.auth package."""
-
-
-class GoogleAuthError(Exception):
-    """Base class for all google.auth errors."""
-
-
-class TransportError(GoogleAuthError):
-    """Used to indicate an error occurred during an HTTP request."""
-
-
-class RefreshError(GoogleAuthError):
-    """Used to indicate that an refreshing the credentials' access token
-    failed."""
-
-
-class DefaultCredentialsError(GoogleAuthError):
-    """Used to indicate that acquiring default credentials failed."""

src/google/auth/iam.py

@@ -1,102 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Tools for using the Google `Cloud Identity and Access Management (IAM)
-API`_'s auth-related functionality.
-
-.. _Cloud Identity and Access Management (IAM) API:
-    https://cloud.google.com/iam/docs/
-"""
-
-import base64
-import json
-
-from six.moves import http_client
-
-from google.auth import _helpers
-from google.auth import crypt
-from google.auth import exceptions
-
-_IAM_API_ROOT_URI = 'https://iam.googleapis.com/v1'
-_SIGN_BLOB_URI = (
-    _IAM_API_ROOT_URI + '/projects/-/serviceAccounts/{}:signBlob?alt=json')
-
-
-class Signer(crypt.Signer):
-    """Signs messages using the IAM `signBlob API`_.
-
-    This is useful when you need to sign bytes but do not have access to the
-    credential's private key file.
-
-    .. _signBlob API:
-        https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts
-        /signBlob
-    """
-
-    def __init__(self, request, credentials, service_account_email):
-        """
-        Args:
-            request (google.auth.transport.Request): The object used to make
-                HTTP requests.
-            credentials (google.auth.credentials.Credentials): The credentials
-                that will be used to authenticate the request to the IAM API.
-                The credentials must have of one the following scopes:
-
-                - https://www.googleapis.com/auth/iam
-                - https://www.googleapis.com/auth/cloud-platform
-            service_account_email (str): The service account email identifying
-                which service account to use to sign bytes. Often, this can
-                be the same as the service account email in the given
-                credentials.
-        """
-        self._request = request
-        self._credentials = credentials
-        self._service_account_email = service_account_email
-
-    def _make_signing_request(self, message):
-        """Makes a request to the API signBlob API."""
-        message = _helpers.to_bytes(message)
-
-        method = 'POST'
-        url = _SIGN_BLOB_URI.format(self._service_account_email)
-        headers = {}
-        body = json.dumps({
-            'bytesToSign': base64.b64encode(message).decode('utf-8'),
-        })
-
-        self._credentials.before_request(self._request, method, url, headers)
-        response = self._request(
-            url=url, method=method, body=body, headers=headers)
-
-        if response.status != http_client.OK:
-            raise exceptions.TransportError(
-                'Error calling the IAM signBytes API: {}'.format(
-                    response.data))
-
-        return json.loads(response.data.decode('utf-8'))
-
-    @property
-    def key_id(self):
-        """Optional[str]: The key ID used to identify this private key.
-
-        .. warning::
-           This is always ``None``. The key ID used by IAM can not
-           be reliably determined ahead of time.
-        """
-        return None
-
-    @_helpers.copy_docstring(crypt.Signer)
-    def sign(self, message):
-        response = self._make_signing_request(message)
-        return base64.b64decode(response['signature'])

src/google/auth/jwt.py

@@ -1,757 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""JSON Web Tokens
-
-Provides support for creating (encoding) and verifying (decoding) JWTs,
-especially JWTs generated and consumed by Google infrastructure.
-
-See `rfc7519`_ for more details on JWTs.
-
-To encode a JWT use :func:`encode`::
-
-    from google.auth import crypt
-    from google.auth import jwt
-
-    signer = crypt.Signer(private_key)
-    payload = {'some': 'payload'}
-    encoded = jwt.encode(signer, payload)
-
-To decode a JWT and verify claims use :func:`decode`::
-
-    claims = jwt.decode(encoded, certs=public_certs)
-
-You can also skip verification::
-
-    claims = jwt.decode(encoded, verify=False)
-
-.. _rfc7519: https://tools.ietf.org/html/rfc7519
-
-"""
-
-import base64
-import collections
-import copy
-import datetime
-import json
-
-import cachetools
-import six
-from six.moves import urllib
-
-from google.auth import _helpers
-from google.auth import _service_account_info
-from google.auth import crypt
-from google.auth import exceptions
-import google.auth.credentials
-
-_DEFAULT_TOKEN_LIFETIME_SECS = 3600  # 1 hour in seconds
-_DEFAULT_MAX_CACHE_SIZE = 10
-
-
-def encode(signer, payload, header=None, key_id=None):
-    """Make a signed JWT.
-
-    Args:
-        signer (google.auth.crypt.Signer): The signer used to sign the JWT.
-        payload (Mapping[str, str]): The JWT payload.
-        header (Mapping[str, str]): Additional JWT header payload.
-        key_id (str): The key id to add to the JWT header. If the
-            signer has a key id it will be used as the default. If this is
-            specified it will override the signer's key id.
-
-    Returns:
-        bytes: The encoded JWT.
-    """
-    if header is None:
-        header = {}
-
-    if key_id is None:
-        key_id = signer.key_id
-
-    header.update({'typ': 'JWT', 'alg': 'RS256'})
-
-    if key_id is not None:
-        header['kid'] = key_id
-
-    segments = [
-        base64.urlsafe_b64encode(json.dumps(header).encode('utf-8')),
-        base64.urlsafe_b64encode(json.dumps(payload).encode('utf-8')),
-    ]
-
-    signing_input = b'.'.join(segments)
-    signature = signer.sign(signing_input)
-    segments.append(base64.urlsafe_b64encode(signature))
-
-    return b'.'.join(segments)
-
-
-def _decode_jwt_segment(encoded_section):
-    """Decodes a single JWT segment."""
-    section_bytes = _helpers.padded_urlsafe_b64decode(encoded_section)
-    try:
-        return json.loads(section_bytes.decode('utf-8'))
-    except ValueError as caught_exc:
-        new_exc = ValueError('Can\'t parse segment: {0}'.format(section_bytes))
-        six.raise_from(new_exc, caught_exc)
-
-
-def _unverified_decode(token):
-    """Decodes a token and does no verification.
-
-    Args:
-        token (Union[str, bytes]): The encoded JWT.
-
-    Returns:
-        Tuple[str, str, str, str]: header, payload, signed_section, and
-            signature.
-
-    Raises:
-        ValueError: if there are an incorrect amount of segments in the token.
-    """
-    token = _helpers.to_bytes(token)
-
-    if token.count(b'.') != 2:
-        raise ValueError(
-            'Wrong number of segments in token: {0}'.format(token))
-
-    encoded_header, encoded_payload, signature = token.split(b'.')
-    signed_section = encoded_header + b'.' + encoded_payload
-    signature = _helpers.padded_urlsafe_b64decode(signature)
-
-    # Parse segments
-    header = _decode_jwt_segment(encoded_header)
-    payload = _decode_jwt_segment(encoded_payload)
-
-    return header, payload, signed_section, signature
-
-
-def decode_header(token):
-    """Return the decoded header of a token.
-
-    No verification is done. This is useful to extract the key id from
-    the header in order to acquire the appropriate certificate to verify
-    the token.
-
-    Args:
-        token (Union[str, bytes]): the encoded JWT.
-
-    Returns:
-        Mapping: The decoded JWT header.
-    """
-    header, _, _, _ = _unverified_decode(token)
-    return header
-
-
-def _verify_iat_and_exp(payload):
-    """Verifies the ``iat`` (Issued At) and ``exp`` (Expires) claims in a token
-    payload.
-
-    Args:
-        payload (Mapping[str, str]): The JWT payload.
-
-    Raises:
-        ValueError: if any checks failed.
-    """
-    now = _helpers.datetime_to_secs(_helpers.utcnow())
-
-    # Make sure the iat and exp claims are present.
-    for key in ('iat', 'exp'):
-        if key not in payload:
-            raise ValueError(
-                'Token does not contain required claim {}'.format(key))
-
-    # Make sure the token wasn't issued in the future.
-    iat = payload['iat']
-    # Err on the side of accepting a token that is slightly early to account
-    # for clock skew.
-    earliest = iat - _helpers.CLOCK_SKEW_SECS
-    if now < earliest:
-        raise ValueError('Token used too early, {} < {}'.format(now, iat))
-
-    # Make sure the token wasn't issued in the past.
-    exp = payload['exp']
-    # Err on the side of accepting a token that is slightly out of date
-    # to account for clow skew.
-    latest = exp + _helpers.CLOCK_SKEW_SECS
-    if latest < now:
-        raise ValueError('Token expired, {} < {}'.format(latest, now))
-
-
-def decode(token, certs=None, verify=True, audience=None):
-    """Decode and verify a JWT.
-
-    Args:
-        token (str): The encoded JWT.
-        certs (Union[str, bytes, Mapping[str, Union[str, bytes]]]): The
-            certificate used to validate the JWT signatyre. If bytes or string,
-            it must the the public key certificate in PEM format. If a mapping,
-            it must be a mapping of key IDs to public key certificates in PEM
-            format. The mapping must contain the same key ID that's specified
-            in the token's header.
-        verify (bool): Whether to perform signature and claim validation.
-            Verification is done by default.
-        audience (str): The audience claim, 'aud', that this JWT should
-            contain. If None then the JWT's 'aud' parameter is not verified.
-
-    Returns:
-        Mapping[str, str]: The deserialized JSON payload in the JWT.
-
-    Raises:
-        ValueError: if any verification checks failed.
-    """
-    header, payload, signed_section, signature = _unverified_decode(token)
-
-    if not verify:
-        return payload
-
-    # If certs is specified as a dictionary of key IDs to certificates, then
-    # use the certificate identified by the key ID in the token header.
-    if isinstance(certs, collections.Mapping):
-        key_id = header.get('kid')
-        if key_id:
-            if key_id not in certs:
-                raise ValueError(
-                    'Certificate for key id {} not found.'.format(key_id))
-            certs_to_check = [certs[key_id]]
-        # If there's no key id in the header, check against all of the certs.
-        else:
-            certs_to_check = certs.values()
-    else:
-        certs_to_check = certs
-
-    # Verify that the signature matches the message.
-    if not crypt.verify_signature(signed_section, signature, certs_to_check):
-        raise ValueError('Could not verify token signature.')
-
-    # Verify the issued at and created times in the payload.
-    _verify_iat_and_exp(payload)
-
-    # Check audience.
-    if audience is not None:
-        claim_audience = payload.get('aud')
-        if audience != claim_audience:
-            raise ValueError(
-                'Token has wrong audience {}, expected {}'.format(
-                    claim_audience, audience))
-
-    return payload
-
-
-class Credentials(google.auth.credentials.Signing,
-                  google.auth.credentials.Credentials):
-    """Credentials that use a JWT as the bearer token.
-
-    These credentials require an "audience" claim. This claim identifies the
-    intended recipient of the bearer token.
-
-    The constructor arguments determine the claims for the JWT that is
-    sent with requests. Usually, you'll construct these credentials with
-    one of the helper constructors as shown in the next section.
-
-    To create JWT credentials using a Google service account private key
-    JSON file::
-
-        audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher'
-        credentials = jwt.Credentials.from_service_account_file(
-            'service-account.json',
-            audience=audience)
-
-    If you already have the service account file loaded and parsed::
-
-        service_account_info = json.load(open('service_account.json'))
-        credentials = jwt.Credentials.from_service_account_info(
-            service_account_info,
-            audience=audience)
-
-    Both helper methods pass on arguments to the constructor, so you can
-    specify the JWT claims::
-
-        credentials = jwt.Credentials.from_service_account_file(
-            'service-account.json',
-            audience=audience,
-            additional_claims={'meta': 'data'})
-
-    You can also construct the credentials directly if you have a
-    :class:`~google.auth.crypt.Signer` instance::
-
-        credentials = jwt.Credentials(
-            signer,
-            issuer='your-issuer',
-            subject='your-subject',
-            audience=audience)
-
-    The claims are considered immutable. If you want to modify the claims,
-    you can easily create another instance using :meth:`with_claims`::
-
-        new_audience = (
-            'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber')
-        new_credentials = credentials.with_claims(audience=new_audience)
-    """
-
-    def __init__(self, signer, issuer, subject, audience,
-                 additional_claims=None,
-                 token_lifetime=_DEFAULT_TOKEN_LIFETIME_SECS):
-        """
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            issuer (str): The `iss` claim.
-            subject (str): The `sub` claim.
-            audience (str): the `aud` claim. The intended audience for the
-                credentials.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT payload.
-            token_lifetime (int): The amount of time in seconds for
-                which the token is valid. Defaults to 1 hour.
-        """
-        super(Credentials, self).__init__()
-        self._signer = signer
-        self._issuer = issuer
-        self._subject = subject
-        self._audience = audience
-        self._token_lifetime = token_lifetime
-
-        if additional_claims is None:
-            additional_claims = {}
-
-        self._additional_claims = additional_claims
-
-    @classmethod
-    def _from_signer_and_info(cls, signer, info, **kwargs):
-        """Creates a Credentials instance from a signer and service account
-        info.
-
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            info (Mapping[str, str]): The service account info.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.Credentials: The constructed credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        kwargs.setdefault('subject', info['client_email'])
-        kwargs.setdefault('issuer', info['client_email'])
-        return cls(signer, **kwargs)
-
-    @classmethod
-    def from_service_account_info(cls, info, **kwargs):
-        """Creates an Credentials instance from a dictionary.
-
-        Args:
-            info (Mapping[str, str]): The service account info in Google
-                format.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.Credentials: The constructed credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        signer = _service_account_info.from_dict(
-            info, require=['client_email'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @classmethod
-    def from_service_account_file(cls, filename, **kwargs):
-        """Creates a Credentials instance from a service account .json file
-        in Google format.
-
-        Args:
-            filename (str): The path to the service account .json file.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.Credentials: The constructed credentials.
-        """
-        info, signer = _service_account_info.from_filename(
-            filename, require=['client_email'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @classmethod
-    def from_signing_credentials(cls, credentials, audience, **kwargs):
-        """Creates a new :class:`google.auth.jwt.Credentials` instance from an
-        existing :class:`google.auth.credentials.Signing` instance.
-
-        The new instance will use the same signer as the existing instance and
-        will use the existing instance's signer email as the issuer and
-        subject by default.
-
-        Example::
-
-            svc_creds = service_account.Credentials.from_service_account_file(
-                'service_account.json')
-            audience = (
-                'https://pubsub.googleapis.com/google.pubsub.v1.Publisher')
-            jwt_creds = jwt.Credentials.from_signing_credentials(
-                svc_creds, audience=audience)
-
-        Args:
-            credentials (google.auth.credentials.Signing): The credentials to
-                use to construct the new credentials.
-            audience (str): the `aud` claim. The intended audience for the
-                credentials.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.Credentials: A new Credentials instance.
-        """
-        kwargs.setdefault('issuer', credentials.signer_email)
-        kwargs.setdefault('subject', credentials.signer_email)
-        return cls(
-            credentials.signer,
-            audience=audience,
-            **kwargs)
-
-    def with_claims(self, issuer=None, subject=None, audience=None,
-                    additional_claims=None):
-        """Returns a copy of these credentials with modified claims.
-
-        Args:
-            issuer (str): The `iss` claim. If unspecified the current issuer
-                claim will be used.
-            subject (str): The `sub` claim. If unspecified the current subject
-                claim will be used.
-            audience (str): the `aud` claim. If unspecified the current
-                audience claim will be used.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT payload. This will be merged with the current
-                additional claims.
-
-        Returns:
-            google.auth.jwt.Credentials: A new credentials instance.
-        """
-        new_additional_claims = copy.deepcopy(self._additional_claims)
-        new_additional_claims.update(additional_claims or {})
-
-        return self.__class__(
-            self._signer,
-            issuer=issuer if issuer is not None else self._issuer,
-            subject=subject if subject is not None else self._subject,
-            audience=audience if audience is not None else self._audience,
-            additional_claims=new_additional_claims)
-
-    def _make_jwt(self):
-        """Make a signed JWT.
-
-        Returns:
-            Tuple[bytes, datetime]: The encoded JWT and the expiration.
-        """
-        now = _helpers.utcnow()
-        lifetime = datetime.timedelta(seconds=self._token_lifetime)
-        expiry = now + lifetime
-
-        payload = {
-            'iss': self._issuer,
-            'sub': self._subject,
-            'iat': _helpers.datetime_to_secs(now),
-            'exp': _helpers.datetime_to_secs(expiry),
-            'aud': self._audience,
-        }
-
-        payload.update(self._additional_claims)
-
-        jwt = encode(self._signer, payload)
-
-        return jwt, expiry
-
-    def refresh(self, request):
-        """Refreshes the access token.
-
-        Args:
-            request (Any): Unused.
-        """
-        # pylint: disable=unused-argument
-        # (pylint doesn't correctly recognize overridden methods.)
-        self.token, self.expiry = self._make_jwt()
-
-    @_helpers.copy_docstring(google.auth.credentials.Signing)
-    def sign_bytes(self, message):
-        return self._signer.sign(message)
-
-    @property
-    @_helpers.copy_docstring(google.auth.credentials.Signing)
-    def signer_email(self):
-        return self._issuer
-
-    @property
-    @_helpers.copy_docstring(google.auth.credentials.Signing)
-    def signer(self):
-        return self._signer
-
-
-class OnDemandCredentials(
-        google.auth.credentials.Signing,
-        google.auth.credentials.Credentials):
-    """On-demand JWT credentials.
-
-    Like :class:`Credentials`, this class uses a JWT as the bearer token for
-    authentication. However, this class does not require the audience at
-    construction time. Instead, it will generate a new token on-demand for
-    each request using the request URI as the audience. It caches tokens
-    so that multiple requests to the same URI do not incur the overhead
-    of generating a new token every time.
-
-    This behavior is especially useful for `gRPC`_ clients. A gRPC service may
-    have multiple audience and gRPC clients may not know all of the audiences
-    required for accessing a particular service. With these credentials,
-    no knowledge of the audiences is required ahead of time.
-
-    .. _grpc: http://www.grpc.io/
-    """
-
-    def __init__(self, signer, issuer, subject,
-                 additional_claims=None,
-                 token_lifetime=_DEFAULT_TOKEN_LIFETIME_SECS,
-                 max_cache_size=_DEFAULT_MAX_CACHE_SIZE):
-        """
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            issuer (str): The `iss` claim.
-            subject (str): The `sub` claim.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT payload.
-            token_lifetime (int): The amount of time in seconds for
-                which the token is valid. Defaults to 1 hour.
-            max_cache_size (int): The maximum number of JWT tokens to keep in
-                cache. Tokens are cached using :class:`cachetools.LRUCache`.
-        """
-        super(OnDemandCredentials, self).__init__()
-        self._signer = signer
-        self._issuer = issuer
-        self._subject = subject
-        self._token_lifetime = token_lifetime
-
-        if additional_claims is None:
-            additional_claims = {}
-
-        self._additional_claims = additional_claims
-        self._cache = cachetools.LRUCache(maxsize=max_cache_size)
-
-    @classmethod
-    def _from_signer_and_info(cls, signer, info, **kwargs):
-        """Creates an OnDemandCredentials instance from a signer and service
-        account info.
-
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            info (Mapping[str, str]): The service account info.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.OnDemandCredentials: The constructed credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        kwargs.setdefault('subject', info['client_email'])
-        kwargs.setdefault('issuer', info['client_email'])
-        return cls(signer, **kwargs)
-
-    @classmethod
-    def from_service_account_info(cls, info, **kwargs):
-        """Creates an OnDemandCredentials instance from a dictionary.
-
-        Args:
-            info (Mapping[str, str]): The service account info in Google
-                format.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.OnDemandCredentials: The constructed credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        signer = _service_account_info.from_dict(
-            info, require=['client_email'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @classmethod
-    def from_service_account_file(cls, filename, **kwargs):
-        """Creates an OnDemandCredentials instance from a service account .json
-        file in Google format.
-
-        Args:
-            filename (str): The path to the service account .json file.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.OnDemandCredentials: The constructed credentials.
-        """
-        info, signer = _service_account_info.from_filename(
-            filename, require=['client_email'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @classmethod
-    def from_signing_credentials(cls, credentials, **kwargs):
-        """Creates a new :class:`google.auth.jwt.OnDemandCredentials` instance
-        from an existing :class:`google.auth.credentials.Signing` instance.
-
-        The new instance will use the same signer as the existing instance and
-        will use the existing instance's signer email as the issuer and
-        subject by default.
-
-        Example::
-
-            svc_creds = service_account.Credentials.from_service_account_file(
-                'service_account.json')
-            jwt_creds = jwt.OnDemandCredentials.from_signing_credentials(
-                svc_creds)
-
-        Args:
-            credentials (google.auth.credentials.Signing): The credentials to
-                use to construct the new credentials.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.Credentials: A new Credentials instance.
-        """
-        kwargs.setdefault('issuer', credentials.signer_email)
-        kwargs.setdefault('subject', credentials.signer_email)
-        return cls(credentials.signer, **kwargs)
-
-    def with_claims(self, issuer=None, subject=None, additional_claims=None):
-        """Returns a copy of these credentials with modified claims.
-
-        Args:
-            issuer (str): The `iss` claim. If unspecified the current issuer
-                claim will be used.
-            subject (str): The `sub` claim. If unspecified the current subject
-                claim will be used.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT payload. This will be merged with the current
-                additional claims.
-
-        Returns:
-            google.auth.jwt.OnDemandCredentials: A new credentials instance.
-        """
-        new_additional_claims = copy.deepcopy(self._additional_claims)
-        new_additional_claims.update(additional_claims or {})
-
-        return self.__class__(
-            self._signer,
-            issuer=issuer if issuer is not None else self._issuer,
-            subject=subject if subject is not None else self._subject,
-            additional_claims=new_additional_claims,
-            max_cache_size=self._cache.maxsize)
-
-    @property
-    def valid(self):
-        """Checks the validity of the credentials.
-
-        These credentials are always valid because it generates tokens on
-        demand.
-        """
-        return True
-
-    def _make_jwt_for_audience(self, audience):
-        """Make a new JWT for the given audience.
-
-        Args:
-            audience (str): The intended audience.
-
-        Returns:
-            Tuple[bytes, datetime]: The encoded JWT and the expiration.
-        """
-        now = _helpers.utcnow()
-        lifetime = datetime.timedelta(seconds=self._token_lifetime)
-        expiry = now + lifetime
-
-        payload = {
-            'iss': self._issuer,
-            'sub': self._subject,
-            'iat': _helpers.datetime_to_secs(now),
-            'exp': _helpers.datetime_to_secs(expiry),
-            'aud': audience,
-        }
-
-        payload.update(self._additional_claims)
-
-        jwt = encode(self._signer, payload)
-
-        return jwt, expiry
-
-    def _get_jwt_for_audience(self, audience):
-        """Get a JWT For a given audience.
-
-        If there is already an existing, non-expired token in the cache for
-        the audience, that token is used. Otherwise, a new token will be
-        created.
-
-        Args:
-            audience (str): The intended audience.
-
-        Returns:
-            bytes: The encoded JWT.
-        """
-        token, expiry = self._cache.get(audience, (None, None))
-
-        if token is None or expiry < _helpers.utcnow():
-            token, expiry = self._make_jwt_for_audience(audience)
-            self._cache[audience] = token, expiry
-
-        return token
-
-    def refresh(self, request):
-        """Raises an exception, these credentials can not be directly
-        refreshed.
-
-        Args:
-            request (Any): Unused.
-
-        Raises:
-            google.auth.RefreshError
-        """
-        # pylint: disable=unused-argument
-        # (pylint doesn't correctly recognize overridden methods.)
-        raise exceptions.RefreshError(
-            'OnDemandCredentials can not be directly refreshed.')
-
-    def before_request(self, request, method, url, headers):
-        """Performs credential-specific before request logic.
-
-        Args:
-            request (Any): Unused. JWT credentials do not need to make an
-                HTTP request to refresh.
-            method (str): The request's HTTP method.
-            url (str): The request's URI. This is used as the audience claim
-                when generating the JWT.
-            headers (Mapping): The request's headers.
-        """
-        # pylint: disable=unused-argument
-        # (pylint doesn't correctly recognize overridden methods.)
-        parts = urllib.parse.urlsplit(url)
-        # Strip query string and fragment
-        audience = urllib.parse.urlunsplit(
-            (parts.scheme, parts.netloc, parts.path, None, None))
-        token = self._get_jwt_for_audience(audience)
-        self.apply(headers, token=token)
-
-    @_helpers.copy_docstring(google.auth.credentials.Signing)
-    def sign_bytes(self, message):
-        return self._signer.sign(message)
-
-    @property
-    @_helpers.copy_docstring(google.auth.credentials.Signing)
-    def signer_email(self):
-        return self._issuer
-
-    @property
-    @_helpers.copy_docstring(google.auth.credentials.Signing)
-    def signer(self):
-        return self._signer

src/google/auth/transport/__init__.py

@@ -1,96 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Transport - HTTP client library support.
-
-:mod:`google.auth` is designed to work with various HTTP client libraries such
-as urllib3 and requests. In order to work across these libraries with different
-interfaces some abstraction is needed.
-
-This module provides two interfaces that are implemented by transport adapters
-to support HTTP libraries. :class:`Request` defines the interface expected by
-:mod:`google.auth` to make requests. :class:`Response` defines the interface
-for the return value of :class:`Request`.
-"""
-
-import abc
-
-import six
-from six.moves import http_client
-
-DEFAULT_REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,)
-"""Sequence[int]:  Which HTTP status code indicate that credentials should be
-refreshed and a request should be retried.
-"""
-
-DEFAULT_MAX_REFRESH_ATTEMPTS = 2
-"""int: How many times to refresh the credentials and retry a request."""
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Response(object):
-    """HTTP Response data."""
-
-    @abc.abstractproperty
-    def status(self):
-        """int: The HTTP status code."""
-        raise NotImplementedError('status must be implemented.')
-
-    @abc.abstractproperty
-    def headers(self):
-        """Mapping[str, str]: The HTTP response headers."""
-        raise NotImplementedError('headers must be implemented.')
-
-    @abc.abstractproperty
-    def data(self):
-        """bytes: The response body."""
-        raise NotImplementedError('data must be implemented.')
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Request(object):
-    """Interface for a callable that makes HTTP requests.
-
-    Specific transport implementations should provide an implementation of
-    this that adapts their specific request / response API.
-
-    .. automethod:: __call__
-    """
-
-    @abc.abstractmethod
-    def __call__(self, url, method='GET', body=None, headers=None,
-                 timeout=None, **kwargs):
-        """Make an HTTP request.
-
-        Args:
-            url (str): The URI to be requested.
-            method (str): The HTTP method to use for the request. Defaults
-                to 'GET'.
-            body (bytes): The payload / body in HTTP request.
-            headers (Mapping[str, str]): Request headers.
-            timeout (Optional[int]): The number of seconds to wait for a
-                response from the server. If not specified or if None, the
-                transport-specific default timeout will be used.
-            kwargs: Additionally arguments passed on to the transport's
-                request method.
-
-        Returns:
-            Response: The HTTP response.
-
-        Raises:
-            google.auth.exceptions.TransportError: If any exception occurred.
-        """
-        # pylint: disable=redundant-returns-doc, missing-raises-doc
-        # (pylint doesn't play well with abstract docstrings.)
-        raise NotImplementedError('__call__ must be implemented.')

src/google/auth/transport/_http_client.py

@@ -1,113 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Transport adapter for http.client, for internal use only."""
-
-import logging
-import socket
-
-import six
-from six.moves import http_client
-from six.moves import urllib
-
-from google.auth import exceptions
-from google.auth import transport
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class Response(transport.Response):
-    """http.client transport response adapter.
-
-    Args:
-        response (http.client.HTTPResponse): The raw http client response.
-    """
-    def __init__(self, response):
-        self._status = response.status
-        self._headers = {
-            key.lower(): value for key, value in response.getheaders()}
-        self._data = response.read()
-
-    @property
-    def status(self):
-        return self._status
-
-    @property
-    def headers(self):
-        return self._headers
-
-    @property
-    def data(self):
-        return self._data
-
-
-class Request(transport.Request):
-    """http.client transport request adapter."""
-
-    def __call__(self, url, method='GET', body=None, headers=None,
-                 timeout=None, **kwargs):
-        """Make an HTTP request using http.client.
-
-        Args:
-            url (str): The URI to be requested.
-            method (str): The HTTP method to use for the request. Defaults
-                to 'GET'.
-            body (bytes): The payload / body in HTTP request.
-            headers (Mapping): Request headers.
-            timeout (Optional(int)): The number of seconds to wait for a
-                response from the server. If not specified or if None, the
-                socket global default timeout will be used.
-            kwargs: Additional arguments passed throught to the underlying
-                :meth:`~http.client.HTTPConnection.request` method.
-
-        Returns:
-            Response: The HTTP response.
-
-        Raises:
-            google.auth.exceptions.TransportError: If any exception occurred.
-        """
-        # socket._GLOBAL_DEFAULT_TIMEOUT is the default in http.client.
-        if timeout is None:
-            timeout = socket._GLOBAL_DEFAULT_TIMEOUT
-
-        # http.client doesn't allow None as the headers argument.
-        if headers is None:
-            headers = {}
-
-        # http.client needs the host and path parts specified separately.
-        parts = urllib.parse.urlsplit(url)
-        path = urllib.parse.urlunsplit(
-            ('', '', parts.path, parts.query, parts.fragment))
-
-        if parts.scheme != 'http':
-            raise exceptions.TransportError(
-                'http.client transport only supports the http scheme, {}'
-                'was specified'.format(parts.scheme))
-
-        connection = http_client.HTTPConnection(parts.netloc, timeout=timeout)
-
-        try:
-            _LOGGER.debug('Making request: %s %s', method, url)
-
-            connection.request(
-                method, path, body=body, headers=headers, **kwargs)
-            response = connection.getresponse()
-            return Response(response)
-
-        except (http_client.HTTPException, socket.error) as caught_exc:
-            new_exc = exceptions.TransportError(caught_exc)
-            six.raise_from(new_exc, caught_exc)
-
-        finally:
-            connection.close()

src/google/auth/transport/grpc.py

@@ -1,135 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Authorization support for gRPC."""
-
-from __future__ import absolute_import
-
-import six
-try:
-    import grpc
-except ImportError as caught_exc:  # pragma: NO COVER
-    six.raise_from(
-        ImportError(
-            'gRPC is not installed, please install the grpcio package '
-            'to use the gRPC transport.'
-        ),
-        caught_exc,
-    )
-
-
-class AuthMetadataPlugin(grpc.AuthMetadataPlugin):
-    """A `gRPC AuthMetadataPlugin`_ that inserts the credentials into each
-    request.
-
-    .. _gRPC AuthMetadataPlugin:
-        http://www.grpc.io/grpc/python/grpc.html#grpc.AuthMetadataPlugin
-
-    Args:
-        credentials (google.auth.credentials.Credentials): The credentials to
-            add to requests.
-        request (google.auth.transport.Request): A HTTP transport request
-            object used to refresh credentials as needed.
-    """
-    def __init__(self, credentials, request):
-        # pylint: disable=no-value-for-parameter
-        # pylint doesn't realize that the super method takes no arguments
-        # because this class is the same name as the superclass.
-        super(AuthMetadataPlugin, self).__init__()
-        self._credentials = credentials
-        self._request = request
-
-    def _get_authorization_headers(self, context):
-        """Gets the authorization headers for a request.
-
-        Returns:
-            Sequence[Tuple[str, str]]: A list of request headers (key, value)
-                to add to the request.
-        """
-        headers = {}
-        self._credentials.before_request(
-            self._request,
-            context.method_name,
-            context.service_url,
-            headers)
-
-        return list(six.iteritems(headers))
-
-    def __call__(self, context, callback):
-        """Passes authorization metadata into the given callback.
-
-        Args:
-            context (grpc.AuthMetadataContext): The RPC context.
-            callback (grpc.AuthMetadataPluginCallback): The callback that will
-                be invoked to pass in the authorization metadata.
-        """
-        callback(self._get_authorization_headers(context), None)
-
-
-def secure_authorized_channel(
-        credentials, request, target, ssl_credentials=None, **kwargs):
-    """Creates a secure authorized gRPC channel.
-
-    This creates a channel with SSL and :class:`AuthMetadataPlugin`. This
-    channel can be used to create a stub that can make authorized requests.
-
-    Example::
-
-        import google.auth
-        import google.auth.transport.grpc
-        import google.auth.transport.requests
-        from google.cloud.speech.v1 import cloud_speech_pb2
-
-        # Get credentials.
-        credentials, _ = google.auth.default()
-
-        # Get an HTTP request function to refresh credentials.
-        request = google.auth.transport.requests.Request()
-
-        # Create a channel.
-        channel = google.auth.transport.grpc.secure_authorized_channel(
-            credentials, 'speech.googleapis.com:443', request)
-
-        # Use the channel to create a stub.
-        cloud_speech.create_Speech_stub(channel)
-
-    Args:
-        credentials (google.auth.credentials.Credentials): The credentials to
-            add to requests.
-        request (google.auth.transport.Request): A HTTP transport request
-            object used to refresh credentials as needed. Even though gRPC
-            is a separate transport, there's no way to refresh the credentials
-            without using a standard http transport.
-        target (str): The host and port of the service.
-        ssl_credentials (grpc.ChannelCredentials): Optional SSL channel
-            credentials. This can be used to specify different certificates.
-        kwargs: Additional arguments to pass to :func:`grpc.secure_channel`.
-
-    Returns:
-        grpc.Channel: The created gRPC channel.
-    """
-    # Create the metadata plugin for inserting the authorization header.
-    metadata_plugin = AuthMetadataPlugin(credentials, request)
-
-    # Create a set of grpc.CallCredentials using the metadata plugin.
-    google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin)
-
-    if ssl_credentials is None:
-        ssl_credentials = grpc.ssl_channel_credentials()
-
-    # Combine the ssl credentials and the authorization credentials.
-    composite_credentials = grpc.composite_channel_credentials(
-        ssl_credentials, google_auth_credentials)
-
-    return grpc.secure_channel(target, composite_credentials, **kwargs)

src/google/auth/transport/requests.py

@@ -1,226 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Transport adapter for Requests."""
-
-from __future__ import absolute_import
-
-import functools
-import logging
-
-try:
-    import requests
-except ImportError as caught_exc:  # pragma: NO COVER
-    import six
-    six.raise_from(
-        ImportError(
-            'The requests library is not installed, please install the '
-            'requests package to use the requests transport.'
-        ),
-        caught_exc,
-    )
-import requests.adapters  # pylint: disable=ungrouped-imports
-import requests.exceptions  # pylint: disable=ungrouped-imports
-import six  # pylint: disable=ungrouped-imports
-
-from google.auth import exceptions
-from google.auth import transport
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class _Response(transport.Response):
-    """Requests transport response adapter.
-
-    Args:
-        response (requests.Response): The raw Requests response.
-    """
-    def __init__(self, response):
-        self._response = response
-
-    @property
-    def status(self):
-        return self._response.status_code
-
-    @property
-    def headers(self):
-        return self._response.headers
-
-    @property
-    def data(self):
-        return self._response.content
-
-
-class Request(transport.Request):
-    """Requests request adapter.
-
-    This class is used internally for making requests using various transports
-    in a consistent way. If you use :class:`AuthorizedSession` you do not need
-    to construct or use this class directly.
-
-    This class can be useful if you want to manually refresh a
-    :class:`~google.auth.credentials.Credentials` instance::
-
-        import google.auth.transport.requests
-        import requests
-
-        request = google.auth.transport.requests.Request()
-
-        credentials.refresh(request)
-
-    Args:
-        session (requests.Session): An instance :class:`requests.Session` used
-            to make HTTP requests. If not specified, a session will be created.
-
-    .. automethod:: __call__
-    """
-    def __init__(self, session=None):
-        if not session:
-            session = requests.Session()
-
-        self.session = session
-
-    def __call__(self, url, method='GET', body=None, headers=None,
-                 timeout=None, **kwargs):
-        """Make an HTTP request using requests.
-
-        Args:
-            url (str): The URI to be requested.
-            method (str): The HTTP method to use for the request. Defaults
-                to 'GET'.
-            body (bytes): The payload / body in HTTP request.
-            headers (Mapping[str, str]): Request headers.
-            timeout (Optional[int]): The number of seconds to wait for a
-                response from the server. If not specified or if None, the
-                requests default timeout will be used.
-            kwargs: Additional arguments passed through to the underlying
-                requests :meth:`~requests.Session.request` method.
-
-        Returns:
-            google.auth.transport.Response: The HTTP response.
-
-        Raises:
-            google.auth.exceptions.TransportError: If any exception occurred.
-        """
-        try:
-            _LOGGER.debug('Making request: %s %s', method, url)
-            response = self.session.request(
-                method, url, data=body, headers=headers, timeout=timeout,
-                **kwargs)
-            return _Response(response)
-        except requests.exceptions.RequestException as caught_exc:
-            new_exc = exceptions.TransportError(caught_exc)
-            six.raise_from(new_exc, caught_exc)
-
-
-class AuthorizedSession(requests.Session):
-    """A Requests Session class with credentials.
-
-    This class is used to perform requests to API endpoints that require
-    authorization::
-
-        from google.auth.transport.requests import AuthorizedSession
-
-        authed_session = AuthorizedSession(credentials)
-
-        response = authed_session.request(
-            'GET', 'https://www.googleapis.com/storage/v1/b')
-
-    The underlying :meth:`request` implementation handles adding the
-    credentials' headers to the request and refreshing credentials as needed.
-
-    Args:
-        credentials (google.auth.credentials.Credentials): The credentials to
-            add to the request.
-        refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
-            that credentials should be refreshed and the request should be
-            retried.
-        max_refresh_attempts (int): The maximum number of times to attempt to
-            refresh the credentials and retry the request.
-        refresh_timeout (Optional[int]): The timeout value in seconds for
-            credential refresh HTTP requests.
-        kwargs: Additional arguments passed to the :class:`requests.Session`
-            constructor.
-    """
-    def __init__(self, credentials,
-                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
-                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
-                 refresh_timeout=None,
-                 **kwargs):
-        super(AuthorizedSession, self).__init__(**kwargs)
-        self.credentials = credentials
-        self._refresh_status_codes = refresh_status_codes
-        self._max_refresh_attempts = max_refresh_attempts
-        self._refresh_timeout = refresh_timeout
-
-        auth_request_session = requests.Session()
-
-        # Using an adapter to make HTTP requests robust to network errors.
-        # This adapter retrys HTTP requests when network errors occur
-        # and the requests seems safely retryable.
-        retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
-        auth_request_session.mount("https://", retry_adapter)
-
-        # Request instance used by internal methods (for example,
-        # credentials.refresh).
-        # Do not pass `self` as the session here, as it can lead to infinite
-        # recursion.
-        self._auth_request = Request(auth_request_session)
-
-    def request(self, method, url, data=None, headers=None, **kwargs):
-        """Implementation of Requests' request."""
-        # pylint: disable=arguments-differ
-        # Requests has a ton of arguments to request, but only two
-        # (method, url) are required. We pass through all of the other
-        # arguments to super, so no need to exhaustively list them here.
-
-        # Use a kwarg for this instead of an attribute to maintain
-        # thread-safety.
-        _credential_refresh_attempt = kwargs.pop(
-            '_credential_refresh_attempt', 0)
-
-        # Make a copy of the headers. They will be modified by the credentials
-        # and we want to pass the original headers if we recurse.
-        request_headers = headers.copy() if headers is not None else {}
-
-        self.credentials.before_request(
-            self._auth_request, method, url, request_headers)
-
-        response = super(AuthorizedSession, self).request(
-            method, url, data=data, headers=request_headers, **kwargs)
-
-        # If the response indicated that the credentials needed to be
-        # refreshed, then refresh the credentials and re-attempt the
-        # request.
-        # A stored token may expire between the time it is retrieved and
-        # the time the request is made, so we may need to try twice.
-        if (response.status_code in self._refresh_status_codes
-                and _credential_refresh_attempt < self._max_refresh_attempts):
-
-            _LOGGER.info(
-                'Refreshing credentials due to a %s response. Attempt %s/%s.',
-                response.status_code, _credential_refresh_attempt + 1,
-                self._max_refresh_attempts)
-
-            auth_request_with_timeout = functools.partial(
-                self._auth_request, timeout=self._refresh_timeout)
-            self.credentials.refresh(auth_request_with_timeout)
-
-            # Recurse. Pass in the original headers, not our modified set.
-            return self.request(
-                method, url, data=data, headers=headers,
-                _credential_refresh_attempt=_credential_refresh_attempt + 1,
-                **kwargs)
-
-        return response

src/google/auth/transport/urllib3.py

@@ -1,266 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Transport adapter for urllib3."""
-
-from __future__ import absolute_import
-
-import logging
-
-
-# Certifi is Mozilla's certificate bundle. Urllib3 needs a certificate bundle
-# to verify HTTPS requests, and certifi is the recommended and most reliable
-# way to get a root certificate bundle. See
-# http://urllib3.readthedocs.io/en/latest/user-guide.html\
-#   #certificate-verification
-# For more details.
-try:
-    import certifi
-except ImportError:  # pragma: NO COVER
-    certifi = None
-
-try:
-    import urllib3
-except ImportError as caught_exc:  # pragma: NO COVER
-    import six
-    six.raise_from(
-        ImportError(
-            'The urllib3 library is not installed, please install the '
-            'urllib3 package to use the urllib3 transport.'
-        ),
-        caught_exc,
-    )
-import six
-import urllib3.exceptions  # pylint: disable=ungrouped-imports
-
-from google.auth import exceptions
-from google.auth import transport
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class _Response(transport.Response):
-    """urllib3 transport response adapter.
-
-    Args:
-        response (urllib3.response.HTTPResponse): The raw urllib3 response.
-    """
-    def __init__(self, response):
-        self._response = response
-
-    @property
-    def status(self):
-        return self._response.status
-
-    @property
-    def headers(self):
-        return self._response.headers
-
-    @property
-    def data(self):
-        return self._response.data
-
-
-class Request(transport.Request):
-    """urllib3 request adapter.
-
-    This class is used internally for making requests using various transports
-    in a consistent way. If you use :class:`AuthorizedHttp` you do not need
-    to construct or use this class directly.
-
-    This class can be useful if you want to manually refresh a
-    :class:`~google.auth.credentials.Credentials` instance::
-
-        import google.auth.transport.urllib3
-        import urllib3
-
-        http = urllib3.PoolManager()
-        request = google.auth.transport.urllib3.Request(http)
-
-        credentials.refresh(request)
-
-    Args:
-        http (urllib3.request.RequestMethods): An instance of any urllib3
-            class that implements :class:`~urllib3.request.RequestMethods`,
-            usually :class:`urllib3.PoolManager`.
-
-    .. automethod:: __call__
-    """
-    def __init__(self, http):
-        self.http = http
-
-    def __call__(self, url, method='GET', body=None, headers=None,
-                 timeout=None, **kwargs):
-        """Make an HTTP request using urllib3.
-
-        Args:
-            url (str): The URI to be requested.
-            method (str): The HTTP method to use for the request. Defaults
-                to 'GET'.
-            body (bytes): The payload / body in HTTP request.
-            headers (Mapping[str, str]): Request headers.
-            timeout (Optional[int]): The number of seconds to wait for a
-                response from the server. If not specified or if None, the
-                urllib3 default timeout will be used.
-            kwargs: Additional arguments passed throught to the underlying
-                urllib3 :meth:`urlopen` method.
-
-        Returns:
-            google.auth.transport.Response: The HTTP response.
-
-        Raises:
-            google.auth.exceptions.TransportError: If any exception occurred.
-        """
-        # urllib3 uses a sentinel default value for timeout, so only set it if
-        # specified.
-        if timeout is not None:
-            kwargs['timeout'] = timeout
-
-        try:
-            _LOGGER.debug('Making request: %s %s', method, url)
-            response = self.http.request(
-                method, url, body=body, headers=headers, **kwargs)
-            return _Response(response)
-        except urllib3.exceptions.HTTPError as caught_exc:
-            new_exc = exceptions.TransportError(caught_exc)
-            six.raise_from(new_exc, caught_exc)
-
-
-def _make_default_http():
-    if certifi is not None:
-        return urllib3.PoolManager(
-            cert_reqs='CERT_REQUIRED',
-            ca_certs=certifi.where())
-    else:
-        return urllib3.PoolManager()
-
-
-class AuthorizedHttp(urllib3.request.RequestMethods):
-    """A urllib3 HTTP class with credentials.
-
-    This class is used to perform requests to API endpoints that require
-    authorization::
-
-        from google.auth.transport.urllib3 import AuthorizedHttp
-
-        authed_http = AuthorizedHttp(credentials)
-
-        response = authed_http.request(
-            'GET', 'https://www.googleapis.com/storage/v1/b')
-
-    This class implements :class:`urllib3.request.RequestMethods` and can be
-    used just like any other :class:`urllib3.PoolManager`.
-
-    The underlying :meth:`urlopen` implementation handles adding the
-    credentials' headers to the request and refreshing credentials as needed.
-
-    Args:
-        credentials (google.auth.credentials.Credentials): The credentials to
-            add to the request.
-        http (urllib3.PoolManager): The underlying HTTP object to
-            use to make requests. If not specified, a
-            :class:`urllib3.PoolManager` instance will be constructed with
-            sane defaults.
-        refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
-            that credentials should be refreshed and the request should be
-            retried.
-        max_refresh_attempts (int): The maximum number of times to attempt to
-            refresh the credentials and retry the request.
-    """
-    def __init__(self, credentials, http=None,
-                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
-                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS):
-
-        if http is None:
-            http = _make_default_http()
-
-        self.credentials = credentials
-        self.http = http
-        self._refresh_status_codes = refresh_status_codes
-        self._max_refresh_attempts = max_refresh_attempts
-        # Request instance used by internal methods (for example,
-        # credentials.refresh).
-        self._request = Request(self.http)
-
-        super(AuthorizedHttp, self).__init__()
-
-    def urlopen(self, method, url, body=None, headers=None, **kwargs):
-        """Implementation of urllib3's urlopen."""
-        # pylint: disable=arguments-differ
-        # We use kwargs to collect additional args that we don't need to
-        # introspect here. However, we do explicitly collect the two
-        # positional arguments.
-
-        # Use a kwarg for this instead of an attribute to maintain
-        # thread-safety.
-        _credential_refresh_attempt = kwargs.pop(
-            '_credential_refresh_attempt', 0)
-
-        if headers is None:
-            headers = self.headers
-
-        # Make a copy of the headers. They will be modified by the credentials
-        # and we want to pass the original headers if we recurse.
-        request_headers = headers.copy()
-
-        self.credentials.before_request(
-            self._request, method, url, request_headers)
-
-        response = self.http.urlopen(
-            method, url, body=body, headers=request_headers, **kwargs)
-
-        # If the response indicated that the credentials needed to be
-        # refreshed, then refresh the credentials and re-attempt the
-        # request.
-        # A stored token may expire between the time it is retrieved and
-        # the time the request is made, so we may need to try twice.
-        # The reason urllib3's retries aren't used is because they
-        # don't allow you to modify the request headers. :/
-        if (response.status in self._refresh_status_codes
-                and _credential_refresh_attempt < self._max_refresh_attempts):
-
-            _LOGGER.info(
-                'Refreshing credentials due to a %s response. Attempt %s/%s.',
-                response.status, _credential_refresh_attempt + 1,
-                self._max_refresh_attempts)
-
-            self.credentials.refresh(self._request)
-
-            # Recurse. Pass in the original headers, not our modified set.
-            return self.urlopen(
-                method, url, body=body, headers=headers,
-                _credential_refresh_attempt=_credential_refresh_attempt + 1,
-                **kwargs)
-
-        return response
-
-    # Proxy methods for compliance with the urllib3.PoolManager interface
-
-    def __enter__(self):
-        """Proxy to ``self.http``."""
-        return self.http.__enter__()
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        """Proxy to ``self.http``."""
-        return self.http.__exit__(exc_type, exc_val, exc_tb)
-
-    @property
-    def headers(self):
-        """Proxy to ``self.http``."""
-        return self.http.headers
-
-    @headers.setter
-    def headers(self, value):
-        """Proxy to ``self.http``."""
-        self.http.headers = value

src/google/oauth2/__init__.py

@@ -1,15 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google OAuth 2.0 Library for Python."""

src/google/oauth2/_client.py

@@ -1,249 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""OAuth 2.0 client.
-
-This is a client for interacting with an OAuth 2.0 authorization server's
-token endpoint.
-
-For more information about the token endpoint, see
-`Section 3.1 of rfc6749`_
-
-.. _Section 3.1 of rfc6749: https://tools.ietf.org/html/rfc6749#section-3.2
-"""
-
-import datetime
-import json
-
-import six
-from six.moves import http_client
-from six.moves import urllib
-
-from google.auth import _helpers
-from google.auth import exceptions
-from google.auth import jwt
-
-_URLENCODED_CONTENT_TYPE = 'application/x-www-form-urlencoded'
-_JWT_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
-_REFRESH_GRANT_TYPE = 'refresh_token'
-
-
-def _handle_error_response(response_body):
-    """"Translates an error response into an exception.
-
-    Args:
-        response_body (str): The decoded response data.
-
-    Raises:
-        google.auth.exceptions.RefreshError
-    """
-    try:
-        error_data = json.loads(response_body)
-        error_details = '{}: {}'.format(
-            error_data['error'],
-            error_data.get('error_description'))
-    # If no details could be extracted, use the response data.
-    except (KeyError, ValueError):
-        error_details = response_body
-
-    raise exceptions.RefreshError(
-        error_details, response_body)
-
-
-def _parse_expiry(response_data):
-    """Parses the expiry field from a response into a datetime.
-
-    Args:
-        response_data (Mapping): The JSON-parsed response data.
-
-    Returns:
-        Optional[datetime]: The expiration or ``None`` if no expiration was
-            specified.
-    """
-    expires_in = response_data.get('expires_in', None)
-
-    if expires_in is not None:
-        return _helpers.utcnow() + datetime.timedelta(
-            seconds=expires_in)
-    else:
-        return None
-
-
-def _token_endpoint_request(request, token_uri, body):
-    """Makes a request to the OAuth 2.0 authorization server's token endpoint.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        token_uri (str): The OAuth 2.0 authorizations server's token endpoint
-            URI.
-        body (Mapping[str, str]): The parameters to send in the request body.
-
-    Returns:
-        Mapping[str, str]: The JSON-decoded response data.
-
-    Raises:
-        google.auth.exceptions.RefreshError: If the token endpoint returned
-            an error.
-    """
-    body = urllib.parse.urlencode(body)
-    headers = {
-        'content-type': _URLENCODED_CONTENT_TYPE,
-    }
-
-    response = request(
-        method='POST', url=token_uri, headers=headers, body=body)
-
-    response_body = response.data.decode('utf-8')
-
-    if response.status != http_client.OK:
-        _handle_error_response(response_body)
-
-    response_data = json.loads(response_body)
-
-    return response_data
-
-
-def jwt_grant(request, token_uri, assertion):
-    """Implements the JWT Profile for OAuth 2.0 Authorization Grants.
-
-    For more details, see `rfc7523 section 4`_.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        token_uri (str): The OAuth 2.0 authorizations server's token endpoint
-            URI.
-        assertion (str): The OAuth 2.0 assertion.
-
-    Returns:
-        Tuple[str, Optional[datetime], Mapping[str, str]]: The access token,
-            expiration, and additional data returned by the token endpoint.
-
-    Raises:
-        google.auth.exceptions.RefreshError: If the token endpoint returned
-            an error.
-
-    .. _rfc7523 section 4: https://tools.ietf.org/html/rfc7523#section-4
-    """
-    body = {
-        'assertion': assertion,
-        'grant_type': _JWT_GRANT_TYPE,
-    }
-
-    response_data = _token_endpoint_request(request, token_uri, body)
-
-    try:
-        access_token = response_data['access_token']
-    except KeyError as caught_exc:
-        new_exc = exceptions.RefreshError(
-            'No access token in response.', response_data)
-        six.raise_from(new_exc, caught_exc)
-
-    expiry = _parse_expiry(response_data)
-
-    return access_token, expiry, response_data
-
-
-def id_token_jwt_grant(request, token_uri, assertion):
-    """Implements the JWT Profile for OAuth 2.0 Authorization Grants, but
-    requests an OpenID Connect ID Token instead of an access token.
-
-    This is a variant on the standard JWT Profile that is currently unique
-    to Google. This was added for the benefit of authenticating to services
-    that require ID Tokens instead of access tokens or JWT bearer tokens.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        token_uri (str): The OAuth 2.0 authorization server's token endpoint
-            URI.
-        assertion (str): JWT token signed by a service account. The token's
-            payload must include a ``target_audience`` claim.
-
-    Returns:
-        Tuple[str, Optional[datetime], Mapping[str, str]]:
-            The (encoded) Open ID Connect ID Token, expiration, and additional
-            data returned by the endpoint.
-
-    Raises:
-        google.auth.exceptions.RefreshError: If the token endpoint returned
-            an error.
-    """
-    body = {
-        'assertion': assertion,
-        'grant_type': _JWT_GRANT_TYPE,
-    }
-
-    response_data = _token_endpoint_request(request, token_uri, body)
-
-    try:
-        id_token = response_data['id_token']
-    except KeyError as caught_exc:
-        new_exc = exceptions.RefreshError(
-            'No ID token in response.', response_data)
-        six.raise_from(new_exc, caught_exc)
-
-    payload = jwt.decode(id_token, verify=False)
-    expiry = datetime.datetime.utcfromtimestamp(payload['exp'])
-
-    return id_token, expiry, response_data
-
-
-def refresh_grant(request, token_uri, refresh_token, client_id, client_secret):
-    """Implements the OAuth 2.0 refresh token grant.
-
-    For more details, see `rfc678 section 6`_.
-
-    Args:
-        request (google.auth.transport.Request): A callable used to make
-            HTTP requests.
-        token_uri (str): The OAuth 2.0 authorizations server's token endpoint
-            URI.
-        refresh_token (str): The refresh token to use to get a new access
-            token.
-        client_id (str): The OAuth 2.0 application's client ID.
-        client_secret (str): The Oauth 2.0 appliaction's client secret.
-
-    Returns:
-        Tuple[str, Optional[str], Optional[datetime], Mapping[str, str]]: The
-            access token, new refresh token, expiration, and additional data
-            returned by the token endpoint.
-
-    Raises:
-        google.auth.exceptions.RefreshError: If the token endpoint returned
-            an error.
-
-    .. _rfc6748 section 6: https://tools.ietf.org/html/rfc6749#section-6
-    """
-    body = {
-        'grant_type': _REFRESH_GRANT_TYPE,
-        'client_id': client_id,
-        'client_secret': client_secret,
-        'refresh_token': refresh_token,
-    }
-
-    response_data = _token_endpoint_request(request, token_uri, body)
-
-    try:
-        access_token = response_data['access_token']
-    except KeyError as caught_exc:
-        new_exc = exceptions.RefreshError(
-            'No access token in response.', response_data)
-        six.raise_from(new_exc, caught_exc)
-
-    refresh_token = response_data.get('refresh_token', refresh_token)
-    expiry = _parse_expiry(response_data)
-
-    return access_token, refresh_token, expiry, response_data

src/google/oauth2/credentials.py

@@ -1,194 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""OAuth 2.0 Credentials.
-
-This module provides credentials based on OAuth 2.0 access and refresh tokens.
-These credentials usually access resources on behalf of a user (resource
-owner).
-
-Specifically, this is intended to use access tokens acquired using the
-`Authorization Code grant`_ and can refresh those tokens using a
-optional `refresh token`_.
-
-Obtaining the initial access and refresh token is outside of the scope of this
-module. Consult `rfc6749 section 4.1`_ for complete details on the
-Authorization Code grant flow.
-
-.. _Authorization Code grant: https://tools.ietf.org/html/rfc6749#section-1.3.1
-.. _refresh token: https://tools.ietf.org/html/rfc6749#section-6
-.. _rfc6749 section 4.1: https://tools.ietf.org/html/rfc6749#section-4.1
-"""
-
-import io
-import json
-
-import six
-
-from google.auth import _helpers
-from google.auth import credentials
-from google.auth import exceptions
-from google.oauth2 import _client
-
-
-# The Google OAuth 2.0 token endpoint. Used for authorized user credentials.
-_GOOGLE_OAUTH2_TOKEN_ENDPOINT = 'https://accounts.google.com/o/oauth2/token'
-
-
-class Credentials(credentials.ReadOnlyScoped, credentials.Credentials):
-    """Credentials using OAuth 2.0 access and refresh tokens."""
-
-    def __init__(self, token, refresh_token=None, id_token=None,
-                 token_uri=None, client_id=None, client_secret=None,
-                 scopes=None):
-        """
-        Args:
-            token (Optional(str)): The OAuth 2.0 access token. Can be None
-                if refresh information is provided.
-            refresh_token (str): The OAuth 2.0 refresh token. If specified,
-                credentials can be refreshed.
-            id_token (str): The Open ID Connect ID Token.
-            token_uri (str): The OAuth 2.0 authorization server's token
-                endpoint URI. Must be specified for refresh, can be left as
-                None if the token can not be refreshed.
-            client_id (str): The OAuth 2.0 client ID. Must be specified for
-                refresh, can be left as None if the token can not be refreshed.
-            client_secret(str): The OAuth 2.0 client secret. Must be specified
-                for refresh, can be left as None if the token can not be
-                refreshed.
-            scopes (Sequence[str]): The scopes that were originally used
-                to obtain authorization. This is a purely informative parameter
-                that can be used by :meth:`has_scopes`. OAuth 2.0 credentials
-                can not request additional scopes after authorization.
-        """
-        super(Credentials, self).__init__()
-        self.token = token
-        self._refresh_token = refresh_token
-        self._id_token = id_token
-        self._scopes = scopes
-        self._token_uri = token_uri
-        self._client_id = client_id
-        self._client_secret = client_secret
-
-    @property
-    def refresh_token(self):
-        """Optional[str]: The OAuth 2.0 refresh token."""
-        return self._refresh_token
-
-    @property
-    def token_uri(self):
-        """Optional[str]: The OAuth 2.0 authorization server's token endpoint
-        URI."""
-        return self._token_uri
-
-    @property
-    def id_token(self):
-        """Optional[str]: The Open ID Connect ID Token.
-
-        Depending on the authorization server and the scopes requested, this
-        may be populated when credentials are obtained and updated when
-        :meth:`refresh` is called. This token is a JWT. It can be verified
-        and decoded using :func:`google.oauth2.id_token.verify_oauth2_token`.
-        """
-        return self._id_token
-
-    @property
-    def client_id(self):
-        """Optional[str]: The OAuth 2.0 client ID."""
-        return self._client_id
-
-    @property
-    def client_secret(self):
-        """Optional[str]: The OAuth 2.0 client secret."""
-        return self._client_secret
-
-    @property
-    def requires_scopes(self):
-        """False: OAuth 2.0 credentials have their scopes set when
-        the initial token is requested and can not be changed."""
-        return False
-
-    @_helpers.copy_docstring(credentials.Credentials)
-    def refresh(self, request):
-        if (self._refresh_token is None or
-                self._token_uri is None or
-                self._client_id is None or
-                self._client_secret is None):
-            raise exceptions.RefreshError(
-                'The credentials do not contain the necessary fields need to '
-                'refresh the access token. You must specify refresh_token, '
-                'token_uri, client_id, and client_secret.')
-
-        access_token, refresh_token, expiry, grant_response = (
-            _client.refresh_grant(
-                request, self._token_uri, self._refresh_token, self._client_id,
-                self._client_secret))
-
-        self.token = access_token
-        self.expiry = expiry
-        self._refresh_token = refresh_token
-        self._id_token = grant_response.get('id_token')
-
-    @classmethod
-    def from_authorized_user_info(cls, info, scopes=None):
-        """Creates a Credentials instance from parsed authorized user info.
-
-        Args:
-            info (Mapping[str, str]): The authorized user info in Google
-                format.
-            scopes (Sequence[str]): Optional list of scopes to include in the
-                credentials.
-
-        Returns:
-            google.oauth2.credentials.Credentials: The constructed
-                credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        keys_needed = set(('refresh_token', 'client_id', 'client_secret'))
-        missing = keys_needed.difference(six.iterkeys(info))
-
-        if missing:
-            raise ValueError(
-                'Authorized user info was not in the expected format, missing '
-                'fields {}.'.format(', '.join(missing)))
-
-        return Credentials(
-            None,  # No access token, must be refreshed.
-            refresh_token=info['refresh_token'],
-            token_uri=_GOOGLE_OAUTH2_TOKEN_ENDPOINT,
-            scopes=scopes,
-            client_id=info['client_id'],
-            client_secret=info['client_secret'])
-
-    @classmethod
-    def from_authorized_user_file(cls, filename, scopes=None):
-        """Creates a Credentials instance from an authorized user json file.
-
-        Args:
-            filename (str): The path to the authorized user json file.
-            scopes (Sequence[str]): Optional list of scopes to include in the
-                credentials.
-
-        Returns:
-            google.oauth2.credentials.Credentials: The constructed
-                credentials.
-
-        Raises:
-            ValueError: If the file is not in the expected format.
-        """
-        with io.open(filename, 'r', encoding='utf-8') as json_file:
-            data = json.load(json_file)
-            return cls.from_authorized_user_info(data, scopes)

src/google/oauth2/id_token.py

@@ -1,159 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google ID Token helpers.
-
-Provides support for verifying `OpenID Connect ID Tokens`_, especially ones
-generated by Google infrastructure.
-
-To parse and verify an ID Token issued by Google's OAuth 2.0 authorization
-server use :func:`verify_oauth2_token`. To verify an ID Token issued by
-Firebase, use :func:`verify_firebase_token`.
-
-A general purpose ID Token verifier is available as :func:`verify_token`.
-
-Example::
-
-    from google.oauth2 import id_token
-    from google.auth.transport import requests
-
-    request = requests.Request()
-
-    id_info = id_token.verify_oauth2_token(
-        token, request, 'my-client-id.example.com')
-
-    if id_info['iss'] != 'https://accounts.google.com':
-        raise ValueError('Wrong issuer.')
-
-    userid = id_info['sub']
-
-By default, this will re-fetch certificates for each verification. Because
-Google's public keys are only changed infrequently (on the order of once per
-day), you may wish to take advantage of caching to reduce latency and the
-potential for network errors. This can be accomplished using an external
-library like `CacheControl`_ to create a cache-aware
-:class:`google.auth.transport.Request`::
-
-    import cachecontrol
-    import google.auth.transport.requests
-    import requests
-
-    session = requests.session()
-    cached_session = cachecontrol.CacheControl(session)
-    request = google.auth.transport.requests.Request(session=cached_session)
-
-.. _OpenID Connect ID Token:
-    http://openid.net/specs/openid-connect-core-1_0.html#IDToken
-.. _CacheControl: https://cachecontrol.readthedocs.io
-"""
-
-import json
-
-from six.moves import http_client
-
-from google.auth import exceptions
-from google.auth import jwt
-
-# The URL that provides public certificates for verifying ID tokens issued
-# by Google's OAuth 2.0 authorization server.
-_GOOGLE_OAUTH2_CERTS_URL = 'https://www.googleapis.com/oauth2/v1/certs'
-
-# The URL that provides public certificates for verifying ID tokens issued
-# by Firebase and the Google APIs infrastructure
-_GOOGLE_APIS_CERTS_URL = (
-    'https://www.googleapis.com/robot/v1/metadata/x509'
-    '/securetoken@system.gserviceaccount.com')
-
-
-def _fetch_certs(request, certs_url):
-    """Fetches certificates.
-
-    Google-style cerificate endpoints return JSON in the format of
-    ``{'key id': 'x509 certificate'}``.
-
-    Args:
-        request (google.auth.transport.Request): The object used to make
-            HTTP requests.
-        certs_url (str): The certificate endpoint URL.
-
-    Returns:
-        Mapping[str, str]: A mapping of public key ID to x.509 certificate
-            data.
-    """
-    response = request(certs_url, method='GET')
-
-    if response.status != http_client.OK:
-        raise exceptions.TransportError(
-            'Could not fetch certificates at {}'.format(certs_url))
-
-    return json.loads(response.data.decode('utf-8'))
-
-
-def verify_token(id_token, request, audience=None,
-                 certs_url=_GOOGLE_OAUTH2_CERTS_URL):
-    """Verifies an ID token and returns the decoded token.
-
-    Args:
-        id_token (Union[str, bytes]): The encoded token.
-        request (google.auth.transport.Request): The object used to make
-            HTTP requests.
-        audience (str): The audience that this token is intended for. If None
-            then the audience is not verified.
-        certs_url (str): The URL that specifies the certificates to use to
-            verify the token. This URL should return JSON in the format of
-            ``{'key id': 'x509 certificate'}``.
-
-    Returns:
-        Mapping[str, Any]: The decoded token.
-    """
-    certs = _fetch_certs(request, certs_url)
-
-    return jwt.decode(id_token, certs=certs, audience=audience)
-
-
-def verify_oauth2_token(id_token, request, audience=None):
-    """Verifies an ID Token issued by Google's OAuth 2.0 authorization server.
-
-    Args:
-        id_token (Union[str, bytes]): The encoded token.
-        request (google.auth.transport.Request): The object used to make
-            HTTP requests.
-        audience (str): The audience that this token is intended for. This is
-            typically your application's OAuth 2.0 client ID. If None then the
-            audience is not verified.
-
-    Returns:
-        Mapping[str, Any]: The decoded token.
-    """
-    return verify_token(
-        id_token, request, audience=audience,
-        certs_url=_GOOGLE_OAUTH2_CERTS_URL)
-
-
-def verify_firebase_token(id_token, request, audience=None):
-    """Verifies an ID Token issued by Firebase Authentication.
-
-    Args:
-        id_token (Union[str, bytes]): The encoded token.
-        request (google.auth.transport.Request): The object used to make
-            HTTP requests.
-        audience (str): The audience that this token is intended for. This is
-            typically your Firebase application ID. If None then the audience
-            is not verified.
-
-    Returns:
-        Mapping[str, Any]: The decoded token.
-    """
-    return verify_token(
-        id_token, request, audience=audience, certs_url=_GOOGLE_APIS_CERTS_URL)

src/google/oauth2/service_account.py

@@ -1,542 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Service Accounts: JSON Web Token (JWT) Profile for OAuth 2.0
-
-This module implements the JWT Profile for OAuth 2.0 Authorization Grants
-as defined by `RFC 7523`_ with particular support for how this RFC is
-implemented in Google's infrastructure. Google refers to these credentials
-as *Service Accounts*.
-
-Service accounts are used for server-to-server communication, such as
-interactions between a web application server and a Google service. The
-service account belongs to your application instead of to an individual end
-user. In contrast to other OAuth 2.0 profiles, no users are involved and your
-application "acts" as the service account.
-
-Typically an application uses a service account when the application uses
-Google APIs to work with its own data rather than a user's data. For example,
-an application that uses Google Cloud Datastore for data persistence would use
-a service account to authenticate its calls to the Google Cloud Datastore API.
-However, an application that needs to access a user's Drive documents would
-use the normal OAuth 2.0 profile.
-
-Additionally, Google Apps domain administrators can grant service accounts
-`domain-wide delegation`_ authority to access user data on behalf of users in
-the domain.
-
-This profile uses a JWT to acquire an OAuth 2.0 access token. The JWT is used
-in place of the usual authorization token returned during the standard
-OAuth 2.0 Authorization Code grant. The JWT is only used for this purpose, as
-the acquired access token is used as the bearer token when making requests
-using these credentials.
-
-This profile differs from normal OAuth 2.0 profile because no user consent
-step is required. The use of the private key allows this profile to assert
-identity directly.
-
-This profile also differs from the :mod:`google.auth.jwt` authentication
-because the JWT credentials use the JWT directly as the bearer token. This
-profile instead only uses the JWT to obtain an OAuth 2.0 access token. The
-obtained OAuth 2.0 access token is used as the bearer token.
-
-Domain-wide delegation
-----------------------
-
-Domain-wide delegation allows a service account to access user data on
-behalf of any user in a Google Apps domain without consent from the user.
-For example, an application that uses the Google Calendar API to add events to
-the calendars of all users in a Google Apps domain would use a service account
-to access the Google Calendar API on behalf of users.
-
-The Google Apps administrator must explicitly authorize the service account to
-do this. This authorization step is referred to as "delegating domain-wide
-authority" to a service account.
-
-You can use domain-wise delegation by creating a set of credentials with a
-specific subject using :meth:`~Credentials.with_subject`.
-
-.. _RFC 7523: https://tools.ietf.org/html/rfc7523
-"""
-
-import copy
-import datetime
-
-from google.auth import _helpers
-from google.auth import _service_account_info
-from google.auth import credentials
-from google.auth import jwt
-from google.oauth2 import _client
-
-_DEFAULT_TOKEN_LIFETIME_SECS = 3600  # 1 hour in seconds
-
-
-class Credentials(credentials.Signing,
-                  credentials.Scoped,
-                  credentials.Credentials):
-    """Service account credentials
-
-    Usually, you'll create these credentials with one of the helper
-    constructors. To create credentials using a Google service account
-    private key JSON file::
-
-        credentials = service_account.Credentials.from_service_account_file(
-            'service-account.json')
-
-    Or if you already have the service account file loaded::
-
-        service_account_info = json.load(open('service_account.json'))
-        credentials = service_account.Credentials.from_service_account_info(
-            service_account_info)
-
-    Both helper methods pass on arguments to the constructor, so you can
-    specify additional scopes and a subject if necessary::
-
-        credentials = service_account.Credentials.from_service_account_file(
-            'service-account.json',
-            scopes=['email'],
-            subject='user@example.com')
-
-    The credentials are considered immutable. If you want to modify the scopes
-    or the subject used for delegation, use :meth:`with_scopes` or
-    :meth:`with_subject`::
-
-        scoped_credentials = credentials.with_scopes(['email'])
-        delegated_credentials = credentials.with_subject(subject)
-    """
-
-    def __init__(self, signer, service_account_email, token_uri, scopes=None,
-                 subject=None, project_id=None, additional_claims=None):
-        """
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            service_account_email (str): The service account's email.
-            scopes (Sequence[str]): Scopes to request during the authorization
-                grant.
-            token_uri (str): The OAuth 2.0 Token URI.
-            subject (str): For domain-wide delegation, the email address of the
-                user to for which to request delegated access.
-            project_id  (str): Project ID associated with the service account
-                credential.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT assertion used in the authorization grant.
-
-        .. note:: Typically one of the helper constructors
-            :meth:`from_service_account_file` or
-            :meth:`from_service_account_info` are used instead of calling the
-            constructor directly.
-        """
-        super(Credentials, self).__init__()
-
-        self._scopes = scopes
-        self._signer = signer
-        self._service_account_email = service_account_email
-        self._subject = subject
-        self._project_id = project_id
-        self._token_uri = token_uri
-
-        if additional_claims is not None:
-            self._additional_claims = additional_claims
-        else:
-            self._additional_claims = {}
-
-    @classmethod
-    def _from_signer_and_info(cls, signer, info, **kwargs):
-        """Creates a Credentials instance from a signer and service account
-        info.
-
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            info (Mapping[str, str]): The service account info.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.Credentials: The constructed credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        return cls(
-            signer,
-            service_account_email=info['client_email'],
-            token_uri=info['token_uri'],
-            project_id=info.get('project_id'), **kwargs)
-
-    @classmethod
-    def from_service_account_info(cls, info, **kwargs):
-        """Creates a Credentials instance from parsed service account info.
-
-        Args:
-            info (Mapping[str, str]): The service account info in Google
-                format.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.service_account.Credentials: The constructed
-                credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        signer = _service_account_info.from_dict(
-            info, require=['client_email', 'token_uri'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @classmethod
-    def from_service_account_file(cls, filename, **kwargs):
-        """Creates a Credentials instance from a service account json file.
-
-        Args:
-            filename (str): The path to the service account json file.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.service_account.Credentials: The constructed
-                credentials.
-        """
-        info, signer = _service_account_info.from_filename(
-            filename, require=['client_email', 'token_uri'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @property
-    def service_account_email(self):
-        """The service account email."""
-        return self._service_account_email
-
-    @property
-    def project_id(self):
-        """Project ID associated with this credential."""
-        return self._project_id
-
-    @property
-    def requires_scopes(self):
-        """Checks if the credentials requires scopes.
-
-        Returns:
-            bool: True if there are no scopes set otherwise False.
-        """
-        return True if not self._scopes else False
-
-    @_helpers.copy_docstring(credentials.Scoped)
-    def with_scopes(self, scopes):
-        return self.__class__(
-            self._signer,
-            service_account_email=self._service_account_email,
-            scopes=scopes,
-            token_uri=self._token_uri,
-            subject=self._subject,
-            project_id=self._project_id,
-            additional_claims=self._additional_claims.copy())
-
-    def with_subject(self, subject):
-        """Create a copy of these credentials with the specified subject.
-
-        Args:
-            subject (str): The subject claim.
-
-        Returns:
-            google.auth.service_account.Credentials: A new credentials
-                instance.
-        """
-        return self.__class__(
-            self._signer,
-            service_account_email=self._service_account_email,
-            scopes=self._scopes,
-            token_uri=self._token_uri,
-            subject=subject,
-            project_id=self._project_id,
-            additional_claims=self._additional_claims.copy())
-
-    def with_claims(self, additional_claims):
-        """Returns a copy of these credentials with modified claims.
-
-        Args:
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT payload. This will be merged with the current
-                additional claims.
-
-        Returns:
-            google.auth.service_account.Credentials: A new credentials
-                instance.
-        """
-        new_additional_claims = copy.deepcopy(self._additional_claims)
-        new_additional_claims.update(additional_claims or {})
-
-        return self.__class__(
-            self._signer,
-            service_account_email=self._service_account_email,
-            scopes=self._scopes,
-            token_uri=self._token_uri,
-            subject=self._subject,
-            project_id=self._project_id,
-            additional_claims=new_additional_claims)
-
-    def _make_authorization_grant_assertion(self):
-        """Create the OAuth 2.0 assertion.
-
-        This assertion is used during the OAuth 2.0 grant to acquire an
-        access token.
-
-        Returns:
-            bytes: The authorization grant assertion.
-        """
-        now = _helpers.utcnow()
-        lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS)
-        expiry = now + lifetime
-
-        payload = {
-            'iat': _helpers.datetime_to_secs(now),
-            'exp': _helpers.datetime_to_secs(expiry),
-            # The issuer must be the service account email.
-            'iss': self._service_account_email,
-            # The audience must be the auth token endpoint's URI
-            'aud': self._token_uri,
-            'scope': _helpers.scopes_to_string(self._scopes or ())
-        }
-
-        payload.update(self._additional_claims)
-
-        # The subject can be a user email for domain-wide delegation.
-        if self._subject:
-            payload.setdefault('sub', self._subject)
-
-        token = jwt.encode(self._signer, payload)
-
-        return token
-
-    @_helpers.copy_docstring(credentials.Credentials)
-    def refresh(self, request):
-        assertion = self._make_authorization_grant_assertion()
-        access_token, expiry, _ = _client.jwt_grant(
-            request, self._token_uri, assertion)
-        self.token = access_token
-        self.expiry = expiry
-
-    @_helpers.copy_docstring(credentials.Signing)
-    def sign_bytes(self, message):
-        return self._signer.sign(message)
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer(self):
-        return self._signer
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer_email(self):
-        return self._service_account_email
-
-
-class IDTokenCredentials(credentials.Signing, credentials.Credentials):
-    """Open ID Connect ID Token-based service account credentials.
-
-    These credentials are largely similar to :class:`.Credentials`, but instead
-    of using an OAuth 2.0 Access Token as the bearer token, they use an Open
-    ID Connect ID Token as the bearer token. These credentials are useful when
-    communicating to services that require ID Tokens and can not accept access
-    tokens.
-
-    Usually, you'll create these credentials with one of the helper
-    constructors. To create credentials using a Google service account
-    private key JSON file::
-
-        credentials = (
-            service_account.IDTokenCredentials.from_service_account_file(
-                'service-account.json'))
-
-    Or if you already have the service account file loaded::
-
-        service_account_info = json.load(open('service_account.json'))
-        credentials = (
-            service_account.IDTokenCredentials.from_service_account_info(
-                service_account_info))
-
-    Both helper methods pass on arguments to the constructor, so you can
-    specify additional scopes and a subject if necessary::
-
-        credentials = (
-            service_account.IDTokenCredentials.from_service_account_file(
-                'service-account.json',
-                scopes=['email'],
-                subject='user@example.com'))
-`
-    The credentials are considered immutable. If you want to modify the scopes
-    or the subject used for delegation, use :meth:`with_scopes` or
-    :meth:`with_subject`::
-
-        scoped_credentials = credentials.with_scopes(['email'])
-        delegated_credentials = credentials.with_subject(subject)
-
-    """
-    def __init__(self, signer, service_account_email, token_uri,
-                 target_audience, additional_claims=None):
-        """
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            service_account_email (str): The service account's email.
-            token_uri (str): The OAuth 2.0 Token URI.
-            target_audience (str): The intended audience for these credentials,
-                used when requesting the ID Token. The ID Token's ``aud`` claim
-                will be set to this string.
-            additional_claims (Mapping[str, str]): Any additional claims for
-                the JWT assertion used in the authorization grant.
-
-        .. note:: Typically one of the helper constructors
-            :meth:`from_service_account_file` or
-            :meth:`from_service_account_info` are used instead of calling the
-            constructor directly.
-        """
-        super(IDTokenCredentials, self).__init__()
-        self._signer = signer
-        self._service_account_email = service_account_email
-        self._token_uri = token_uri
-        self._target_audience = target_audience
-
-        if additional_claims is not None:
-            self._additional_claims = additional_claims
-        else:
-            self._additional_claims = {}
-
-    @classmethod
-    def _from_signer_and_info(cls, signer, info, **kwargs):
-        """Creates a credentials instance from a signer and service account
-        info.
-
-        Args:
-            signer (google.auth.crypt.Signer): The signer used to sign JWTs.
-            info (Mapping[str, str]): The service account info.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.jwt.IDTokenCredentials: The constructed credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        kwargs.setdefault('service_account_email', info['client_email'])
-        kwargs.setdefault('token_uri', info['token_uri'])
-        return cls(signer, **kwargs)
-
-    @classmethod
-    def from_service_account_info(cls, info, **kwargs):
-        """Creates a credentials instance from parsed service account info.
-
-        Args:
-            info (Mapping[str, str]): The service account info in Google
-                format.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.service_account.IDTokenCredentials: The constructed
-                credentials.
-
-        Raises:
-            ValueError: If the info is not in the expected format.
-        """
-        signer = _service_account_info.from_dict(
-            info, require=['client_email', 'token_uri'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    @classmethod
-    def from_service_account_file(cls, filename, **kwargs):
-        """Creates a credentials instance from a service account json file.
-
-        Args:
-            filename (str): The path to the service account json file.
-            kwargs: Additional arguments to pass to the constructor.
-
-        Returns:
-            google.auth.service_account.IDTokenCredentials: The constructed
-                credentials.
-        """
-        info, signer = _service_account_info.from_filename(
-            filename, require=['client_email', 'token_uri'])
-        return cls._from_signer_and_info(signer, info, **kwargs)
-
-    def with_target_audience(self, target_audience):
-        """Create a copy of these credentials with the specified target
-        audience.
-
-        Args:
-            target_audience (str): The intended audience for these credentials,
-            used when requesting the ID Token.
-
-        Returns:
-            google.auth.service_account.IDTokenCredentials: A new credentials
-                instance.
-        """
-        return self.__class__(
-            self._signer,
-            service_account_email=self._service_account_email,
-            token_uri=self._token_uri,
-            target_audience=target_audience,
-            additional_claims=self._additional_claims.copy())
-
-    def _make_authorization_grant_assertion(self):
-        """Create the OAuth 2.0 assertion.
-
-        This assertion is used during the OAuth 2.0 grant to acquire an
-        ID token.
-
-        Returns:
-            bytes: The authorization grant assertion.
-        """
-        now = _helpers.utcnow()
-        lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS)
-        expiry = now + lifetime
-
-        payload = {
-            'iat': _helpers.datetime_to_secs(now),
-            'exp': _helpers.datetime_to_secs(expiry),
-            # The issuer must be the service account email.
-            'iss': self.service_account_email,
-            # The audience must be the auth token endpoint's URI
-            'aud': self._token_uri,
-            # The target audience specifies which service the ID token is
-            # intended for.
-            'target_audience': self._target_audience
-        }
-
-        payload.update(self._additional_claims)
-
-        token = jwt.encode(self._signer, payload)
-
-        return token
-
-    @_helpers.copy_docstring(credentials.Credentials)
-    def refresh(self, request):
-        assertion = self._make_authorization_grant_assertion()
-        access_token, expiry, _ = _client.id_token_jwt_grant(
-            request, self._token_uri, assertion)
-        self.token = access_token
-        self.expiry = expiry
-
-    @property
-    def service_account_email(self):
-        """The service account email."""
-        return self._service_account_email
-
-    @_helpers.copy_docstring(credentials.Signing)
-    def sign_bytes(self, message):
-        return self._signer.sign(message)
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer(self):
-        return self._signer
-
-    @property
-    @_helpers.copy_docstring(credentials.Signing)
-    def signer_email(self):
-        return self._service_account_email

src/google_auth_httplib2/__init__.py

@@ -1,238 +0,0 @@
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Transport adapter for httplib2."""
-
-from __future__ import absolute_import
-
-import logging
-
-from google.auth import exceptions
-from google.auth import transport
-import httplib2
-from six.moves import http_client
-
-
-_LOGGER = logging.getLogger(__name__)
-# Properties present in file-like streams / buffers.
-_STREAM_PROPERTIES = ('read', 'seek', 'tell')
-
-
-class _Response(transport.Response):
-    """httplib2 transport response adapter.
-
-    Args:
-        response (httplib2.Response): The raw httplib2 response.
-        data (bytes): The response body.
-    """
-    def __init__(self, response, data):
-        self._response = response
-        self._data = data
-
-    @property
-    def status(self):
-        """int: The HTTP status code."""
-        return self._response.status
-
-    @property
-    def headers(self):
-        """Mapping[str, str]: The HTTP response headers."""
-        return dict(self._response)
-
-    @property
-    def data(self):
-        """bytes: The response body."""
-        return self._data
-
-
-class Request(transport.Request):
-    """httplib2 request adapter.
-
-    This class is used internally for making requests using various transports
-    in a consistent way. If you use :class:`AuthorizedHttp` you do not need
-    to construct or use this class directly.
-
-    This class can be useful if you want to manually refresh a
-    :class:`~google.auth.credentials.Credentials` instance::
-
-        import google_auth_httplib2
-        import httplib2
-
-        http = httplib2.Http()
-        request = google_auth_httplib2.Request(http)
-
-        credentials.refresh(request)
-
-    Args:
-        http (httplib2.Http): The underlying http object to use to make
-            requests.
-
-    .. automethod:: __call__
-    """
-    def __init__(self, http):
-        self.http = http
-
-    def __call__(self, url, method='GET', body=None, headers=None,
-                 timeout=None, **kwargs):
-        """Make an HTTP request using httplib2.
-
-        Args:
-            url (str): The URI to be requested.
-            method (str): The HTTP method to use for the request. Defaults
-                to 'GET'.
-            body (bytes): The payload / body in HTTP request.
-            headers (Mapping[str, str]): Request headers.
-            timeout (Optional[int]): The number of seconds to wait for a
-                response from the server. This is ignored by httplib2 and will
-                issue a warning.
-            kwargs: Additional arguments passed throught to the underlying
-                :meth:`httplib2.Http.request` method.
-
-        Returns:
-            google.auth.transport.Response: The HTTP response.
-
-        Raises:
-            google.auth.exceptions.TransportError: If any exception occurred.
-        """
-        if timeout is not None:
-            _LOGGER.warning(
-                'httplib2 transport does not support per-request timeout. '
-                'Set the timeout when constructing the httplib2.Http instance.'
-            )
-
-        try:
-            _LOGGER.debug('Making request: %s %s', method, url)
-            response, data = self.http.request(
-                url, method=method, body=body, headers=headers, **kwargs)
-            return _Response(response, data)
-        # httplib2 should catch the lower http error, this is a bug and
-        # needs to be fixed there.  Catch the error for the meanwhile.
-        except (httplib2.HttpLib2Error, http_client.HTTPException) as exc:
-            raise exceptions.TransportError(exc)
-
-
-def _make_default_http():
-    """Returns a default httplib2.Http instance."""
-    return httplib2.Http()
-
-
-class AuthorizedHttp(object):
-    """A httplib2 HTTP class with credentials.
-
-    This class is used to perform requests to API endpoints that require
-    authorization::
-
-        from google.auth.transport._httplib2 import AuthorizedHttp
-
-        authed_http = AuthorizedHttp(credentials)
-
-        response = authed_http.request(
-            'https://www.googleapis.com/storage/v1/b')
-
-    This class implements :meth:`request` in the same way as
-    :class:`httplib2.Http` and can usually be used just like any other
-    instance of :class:``httplib2.Http`.
-
-    The underlying :meth:`request` implementation handles adding the
-    credentials' headers to the request and refreshing credentials as needed.
-    """
-    def __init__(self, credentials, http=None,
-                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
-                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS):
-        """
-        Args:
-            credentials (google.auth.credentials.Credentials): The credentials
-                to add to the request.
-            http (httplib2.Http): The underlying HTTP object to
-                use to make requests. If not specified, a
-                :class:`httplib2.Http` instance will be constructed.
-            refresh_status_codes (Sequence[int]): Which HTTP status codes
-                indicate that credentials should be refreshed and the request
-                should be retried.
-            max_refresh_attempts (int): The maximum number of times to attempt
-                to refresh the credentials and retry the request.
-        """
-
-        if http is None:
-            http = _make_default_http()
-
-        self.http = http
-        self.credentials = credentials
-        self._refresh_status_codes = refresh_status_codes
-        self._max_refresh_attempts = max_refresh_attempts
-        # Request instance used by internal methods (for example,
-        # credentials.refresh).
-        self._request = Request(self.http)
-
-    def request(self, uri, method='GET', body=None, headers=None,
-                **kwargs):
-        """Implementation of httplib2's Http.request."""
-
-        _credential_refresh_attempt = kwargs.pop(
-            '_credential_refresh_attempt', 0)
-
-        # Make a copy of the headers. They will be modified by the credentials
-        # and we want to pass the original headers if we recurse.
-        request_headers = headers.copy() if headers is not None else {}
-
-        self.credentials.before_request(
-            self._request, method, uri, request_headers)
-
-        # Check if the body is a file-like stream, and if so, save the body
-        # stream position so that it can be restored in case of refresh.
-        body_stream_position = None
-        if all(getattr(body, stream_prop, None) for stream_prop in
-               _STREAM_PROPERTIES):
-            body_stream_position = body.tell()
-
-        # Make the request.
-        response, content = self.http.request(
-            uri, method, body=body, headers=request_headers, **kwargs)
-
-        # If the response indicated that the credentials needed to be
-        # refreshed, then refresh the credentials and re-attempt the
-        # request.
-        # A stored token may expire between the time it is retrieved and
-        # the time the request is made, so we may need to try twice.
-        if (response.status in self._refresh_status_codes
-                and _credential_refresh_attempt < self._max_refresh_attempts):
-
-            _LOGGER.info(
-                'Refreshing credentials due to a %s response. Attempt %s/%s.',
-                response.status, _credential_refresh_attempt + 1,
-                self._max_refresh_attempts)
-
-            self.credentials.refresh(self._request)
-
-            # Restore the body's stream position if needed.
-            if body_stream_position is not None:
-                body.seek(body_stream_position)
-
-            # Recurse. Pass in the original headers, not our modified set.
-            return self.request(
-                uri, method, body=body, headers=headers,
-                _credential_refresh_attempt=_credential_refresh_attempt + 1,
-                **kwargs)
-
-        return response, content
-
-    @property
-    def connections(self):
-        """Proxy to httplib2.Http.connections."""
-        return self.http.connections
-
-    @connections.setter
-    def connections(self, value):
-        """Proxy to httplib2.Http.connections."""
-        self.http.connections = value

src/googleapiclient/__init__.py

@@ -1,27 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-__version__ = "1.7.3"
-
-# Set default logging handler to avoid "No handler found" warnings.
-import logging
-
-try:  # Python 2.7+
-    from logging import NullHandler
-except ImportError:
-    class NullHandler(logging.Handler):
-        def emit(self, record):
-            pass
-
-logging.getLogger(__name__).addHandler(NullHandler())

src/googleapiclient/_auth.py

@@ -1,147 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helpers for authentication using oauth2client or google-auth."""
-
-import httplib2
-
-try:
-    import google.auth
-    import google.auth.credentials
-    HAS_GOOGLE_AUTH = True
-except ImportError:  # pragma: NO COVER
-    HAS_GOOGLE_AUTH = False
-
-try:
-    import google_auth_httplib2
-except ImportError:  # pragma: NO COVER
-    google_auth_httplib2 = None
-
-try:
-    import oauth2client
-    import oauth2client.client
-    HAS_OAUTH2CLIENT = True
-except ImportError:  # pragma: NO COVER
-    HAS_OAUTH2CLIENT = False
-
-
-def default_credentials():
-    """Returns Application Default Credentials."""
-    if HAS_GOOGLE_AUTH:
-        credentials, _ = google.auth.default()
-        return credentials
-    elif HAS_OAUTH2CLIENT:
-        return oauth2client.client.GoogleCredentials.get_application_default()
-    else:
-        raise EnvironmentError(
-            'No authentication library is available. Please install either '
-            'google-auth or oauth2client.')
-
-
-def with_scopes(credentials, scopes):
-    """Scopes the credentials if necessary.
-
-    Args:
-        credentials (Union[
-            google.auth.credentials.Credentials,
-            oauth2client.client.Credentials]): The credentials to scope.
-        scopes (Sequence[str]): The list of scopes.
-
-    Returns:
-        Union[google.auth.credentials.Credentials,
-            oauth2client.client.Credentials]: The scoped credentials.
-    """
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        return google.auth.credentials.with_scopes_if_required(
-            credentials, scopes)
-    else:
-        try:
-            if credentials.create_scoped_required():
-                return credentials.create_scoped(scopes)
-            else:
-                return credentials
-        except AttributeError:
-            return credentials
-
-
-def authorized_http(credentials):
-    """Returns an http client that is authorized with the given credentials.
-
-    Args:
-        credentials (Union[
-            google.auth.credentials.Credentials,
-            oauth2client.client.Credentials]): The credentials to use.
-
-    Returns:
-        Union[httplib2.Http, google_auth_httplib2.AuthorizedHttp]: An
-            authorized http client.
-    """
-    from googleapiclient.http import build_http
-
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        if google_auth_httplib2 is None:
-            raise ValueError(
-                'Credentials from google.auth specified, but '
-                'google-api-python-client is unable to use these credentials '
-                'unless google-auth-httplib2 is installed. Please install '
-                'google-auth-httplib2.')
-        return google_auth_httplib2.AuthorizedHttp(credentials,
-                                                   http=build_http())
-    else:
-        return credentials.authorize(build_http())
-
-
-def refresh_credentials(credentials):
-    # Refresh must use a new http instance, as the one associated with the
-    # credentials could be a AuthorizedHttp or an oauth2client-decorated
-    # Http instance which would cause a weird recursive loop of refreshing
-    # and likely tear a hole in spacetime.
-    refresh_http = httplib2.Http()
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        request = google_auth_httplib2.Request(refresh_http)
-        return credentials.refresh(request)
-    else:
-        return credentials.refresh(refresh_http)
-
-
-def apply_credentials(credentials, headers):
-    # oauth2client and google-auth have the same interface for this.
-    if not is_valid(credentials):
-        refresh_credentials(credentials)
-    return credentials.apply(headers)
-
-
-def is_valid(credentials):
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        return credentials.valid
-    else:
-        return (
-            credentials.access_token is not None and
-            not credentials.access_token_expired)
-
-
-def get_credentials_from_http(http):
-    if http is None:
-        return None
-    elif hasattr(http.request, 'credentials'):
-        return http.request.credentials
-    elif (hasattr(http, 'credentials')
-          and not isinstance(http.credentials, httplib2.Credentials)):
-        return http.credentials
-    else:
-        return None

src/googleapiclient/_helpers.py

@@ -1,204 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helper functions for commonly used utilities."""
-
-import functools
-import inspect
-import logging
-import warnings
-
-import six
-from six.moves import urllib
-
-
-logger = logging.getLogger(__name__)
-
-POSITIONAL_WARNING = 'WARNING'
-POSITIONAL_EXCEPTION = 'EXCEPTION'
-POSITIONAL_IGNORE = 'IGNORE'
-POSITIONAL_SET = frozenset([POSITIONAL_WARNING, POSITIONAL_EXCEPTION,
-                            POSITIONAL_IGNORE])
-
-positional_parameters_enforcement = POSITIONAL_WARNING
-
-_SYM_LINK_MESSAGE = 'File: {0}: Is a symbolic link.'
-_IS_DIR_MESSAGE = '{0}: Is a directory'
-_MISSING_FILE_MESSAGE = 'Cannot access {0}: No such file or directory'
-
-
-def positional(max_positional_args):
-    """A decorator to declare that only the first N arguments my be positional.
-
-    This decorator makes it easy to support Python 3 style keyword-only
-    parameters. For example, in Python 3 it is possible to write::
-
-        def fn(pos1, *, kwonly1=None, kwonly1=None):
-            ...
-
-    All named parameters after ``*`` must be a keyword::
-
-        fn(10, 'kw1', 'kw2')  # Raises exception.
-        fn(10, kwonly1='kw1')  # Ok.
-
-    Example
-    ^^^^^^^
-
-    To define a function like above, do::
-
-        @positional(1)
-        def fn(pos1, kwonly1=None, kwonly2=None):
-            ...
-
-    If no default value is provided to a keyword argument, it becomes a
-    required keyword argument::
-
-        @positional(0)
-        def fn(required_kw):
-            ...
-
-    This must be called with the keyword parameter::
-
-        fn()  # Raises exception.
-        fn(10)  # Raises exception.
-        fn(required_kw=10)  # Ok.
-
-    When defining instance or class methods always remember to account for
-    ``self`` and ``cls``::
-
-        class MyClass(object):
-
-            @positional(2)
-            def my_method(self, pos1, kwonly1=None):
-                ...
-
-            @classmethod
-            @positional(2)
-            def my_method(cls, pos1, kwonly1=None):
-                ...
-
-    The positional decorator behavior is controlled by
-    ``_helpers.positional_parameters_enforcement``, which may be set to
-    ``POSITIONAL_EXCEPTION``, ``POSITIONAL_WARNING`` or
-    ``POSITIONAL_IGNORE`` to raise an exception, log a warning, or do
-    nothing, respectively, if a declaration is violated.
-
-    Args:
-        max_positional_arguments: Maximum number of positional arguments. All
-                                  parameters after the this index must be
-                                  keyword only.
-
-    Returns:
-        A decorator that prevents using arguments after max_positional_args
-        from being used as positional parameters.
-
-    Raises:
-        TypeError: if a key-word only argument is provided as a positional
-                   parameter, but only if
-                   _helpers.positional_parameters_enforcement is set to
-                   POSITIONAL_EXCEPTION.
-    """
-
-    def positional_decorator(wrapped):
-        @functools.wraps(wrapped)
-        def positional_wrapper(*args, **kwargs):
-            if len(args) > max_positional_args:
-                plural_s = ''
-                if max_positional_args != 1:
-                    plural_s = 's'
-                message = ('{function}() takes at most {args_max} positional '
-                           'argument{plural} ({args_given} given)'.format(
-                               function=wrapped.__name__,
-                               args_max=max_positional_args,
-                               args_given=len(args),
-                               plural=plural_s))
-                if positional_parameters_enforcement == POSITIONAL_EXCEPTION:
-                    raise TypeError(message)
-                elif positional_parameters_enforcement == POSITIONAL_WARNING:
-                    logger.warning(message)
-            return wrapped(*args, **kwargs)
-        return positional_wrapper
-
-    if isinstance(max_positional_args, six.integer_types):
-        return positional_decorator
-    else:
-        args, _, _, defaults = inspect.getargspec(max_positional_args)
-        return positional(len(args) - len(defaults))(max_positional_args)
-
-
-def parse_unique_urlencoded(content):
-    """Parses unique key-value parameters from urlencoded content.
-
-    Args:
-        content: string, URL-encoded key-value pairs.
-
-    Returns:
-        dict, The key-value pairs from ``content``.
-
-    Raises:
-        ValueError: if one of the keys is repeated.
-    """
-    urlencoded_params = urllib.parse.parse_qs(content)
-    params = {}
-    for key, value in six.iteritems(urlencoded_params):
-        if len(value) != 1:
-            msg = ('URL-encoded content contains a repeated value:'
-                   '%s -> %s' % (key, ', '.join(value)))
-            raise ValueError(msg)
-        params[key] = value[0]
-    return params
-
-
-def update_query_params(uri, params):
-    """Updates a URI with new query parameters.
-
-    If a given key from ``params`` is repeated in the ``uri``, then
-    the URI will be considered invalid and an error will occur.
-
-    If the URI is valid, then each value from ``params`` will
-    replace the corresponding value in the query parameters (if
-    it exists).
-
-    Args:
-        uri: string, A valid URI, with potential existing query parameters.
-        params: dict, A dictionary of query parameters.
-
-    Returns:
-        The same URI but with the new query parameters added.
-    """
-    parts = urllib.parse.urlparse(uri)
-    query_params = parse_unique_urlencoded(parts.query)
-    query_params.update(params)
-    new_query = urllib.parse.urlencode(query_params)
-    new_parts = parts._replace(query=new_query)
-    return urllib.parse.urlunparse(new_parts)
-
-
-def _add_query_parameter(url, name, value):
-    """Adds a query parameter to a url.
-
-    Replaces the current value if it already exists in the URL.
-
-    Args:
-        url: string, url to add the query parameter to.
-        name: string, query parameter name.
-        value: string, query parameter value.
-
-    Returns:
-        Updated query parameter. Does not update the url if value is None.
-    """
-    if value is None:
-        return url
-    else:
-        return update_query_params(url, {name: value})

src/googleapiclient/channel.py

@@ -1,287 +0,0 @@
-"""Channel notifications support.
-
-Classes and functions to support channel subscriptions and notifications
-on those channels.
-
-Notes:
-  - This code is based on experimental APIs and is subject to change.
-  - Notification does not do deduplication of notification ids, that's up to
-    the receiver.
-  - Storing the Channel between calls is up to the caller.
-
-
-Example setting up a channel:
-
-  # Create a new channel that gets notifications via webhook.
-  channel = new_webhook_channel("https://example.com/my_web_hook")
-
-  # Store the channel, keyed by 'channel.id'. Store it before calling the
-  # watch method because notifications may start arriving before the watch
-  # method returns.
-  ...
-
-  resp = service.objects().watchAll(
-    bucket="some_bucket_id", body=channel.body()).execute()
-  channel.update(resp)
-
-  # Store the channel, keyed by 'channel.id'. Store it after being updated
-  # since the resource_id value will now be correct, and that's needed to
-  # stop a subscription.
-  ...
-
-
-An example Webhook implementation using webapp2. Note that webapp2 puts
-headers in a case insensitive dictionary, as headers aren't guaranteed to
-always be upper case.
-
-  id = self.request.headers[X_GOOG_CHANNEL_ID]
-
-  # Retrieve the channel by id.
-  channel = ...
-
-  # Parse notification from the headers, including validating the id.
-  n = notification_from_headers(channel, self.request.headers)
-
-  # Do app specific stuff with the notification here.
-  if n.resource_state == 'sync':
-    # Code to handle sync state.
-  elif n.resource_state == 'exists':
-    # Code to handle the exists state.
-  elif n.resource_state == 'not_exists':
-    # Code to handle the not exists state.
-
-
-Example of unsubscribing.
-
-  service.channels().stop(channel.body())
-"""
-from __future__ import absolute_import
-
-import datetime
-import uuid
-
-from googleapiclient import errors
-from googleapiclient import _helpers as util
-import six
-
-
-# The unix time epoch starts at midnight 1970.
-EPOCH = datetime.datetime.utcfromtimestamp(0)
-
-# Map the names of the parameters in the JSON channel description to
-# the parameter names we use in the Channel class.
-CHANNEL_PARAMS = {
-    'address': 'address',
-    'id': 'id',
-    'expiration': 'expiration',
-    'params': 'params',
-    'resourceId': 'resource_id',
-    'resourceUri': 'resource_uri',
-    'type': 'type',
-    'token': 'token',
-    }
-
-X_GOOG_CHANNEL_ID     = 'X-GOOG-CHANNEL-ID'
-X_GOOG_MESSAGE_NUMBER = 'X-GOOG-MESSAGE-NUMBER'
-X_GOOG_RESOURCE_STATE = 'X-GOOG-RESOURCE-STATE'
-X_GOOG_RESOURCE_URI   = 'X-GOOG-RESOURCE-URI'
-X_GOOG_RESOURCE_ID    = 'X-GOOG-RESOURCE-ID'
-
-
-def _upper_header_keys(headers):
-  new_headers = {}
-  for k, v in six.iteritems(headers):
-    new_headers[k.upper()] = v
-  return new_headers
-
-
-class Notification(object):
-  """A Notification from a Channel.
-
-  Notifications are not usually constructed directly, but are returned
-  from functions like notification_from_headers().
-
-  Attributes:
-    message_number: int, The unique id number of this notification.
-    state: str, The state of the resource being monitored.
-    uri: str, The address of the resource being monitored.
-    resource_id: str, The unique identifier of the version of the resource at
-      this event.
-  """
-  @util.positional(5)
-  def __init__(self, message_number, state, resource_uri, resource_id):
-    """Notification constructor.
-
-    Args:
-      message_number: int, The unique id number of this notification.
-      state: str, The state of the resource being monitored. Can be one
-        of "exists", "not_exists", or "sync".
-      resource_uri: str, The address of the resource being monitored.
-      resource_id: str, The identifier of the watched resource.
-    """
-    self.message_number = message_number
-    self.state = state
-    self.resource_uri = resource_uri
-    self.resource_id = resource_id
-
-
-class Channel(object):
-  """A Channel for notifications.
-
-  Usually not constructed directly, instead it is returned from helper
-  functions like new_webhook_channel().
-
-  Attributes:
-    type: str, The type of delivery mechanism used by this channel. For
-      example, 'web_hook'.
-    id: str, A UUID for the channel.
-    token: str, An arbitrary string associated with the channel that
-      is delivered to the target address with each event delivered
-      over this channel.
-    address: str, The address of the receiving entity where events are
-      delivered. Specific to the channel type.
-    expiration: int, The time, in milliseconds from the epoch, when this
-      channel will expire.
-    params: dict, A dictionary of string to string, with additional parameters
-      controlling delivery channel behavior.
-    resource_id: str, An opaque id that identifies the resource that is
-      being watched. Stable across different API versions.
-    resource_uri: str, The canonicalized ID of the watched resource.
-  """
-
-  @util.positional(5)
-  def __init__(self, type, id, token, address, expiration=None,
-               params=None, resource_id="", resource_uri=""):
-    """Create a new Channel.
-
-    In user code, this Channel constructor will not typically be called
-    manually since there are functions for creating channels for each specific
-    type with a more customized set of arguments to pass.
-
-    Args:
-      type: str, The type of delivery mechanism used by this channel. For
-        example, 'web_hook'.
-      id: str, A UUID for the channel.
-      token: str, An arbitrary string associated with the channel that
-        is delivered to the target address with each event delivered
-        over this channel.
-      address: str,  The address of the receiving entity where events are
-        delivered. Specific to the channel type.
-      expiration: int, The time, in milliseconds from the epoch, when this
-        channel will expire.
-      params: dict, A dictionary of string to string, with additional parameters
-        controlling delivery channel behavior.
-      resource_id: str, An opaque id that identifies the resource that is
-        being watched. Stable across different API versions.
-      resource_uri: str, The canonicalized ID of the watched resource.
-    """
-    self.type = type
-    self.id = id
-    self.token = token
-    self.address = address
-    self.expiration = expiration
-    self.params = params
-    self.resource_id = resource_id
-    self.resource_uri = resource_uri
-
-  def body(self):
-    """Build a body from the Channel.
-
-    Constructs a dictionary that's appropriate for passing into watch()
-    methods as the value of body argument.
-
-    Returns:
-      A dictionary representation of the channel.
-    """
-    result = {
-        'id': self.id,
-        'token': self.token,
-        'type': self.type,
-        'address': self.address
-        }
-    if self.params:
-      result['params'] = self.params
-    if self.resource_id:
-      result['resourceId'] = self.resource_id
-    if self.resource_uri:
-      result['resourceUri'] = self.resource_uri
-    if self.expiration:
-      result['expiration'] = self.expiration
-
-    return result
-
-  def update(self, resp):
-    """Update a channel with information from the response of watch().
-
-    When a request is sent to watch() a resource, the response returned
-    from the watch() request is a dictionary with updated channel information,
-    such as the resource_id, which is needed when stopping a subscription.
-
-    Args:
-      resp: dict, The response from a watch() method.
-    """
-    for json_name, param_name in six.iteritems(CHANNEL_PARAMS):
-      value = resp.get(json_name)
-      if value is not None:
-        setattr(self, param_name, value)
-
-
-def notification_from_headers(channel, headers):
-  """Parse a notification from the webhook request headers, validate
-    the notification, and return a Notification object.
-
-  Args:
-    channel: Channel, The channel that the notification is associated with.
-    headers: dict, A dictionary like object that contains the request headers
-      from the webhook HTTP request.
-
-  Returns:
-    A Notification object.
-
-  Raises:
-    errors.InvalidNotificationError if the notification is invalid.
-    ValueError if the X-GOOG-MESSAGE-NUMBER can't be converted to an int.
-  """
-  headers = _upper_header_keys(headers)
-  channel_id = headers[X_GOOG_CHANNEL_ID]
-  if channel.id != channel_id:
-    raise errors.InvalidNotificationError(
-        'Channel id mismatch: %s != %s' % (channel.id, channel_id))
-  else:
-    message_number = int(headers[X_GOOG_MESSAGE_NUMBER])
-    state = headers[X_GOOG_RESOURCE_STATE]
-    resource_uri = headers[X_GOOG_RESOURCE_URI]
-    resource_id = headers[X_GOOG_RESOURCE_ID]
-    return Notification(message_number, state, resource_uri, resource_id)
-
-
-@util.positional(2)
-def new_webhook_channel(url, token=None, expiration=None, params=None):
-    """Create a new webhook Channel.
-
-    Args:
-      url: str, URL to post notifications to.
-      token: str, An arbitrary string associated with the channel that
-        is delivered to the target address with each notification delivered
-        over this channel.
-      expiration: datetime.datetime, A time in the future when the channel
-        should expire. Can also be None if the subscription should use the
-        default expiration. Note that different services may have different
-        limits on how long a subscription lasts. Check the response from the
-        watch() method to see the value the service has set for an expiration
-        time.
-      params: dict, Extra parameters to pass on channel creation. Currently
-        not used for webhook channels.
-    """
-    expiration_ms = 0
-    if expiration:
-      delta = expiration - EPOCH
-      expiration_ms = delta.microseconds/1000 + (
-          delta.seconds + delta.days*24*3600)*1000
-      if expiration_ms < 0:
-        expiration_ms = 0
-
-    return Channel('web_hook', str(uuid.uuid4()),
-                   token, url, expiration=expiration_ms,
-                   params=params)
-