missing import

See example failure at: https://travis-ci.org/github/jay0lee/GAM/jobs/667171534#L879

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - pinned
+  - enhancement
+  - help wanted
+  - security
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs.
+  
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.travis.yml

@@ -0,0 +1,293 @@
+if: tag IS blank
+os: linux
+language: python
+
+env:
+  global:
+    - BUILD_PYTHON_VERSION=3.8.2
+    - MIN_PYTHON_VERSION=3.8.1
+    - BUILD_OPENSSL_VERSION=1.1.1e
+    - MIN_OPENSSL_VERSION=1.1.1d
+    - PATCHELF_VERSION=0.9
+    - PYINSTALLER_VERSION=3.5
+    - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
+    - secure: "J9380tGLOZWa7dSH1y5Il8T5JQpN6ad81gI6VR1HIU0svpRdjgikyDA7ca2MKYDUYYY9yVSkTV6gCl6iIU/9+SKaYugpP+tkvdGYkC2moJdcTgYM/WOnIK9ExQ3BPhN1neGxJjPTwKo1ft27mtZ2I5vuCiBwIcnKWLnKPyW3PD+mWpfqiLuEzkHoAh6G3jC4qbcCrZDeX/knE+PzqESUEi+8k1G8gYcSDWujba9ypSsqZ8T/MXagGla6l7y2Rz+/KZTJmFHwKAA10V+xPLVqxoiqi4ar66yUqy0BamwRXPcseI+ns3Q+4lUpMqVQ5GlRy7LF1xC8myjmcAexXk0F9hg+CMzewKI8UgmQH/ZJvQZEh8s6mW26+CqA4d3zMQkWaR0WtEtpiuH7AGHCflIqvEQ6UiG7ia3B8iZfW2wl0j/kqx4OuHkS3r0pWKVVIIvCj9Ow2BHP7SpiV1AcUGsVxzwbgTh67fitna3Z3c6Uj8ccQlNr7ZIt1az6Wf3w5njijkLOiBpQSLKunTTCTSge/JzBTKUcie3RE9vzirl58gUxAt36nDtPWnory+RttMZrOkBVbTeSxp+IUe8pNwLFPHABsafXsjkfzBOtFmm+0ZXWt2Rlog5NvlemJfQUWDlsL4g+BSakzN+4sIPKzSauWDHyaEeULY7Uprkil6c5zwo="
+    - secure: "szcjWHPr0Bf1KCkyTrV5Fu3ADhWk+pg8YWucjXHdybmhaQIKG7iBNg8LJ5d0OBTwAg31wK4ZgyLVSa2gKrAZ3UeDjykJFsR711xDSQOod51Wrgqu4FbXDewE817DUk3Cwe1l5DCu3/fjEw4vbm8B/qb7iMTRKCq6hJd97FwT5oauP0QHNPer9JjrW4F0Hk9ttkgEU2dXWvBMsTJsDOGNI3ddABE2HskxV4T4thelDYGKBDHhUOAsRwSjXgWy77Tvz98psPIvd+6+WPYNRdRWcPDyAR3Z1O/fNjUymrQI6eMaHoSFrmhDS5lbhjINRfdUmECyfCfIFeLWWiw4g4bq7l+4HBORbei55tAIjhEsxJQoqHi0Q5dD5TFh8IiWqowkFbpvNonMSIpKtB0cyT5jU1G/jRA7MPcIvSrdzHaDkoDNHJgAeZfgjOhzTGYYD19lGIljz5BQBcNFZY2dJbja+Jr4He2CMAOBOdERa4Zn1VyNfOmd8Bn5hu0C9D2ybnSCxjXXq5TRiktR8X7WycVZYfqMZXAwP9FEHVitJ4MZEGUc7S92K5gX4wmjcJjLS+Xo/0nsduQm8PuiMjbcPM7/oGx8Xm1KuSfHdKWMBoaesPaDvRX+YcuiNstXf1DkCWl72TsFABzddlNUMl/s2YSKkCSHAJ5ILqrB28Gx89kzVlg="
+    - secure: "CPsDgSoZIHLyjpUbYEsx1GbB+UZcPXCEsz9qT6XRM+qMFKLlSnHxoJ7gMrJtqfjTh+1gLB3UTjQjFr+jAenqLWzaJh7Zdtpg9BOG6aXC8CAi1h0U5ZMNSA/+5lQxOXuhN8HmXI1r8xPNRFXBdZFea+072/2HJTwmgYlVTTQ8FrbXQJCzs2cFqxnVeFmuG3N49AJuoy3B+P2DMpqPzABbQt7Jf5H9Foq+16iXxhRkYA8/8H2nF7ZE8IdJuRpqhBUoPF/8Mt2QBLIlvNIdIzFEy2O7ZhwL9Dt08AG9v5c91QPzjsLok2e+5hFGaeMpGQJCE1V3uHyrOAeyZs1QYr7qBWbjQCVh0Phxz9yOA9RPfnQdjyJTq4QEj6vVYvCi5K/CGp/DnnLnGyYJKZOt7nBx02fTVI136UXqt29eF5NRkCpnUqah2s0OXkijsw5Y9LsbwiUxELKtCthESOwN8e/ZNvn/gjPfZWIaB0gupRNugL8Xo9Cx6vFmaW8wzm1IutJvo1mWvkWMvuYYjvd4aDyP6s7PFnSX7DoD/pfxoQuzjwHO2OD93nCOx0ofnNojLNooeJPKLikiwS4qKc8exWd+TlnccKSkXO8Y8S+XRgeE652YNlf+9DH79lNLeK0N0W2tRExX5JaEyJQCuvK0WZi02kxvjmUHhwLAOv9ueC2UCRs="
+    - secure: "JMv9mkSQBJXvUznXcyvngFaOfd2fHYEQQTH8BnS03pqAL0HkWSheG/R2HFJlJv1VJ9MDMf+wVzwMvU/kzkOT68nIgyFWnLBkT6fw+Mw7t/dG96/nOh7DPuaTVzKS0xRbMhDaqZOA9GW13TVnpVdIw79vbhQM69N9Gj80j8oM1cHgMi+fkJSDU3EN/vMJOGKSB3DTpyqAG/nYyZ38tLib8ic12za/YL3HIu8QRHa3fr37+cyrVKgGebGg++yK34p8lC1W4apiO30drmHVQhKWrWnmdcssdGmVM7NystMGGAzUwsJcmRFJuREv1LXiijDzjIRVduceeRYgPi1KHB5ZdL9vji0gM1eHYZZefhcJb6WgPDbCtbjdlCId4v+1bNfsh+dMmhM+vBZDtDEt3UC3MBeYTmklQT2w9zCXHuPBQigj7W4zLm6GxnAXife0SQMfmr736QBvSLUJWtd2tRgS+dRG/LvWxrP5Urvfgs8iRtEVZLDpRR+bSjvLs1UEKLN13KMKYuVwieHbxJn2kY7d5wmZVBYdaokl0yrTkZZ6J9xIphxM8GXJU1BnMZY9zn+Xq3jm576QYnNYUwCtUjOis00Ct4UVuuFqIqZ7bJL7GT4AHTENAk+9GtXVCo3/jnv161rxgoSdnUG6VmrpnyA9jSpcIvMVcE129oghBJ2+PkU="
+    - secure: "otePbb9W6CAuwzayL0dDAvCryzF2s5HHAIytvMW/xnzOuRMR03lumj9bhg56YCB/nIuTncAjToYmsMh2Acrp1Xcx8/DzqKse1K9nkfJgsB+EYeXmy8pVJnCxhHeLCzUPJjUfRoPBm+5GHAkeoviUyxenwbOFm3/Uhtay7i2ZU8K1FsECqx4FZXVh8rrGHqFGzHNimqDDvFUQM6f9a+BevEio8+/aDooaFfvVqPixDYXpi2+99DB/0hZKwcUpj6pOUUszATkAl0kJdOSWI6E7bQa63i6SfhcBNHfvbDqhpcAGs1TIXHbXwWGfySABKgT9mkq64CY3bc/QZ6HeWe6P5tJmt4mGBxDMOWMYj4qtCskQCCCiY9+sCx61Sj8W5w8exjQvvvFA088cKtbCKAx8FIil4CuYPtIjDQFkiNI2bC6BmkUen7qe4z7dSF2AZPb6CuSsBjoXL8Ezle3e1pRrkR+SYbxMSaZVcQoG8hinTqaXhlS5gXi97ZzRrJWn3tUB6JWEBeEkIfTJmrJolbLYIMcADNhKenrW8k6nV32JkvMsCgMFCIkJsIZeuQ9ZYgSZ3CXPODux/D2/4/gNa+353Fs3DTlDQxRTgoaYi9UDIRrKZAkoELFclxuGSKMCCheevQ8FMCmXfxUocwLSgjlO7g4rTJa9Kggn7VltucXiVWY="
+    - secure: "WKdB+WpZG/7SKpPpgM6DAwzVg0QQuxFMJEZE324pagJ74xv+GlObTLm6kU4SAZf6TuRSChTXDAD4paJXkxpq3LlXjp9aKxPASbG5PgZMnGL6bZ40c/UUqrwVy2HLnXPKSNuDBX1RidGXmH3u4YvufjguNnlOLvHYfMHIRGjiEL4ZvC38GB/3YDmID1zI6w8NzTCpFvxeNdg1qhhOBUKyt+icRUwBu8DfgLidzTrO0j5jo3UeqO/w3E9t3nnRgSiO25mBYwWySm1QBK1VS3F5iG9jo1J6GKMOFbWi3lOBoqnWotpfwID+p1vMNX35phHwmMrtoBYfMTV4A2CvQpGyQhn58qyMoKnhLz+NIOxlHN83qcu65bTMG+7ji0pgUl3jhp3ZPyWUjRYQpoVAg5f3UAnUzJgBrOUC0N60ukJVacT/kvkO10CLfz+0+eefW53r+GCkTi2m8iDezZn1olinpLs+Mt0M4VfQ52RVtq3WB6uxN7RjgtrK5XGi2zVAsvfHp+vqFQ6uu+iioiWNji88cTlMuKheo+FWwozNQzd8+4Vxe9w01mLek57Q0dpXiKWL3vMS4Qc4T9E6UzutllDyg/c61LW6Afcqw3jL4HN0UNG3zFefno47oKrmB13HcxakLEC8sdNfE1kWGvB7jqD0snXsz/8rz7rf0IqnK0kaCAc="
+    - secure: "Lah0Q4bWyEJXTBLZHCNkuEuU5wlDQzLF6aQwW47RyVtyRDYW8uQRQFkbb/3oj4QAAgXl0sOcDFnRaaWmlOStPIIYKKVY8Mk6ufyDCz8inWoPNJoSbdqbAi761HacGUsEfSNDrPNB7fst47UZaX7WNAO3q1QjCCGSEJQVpJE2MDAjcSJhAgLPt3JRFSOSwvqxcDLZI+wF9AM81OWNp6QisAdI4LPUJK3L4M4dLh9+apFRb1OMld++scWah1SB5Qj6tPzMvX96mp1XfWB1fJXuf8Yvks02H6ZcAXubK2HXe3iJZKVqGB67kShNN52P+wg0ZZO4OP4kZ2PXahnB7hhxIfEfWsNVy/w4ww1N6K907BfT6RDmsgNP1ZP8kpq6H4pTnWgAy9WDxjatzbNFAHtMzakGjYJNFxZJVco2FL0ipNQ7htoVAA4sUb+VbRQv4O/oZTLMNnnco19+TFJzuZuS4Rjxj/zX63gXkj/W7wou3hw+NpI/PL2hUXIc5imXSLfVQwri8Nl+6IOjV2gWR/vR1VhQqbLsTF6TZQKNI4lvbRTs1nNeqNMW1lHizI3r9Kernj2noAsxJK7wFTZ64OUkQvSNphfmVow29JYQKXbpxbmRrdnmfmtnMsxUngpx9WsTPhrprt5hIAqiBspHemrs+H3LiIml6IY/3l9bcPlc0gA="
+    - secure: "sNc7kC0CuH/TCZh2WwEN51GcA1fSpcliCHpFB+WX7ieQiRu3xKn2avby/T7vbvX0viXRER59arFGQF4i/dyr2g4tlZLVRYjPeiApfduKZ7Lb+vZGro3cWesfHG6Abk2VcgZZli1IDrgrHH5qAWnA9xNnKvKBL9NDM73Zj92BmlDFVEzadTii8brWvced/YP3jNXEmM5ZIufgpe2yidBB2bLWYJXb3Cf1MvzMG4tqNAtZTrI32q50mokz/uTqp3MRJ+cR8sOI+2+2xSbT0zZGLSRZf96/7FKtE0QIDxdWAe4XdlHq1CluRVk30Ju5BEn0QzoYLryCIuw3JjDl1Yksw4IA5imljZJlOmWa2l6fX0HNxMw+z0R/1d2HARA8BY7/uQKv4guV3Cf3jpsWoKSsM1WxqOqsuEFOoRQ2eQNJEaSuC6+j/vzNoj61pOuG0R9OC2PFcFCZ9fomIrZMse+7M3WIj4+mp7e+JDK8DgVdUlqkBVCx1Ospseb5pm6lDx8F5NbgqZgGXgyoWVpqZnyYOoOutezMoD6MI2wXzJaepV/L3+LD5f6q3DAa/sRAEEsBFGyMHXiPYbziEiy8Hz09Sz3inT5rzS8OLOinwAI2sHiIYHTl340XfWdYz6AlNhYLCGwwmtkntbjOj5UVW06IgBBx44ujpZSUjv7SOrACPGU="
+    - secure: "t9/mC8eSsxXIB2vjcZGtGISxrSY3Yd+XS4+/i1YG1mxSov3R6UdhVl2blLgrvFfVCSo61YMAzJbXKtZlXPzhLDXxpsSlWiatjNrKKo4D2unnlHsyEMJ4wfz8cJ8pKyynP7Kc1ZuaqTSMcEZgk85tlew8Zy/VH6g3Mc/7DvP4SxEDRsP+DdCplZc0vbxHDaV3iC93bwNRfy1UQypwJJ2WQviRema3Umneg8hVl2V35zbaBoy45ubCkUoCbRCDaUyoHA10GrE1OUOLsioar/dj6K2W3EhAtehHLWrUZhhl07rayrrRDQYDTdebifB/MWlvR5FjM/Dr5M4k2ciss4ol3IR5LypRLD+/YBtzeIuqkDbUkaBowY+oUj6OWlzEbzAUrUNa5mnyR2jhr8ivZUeEEWLxljsu8gWq65mzgiZt/u+kVCnMLciUv0//0nrsNsEMI9pau2ZxbcpItFVKdZYXFdmHWG+qPCgMcbgsUM3xqaIc7fNwbk2Aa6erIGqD2VkwWzD/xksweg4lsgQ0N1tXMfoWWKh4Xj/OFom+S+3w9uSx/jQma7nXm+PKQL8dIo7rqza3fz9biq5T6mhwUrqCpFIJv7mrMwaTT1UfOchjiLL3CGeO7Amv7Avmh3fhMPbypF0sws79d7ewZbyv+oSzn+pxlCdzBU1GYx6veApbzhg="
+    - secure: "ljVx3TgpBJ/ylMKDVmXabi9UNi5YvrRM5UBTrRk7XPu1YFYS4FI1GcGlyvYhToc4fKt4jLhX9qU+s/rZY+odO0x/HpmJglMBCrY+QcWOzuyaP1U5dCET+evuqFdEAZIzLQc4VDjL1aQLZh+OG7bjoBClVAan6a+pmW0yxBC6rNtCWTESG4rY3wOeTpoI0Q1gM7gg5Zkj4Z+yLvYeJdoKHijM7C3/R/VVTqUFqArk+Js7Qb2qTqm03SHP0ahRQA8XSfbPebSkJyX9oLbidanBEaQE6sqnp9Qh+8VGcnn7VkSu6oq2+ZXz4xlSMrH2Iv2JXl68Td51LsLo9BxaMCL68ssgTFfXPSrrcLwholNEt1pXk5nhBl1l6MZ1UwUJyBm+AXZp/4sCK9/P0rGa2d1rOcpOz7nobH7BDktqEJkrR6VzkTMx1aOwtF+JSt7SJQ1RrRdm9uKfOZZsnw17+VgVAHo/ttY0C3cRl10oaF1C/IdliDfa5gJdZ2VSZtJxyewqKwGiZrqCRv2fQyIuGsqfHXsyHVL6q1KfVcHjaXBvh0o6xZ6duieFT4FNHg7clv1qPQV+cLh4L11nugiihRTeYQtKyUnP5YIL+jlcGNM6KqKhF9RN5c+zOqWNmEcz6O8nljY5mFWdIxL6dFfE3+4wpw7snFP0PrWIWl5SmrU5ipY="
+    - secure: "L0ivSUmbOHNmKWVR2zeYcVR+Xr6780dYA829MyksXfg9OrieTS+qVqSODKexFW89dp4Wf8xFdT9f16xSqjA/xSuX12uiISMiTcFKP39ZnQZ//NDkx1T4pZaeNiysqT+2Ys9OaKTWxn8luFtAYp+DaluancQtEw6+M8H3jQyQZimGHl0QB6BK2A5VA2vko+7ebIdwu5Df5E7lc/LG42H+DdSFkSj7Meu5XMyPHSZRAdOE5doO++tqFKzgs2WbDRHRUP4R4LqtDlGn8+5qnQtQKUN4V9UGrKM95R34BhgUlgOqOFAjFDug71/1/rRyv9XnzKkAdTIbxV7nmxSL+APhQDqpwPr01EP9gtZBOycswV/igJYotgqkhzwNAYrmwOA5Ta8S18Ck0feDEqT20w8yKS4QwV2Ihg4q7CqDFu7i+9iSRUCd/RVrGtG0U5Zo1b3+1JcNxXH/ErUeyHPfrk4vktxfHmU+omqwkfvTRy5upn0Ycr57YHOJc/Iyur7vE07HcnBfAwV0d5KO6HJ7M1n6hmMxeyKmf+qiyQQnphySvHHAfa/9Sec7omwwKv4bn33DwFGc8GWvL6cZXWhmGhFvd+LslpZl0vZ+7Bz0tXlAg4t8V48y/ZtyoSpny0HP4UstdA43PttvZ6q2y8LUNk4LBP5btKmp27Fszuj/rldtj5g="
+    - secure: "Dox8JthAJqWT9eh3Jt4Morbf4pGN9OjduJXe/lYMsmFvqNg7b94P2QdumWBPmVjDq4YjDVirMejBA/TNwORgKfg7pI7MOw+qqoHpT9xyPecXi3ecyBay13e16p0GNRlc5pUu5JcU8sgCpttvM0EAw6bOuQIhnnkIFesbOvwoxGYjMzjmWMNuikR3CjKbo0LDtD5NJXT1OMSqrRuh8NM/BoKyn/kCdSaq9wI2GUMbkg09/kFJkQOvtMXPkM7dIhr/9UC0ouMIyqe/MHa8O6Y4xESdqiTql9uz1+eZfHIRrgFlHfxDvkMv87Cx5OuL+O+qeT/a+RYLCRJspMoq94IYHGg2iyEfBO2YAkogl53wiEf2KF2JdiNGT0xId7bxCJj3efTuCAXV1oqaHpJli1Mhvs7zPEtf78B4tkWEgjhGr5pBLIlbhNjS5wtTHJX4BUzoiP+wODj4h7rjPAah42nWF8XOMlboVi56sOCLjiHBOvYObqyhSfiQxoi2XHphsrZqw6H03tr4Kqd9HVmuoSvRiv+NOu24Ubr6MrrQM2/G72TrTx0/aBlt8Dx5nx2oWZ9ZMiDUR3XlvDLUi45SpY5qESXz08nRlcdS9EvUpK7C+77bNvX+A3dIhsxnxuNaf2naf+QnYYbvh7q4Qbrj4v6EMYS90Uky1JHdoc2wMua8J+w="
+    - secure: "Is2Lv/rxKKrXnxFns9KQYseD02tjY9qbgSteVtJavG1cLJDvkTwb6X+Thvgo4cxk5fQPiXScrQaYzHjVVuNleD+dyD1HC/8CU2Xq+tjBhPjdcccHFSbk06DpcETmLGRyMORXG9JkYlgXHLLXtu/9icppWEHgra+zvchVL2YDhofYne8FMNBb/lq0AAC2wgzAS33tW1+57HaYnzl0hf6+Q6lwqoH2/aTfGMRFDgyJ0HK+5IVfLnQJ+OuGFSrj8/0FWSggR5+EXDIddDgovFgaCghMjHYp21bzn0eIAJtuNFFultwk/UC1lT9joXTKEzgLTAh+w13yz1T3x9rNuv6FDKCotBIS/ZDtPmgvyZ8xvB4SzyULnTRSVn7YvspKR6PAO/qxGNudUD5H8tRKer4qKnKjHzSUcVBlRHb4yE6FqqY1Z9RLEcomWO43nJwb6saNHR9BYedyi0gA+EbA+P259QFClW1dWEQ2LQhDa+0VRssOqZ0BQblPFyz+e5Vc9kfAMbOuoss8fjkiYv+twXv7nT27xrVT5okfKDSiy5opZD6d36N5FibZPYiMrVx00YZdkFB+5EqQuJ7lqKUMkZJTeApLzj+h+/4aAOWd3paj5ghv7m+9ReohsNKFHyjaSy97RhMAZjzqgMMdD8rjUSKDhvNKvvQECWHlaXwL129GB0s="
+    - secure: "l7vWxfcu1RgXbStq76Mzz2I5Iu4e31729OyLYqulXZzft3wO+idvgQKy/JSwajiKgOxlpBuI0wrncgIUYshcRvE4yB0y9+QIMDTegJzTADtRSUyVNCIZfTgvtOvzrlW0iCdVsxLBtYcJWJVPdjF2q59ED1jahd1AuJJqX5e61gfr0eZ9+cNiHbX1u3VpmGchFNWQF4KvebE4WKs5xWEds+AtbTODdQq3H6kKQK3fTVJnbz6WMO+bEWgWI7orfSE20lku/3Q3eMLhNOcPwH8WnUoTTvDWol5Cq5NfPhkKF5aV7kIbNXkxswM7yBPAPumBiXdM1BHpfd4+0YQ/fqRtnqxw85HEYpT8dRewHimCl4IccgGCR+G0tK8RNleKL28GWrz86gRVpXMEhOU3ILwb8as3SdQWLwhXXy5uKGJmsdrCNAH4/8eu3XczO5VN2wOo7narYuBgGcl2eLW9TOLyNVKFKxbQnDLBiOybLNoOV42DCRYt7v3Eknkv1Z0dmX6n23q1z9if9kkfAFgRQWsTbNZyWeIwWuX8b2a0Zq0znS3JflSKxzqS7RHADfBOJEVM86AiGkw0XkR4o31yDrY+zOrkJ3GxfV/HpqG7LzCd9tFeexanneDCE1FJhCkDjpnc0Mdyx+1gVf+u2MkgrU0BbCf2EESBAlC/FTRvxTmk/6s="
+    - secure: "bvtQ348bxKwTtB8X0zMxeTsM0jEJozbS6/rzH/88Fk90a+KO8SdXen0Kj9/LahV4duMn2xTTRmxMCVj424FbcVTgBkJpIO7btqTcNASORkmK+9wGTK6Bgb9R5sHLbVrbrMYJzsMQR0MWxE8ibPLlyom+ssUIqr7HAjnRyYSDiKChGhgBfdE/0G9OE/DuQaU/ZkTuEYfc4527QNLJ8Bt1aLDdCHJLxzCojNaTHB215Tz7dmpnJjWa45BXxkMwLTpxIJuY7wA7K+2UBJfLvZv5QiPYyeUlosV7FwhCN9e90+dc2x8HPJbwe/Ysv5dm8/FfNTgTe6IkKw1z9kev9/W1tYCtqMzn9GnlIH2000ZUomhHKbc4UUl8sskF2jGr14rSz7pfaTyvXNvK7nKLKM7mZAO/cAaQvZoGEx0s6B7a7YX38NmJWcM0UUD603n4G9uVZNoxjyr8HWC6pho8MY8/u5aFKMPd+C8DaDWeSzNJDZRyi62v+t2iyyZuQUtVXtnDpv8GQW4tYFIJCkkrTm5VNNgD2x3WRsEr3LsBa3BonOmi3a1VrdIpyidDzBEWFUR59a/C0lU5J2UX5W1vu/Pnr9/8p898sDvvDbluz47OR9xoVOjtmGdt/ENJa4EOY1q/eRIN6zrt/cwwjIKgTcYOVk7DFDMXQuYlDeEFuHSJ5Xs="
+    - secure: "VWY3aJGMFB+E5ObJpUUXNQrnkEpA1pbfey8Z0pe9X/n5ubmDBIAfCPP7hqT/lAF/1hj5+HA6oWEa2FzeJZOL9q8cuymHzkehiX9n3f6GktirMtEoG3Hk/CiW/ZxKf9KyBNRVsqsOiXvfzMcMSU6ZrSV+4ZHIVl0zO5cehSaaiTTqVU5K474N5TCc8SAvpSRpxjZVRXy8rWF7WMaG4Xm6aa/IELccjJL8s6sahk3wT++5pzewxv4XBWf4bNA7+MNLpnzKmm2S5T/uohsw00F/j82g9xFT3qYm6vgaR4ql0aNvxd9gZ/6vdVpqBIUZmcFi9yTSFSsSSH2cNGFk3IU9Ecdh1Psi3B5pTpYzy7BbLWyKo70wBkOoY8qCwJ/wtj+2TcfR72qR8ryXgkbIoXxe04VkPqRNEezk3U+UxUvLLDEWthxjuCRCydT8FaCfYm9N+l5Un8TNmY4W9mOSu5IXKJ7oSQDrJG65Z2uZKQrWm0ToOo1nNLFclFuM+K8JWNOKysfh451LHdbmA0rkoaLUat2ttDBtQpbw3h+Hwn3SW6BfELg6WDJXHyAPNRMPSc2Grk2o7mFfTMJhHQG4U+k2bL+AHJU0nPWzuEJyMraquqKOCcHz56A+6j1PN3wpLbIC18nNPN9Q3TDPiBmwzwEg+zLu0DKTu3Oe4IRKRNTuocI="
+    - secure: "BSZK3v2RYsNKv/8lBZ/dKagYF+znGXIeDY271A2nCS8DXx+y0octwI5RT08+UqdD1AvhPOPjdnSXG+M2NOSLlfBoXF/0t9S39+qy3EhGgxYAcWcVDXnrOrHEOALg1cQkCA2D/CG+eeG/36SmAn6FINTLaAOilxWGGCeL0KOB5mqPPnPogaYhowIwDkj0Hfsfrw8011XzgbufNr15GIYJ+nP7f41NcgIbnNkqXY6Q1jiWs1DuKXNDr5WGPqztHtVf7RtcpVmShzptAcAMdRqCGKF7atCZhXPbd69j3qf7F533/tCSCIrszR/Wp4BKlJfVCRCk1oiEcfRvprJqeFJ+0WJCVzyD7hXfnkBg2Tb5PmzvxOFM+hsqRu4mY3UdjqXukzAe5O0O6Uo8sqzqQUIjUooRnNQ4GdPd+7wMbyBZn4PJaW6YTi/7zg7mAegqot6uyEGGpWb6iFYrf6DX75GUfTciNktv+ez0AeqYFwNBLgcOsmaq+3V+aR+dIxsxIzC/i5GCTHvMYOIp6Q2tPCuIug0tX6uxm++vMOoMLK+vG1fVQ0Cd8m6yQIWEXUu0VBs1MTlJD+vX6LsK8CGo0Dt3ZS3JmC4TGGo9CBSEkeDvnKvQeX6x3NBIFWvgt+Hjxh4S9U+vW+AaUXpV9k+NEhesC/8Ys0UGbGYsTDzHdx9TMBY="
+    - secure: "sskZYJ/+xmHh5GNvw3QRf10+sBGOjlub29jOTxjXIRYUyZfkhjsF8CR9NP59CBFuqgN716Mj1uEVEcx0aaepr/zWweQSnqa9lZgFD9nDYALVNh1b4oyqfhYG1sw57ZHdBsJOBF4mKBnahly/QQj11Ya25GyIS2IewwCWVNtCOJYietSssG9l0+qc+RPG5Ub/lEKA3VRrtbUuApcIYszt33DumgZ4wzkzICl1SuOy8V15vHVlkBqASJheEDa4Hia+eAMo6e7OCE5KWjl7K2MxTvtszwFi7lgMCyPCOy6DkaULACfBnQeloh42B2Qhd/ta04vuRKg6y4fKSrnegpeOAa9aGxF1ufvG9IvPsRTsu4+w63b9xsQUN9MyDgxcJe0YeUHPgPYL6mqAjCIKvL62LlIyrQ8IAteoh3MC+4Xb8crXaLGINTcLwYvLwxsHMuC/58hdQ23I4DqnyZGAS3L1IhpX4QvYN6xc7H+ptaiQttweoH5VpMAduSrmSnNmvLOc4g2PvbRES6D95moQ7qk7iX6vpu+PevL5HtQCbB8SFyFLTYWsqZaG+4NrjpIi2hLzUT35meHwrvxG/MsfeqOVlnBfa93VnX75vxOkRFNY56sOtrTJaiVZ+rQUdszAZz3KGPbyrSlz3KxV+ZKxZH6/0oFteAJttXFjWQdICnKGDSo="
+    - secure: "sdzU5bPs1w7Nzf3F5Gtk/iq2Kfq3zfLQNGcehLZp1gJXzNf7F6HZLOn6GaXQ/5RVlhqR5nOmzMpVQs88rZv+6PE6YqGMugTxHIQeNmXtGnuEDJSBvGT9Ok8ENxcwKwL93g9SCd9P8mTspxklOsW2Bk3No2+Zlc/aQguBcX44TwYF42KuBU4O7oS6pUg8NjnuQp2zTyhp0ouzyAudatPyu1BLci/3lbx+MehutQw1y4Om6g8tfwf950UONZQdqq9MBluu7yYb1oHkdC1J4qgwdCZkJslWIwQHCH5UE4AW9iVG0qVrLpzBGtV27Kfy/Vf8r2gMYzbiur2h2+zzWSDm8/bk8YLn7u1FBpjGJdJ0pX0ZrZr+hMV2vH3e54NvA5WyRU7tw8mZ1PoDYd/FM5KXIYbscSbSRCqTsbPgyVIHuoOWXeeSe+/Ef1ifZv3HHf6ARfUIWupKfAipxChc7QUMI/HEQ9QPsqgBaooZD9chGsWAgv+8tFxdteqkx4Yh+AuZp1rVykB/9vAamUBebQxy0oeGm65j6X1rksfjAPkfeDYB7L4Ruy7tUwPtvYrAHPoWrf9O0g/qDRsw0vdqp42CjszpIxhuPhVRDx0i0wquqw2LnIU3ejRv2R8d1SecVKBBcVsWLFvc9iR4rNIi5JnxITRtiwL1Xv3Vcgx7WwxExtE="
+    - secure: "f9n1KmU5NuV4jGkrhLNiPD+3Cy7t4D7Rq8YsPlmyB2A6u6IHMuOVP95IwH6Zt0cmMDBrZGthj3/0iu5kzxzqD0m135PT17wSEqsDfDMyKRZJQuYSO/ESVxnSca3afRH7Ds7/ipQVd/ljZgwEnW3JMaOQiIdbbIquNOOTeY0/wsXkreDamXZaKKqUoeadkAV4AkKhM0xcMg+Vni1i71TYPBWrZPLVAu3ZrSvU/cE5mtBUIkbr9EgsEE+WR23QCgtwxKzNxrXetBcPXDsJb98/ABgpoItm5Ko/Zk6pkib44f+iQtn7Y6j3lieELCH5Sn0uy3RrMxkl7xicB7zPYME94NEPHCmshyVsH3RxWfcBG4kauRNBCLYLl5HYF2t1lWZ6In5qlx8xN3Tf0KrbM3vzAEOnnfZt83h3q5OuWl+jzTOcv9xHmeW0lnwEEfS0nxUV3KDqLFBczcUxKBmsA2aWnJZ2HV+kls6OaWZ987m6V2pIGR2uviGT7I4ngjCSOJzbwYbvJbJqYAI97FWP/5pqv97xHLCSSJh6TBO4NMQ7Ib/W1XT6NZyPOjNGSLpd79UA3cTzU2+UYr/7RxZpAKONSAMTJh7CX451XQwgovpFZ2quXs2BzqcVpy8AlUc45ygnFOALOANAkcRP5QFhmff0jpXstjPW83/GksbtaiLhIiQ="
+cache:
+  directories:
+    - $HOME/.cache/pip
+    - $HOME/python
+    - $HOME/ssl
+
+jobs:
+  include:
+    - os: linux
+      name: "Linux 64-bit Bionic"
+      dist: bionic
+      language: shell
+      env:
+         - GAMOS=linux
+         - PLATFORM=x86_64
+         - VMTYPE=build
+    - os: linux
+      name: "Linux 64-bit Xenial"
+      dist: xenial
+      language: shell
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: linux
+      dist: bionic
+      arch: arm64
+      name: "Linux ARM64 Bionic"
+      language: shell
+      filter_secrets: false
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
+    - os: linux
+      dist: xenial
+      arch: arm64
+      name: "Linux ARM64 Xenial"
+      language: shell
+      filter_secrets: false
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
+    - os: linux
+      name: "Linux 64-bit Trusty"
+      dist: trusty
+      language: shell
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: linux
+      name: "Linux 64-bit Precise"
+      dist: precise
+      language: shell
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: linux
+      name: "Python 3.6 Source Testing"
+      dist: bionic
+      language: python
+      python: 3.6
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: linux
+      name: "Python 3.7 Source Testing"
+      dist: bionic
+      language: python
+      python: 3.7
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: linux
+      name: "Python nightly Source Testing"
+      dist: bionic
+      language: python
+      python: nightly
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: linux
+      name: "Python PyPi Source Testing"
+      dist: xenial
+      language: python
+      python: pypy3
+      env:
+        - GAMOS=linux
+        - PLATFORM=x86_64
+        - VMTYPE=test
+    - os: osx
+      name: "MacOS 10.12"
+      language: generic
+      osx_image: xcode9.2
+      env:
+        - GAMOS=macos
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: osx
+      name: "MacOS 10.13"
+      language: generic
+      osx_image: xcode10.1
+      env:
+        - GAMOS=macos
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: osx
+      name: "MacOS 10.14"
+      language: generic
+      osx_image: xcode11.3
+      env:
+        - GAMOS=macos
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: windows
+      name: "Windows 64-bit"
+      language: shell
+      env:
+        - GAMOS=windows
+        - PLATFORM=x86_64
+        - VMTYPE=build
+    - os: windows
+      name: "Windows 32-bit"
+      language: shell
+      env:
+        - GAMOS=windows
+        - PLATFORM=x86
+        - VMTYPE=build
+
+before_install:
+- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-before-install.sh
+
+install:
+- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
+
+script:
+# Discover and run all Python unit tests. Buffer output so that it's not sent to the build log.
+- $python -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
+- touch $gampath/nobrowser.txt
+- $gam version extended
+- $gam version | grep travis # travis should be part of the path (not /tmp or such)
+# determine which Python version GAM is built with and ensure it's at least build version from above.
+- if [ "$VMTYPE" == "build" ]; then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $MIN_PYTHON_VERSION; fi
+# determine which OpenSSL version GAM is built with and ensure it's at least build version from above.
+- if [ "$VMTYPE" == "build" ]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $MIN_OPENSSL_VERSION; fi
+- if [ "$VMTYPE" == "build" ]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
+- if [ "$VMTYPE" == "build" ]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
+- export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
+- if [ "$TRAVIS_EVENT_TYPE" != "pull_request" ]; then export e2e=true; fi
+- if [ "$e2e" = true ]; then export gam_user=gam-travis-$jid@pdl.jaylee.us; fi
+- if [ "$e2e" = true ]; then openssl aes-256-cbc -K $encrypted_ab10ec38326e_key -iv $encrypted_ab10ec38326e_iv -in travis/oauth2service.json.enc -out $gampath/oauth2service.json -d; fi
+- if [ "$e2e" = true ]; then cat travis/cfg_template.json | python travis/svars-write.py &> /dev/null; fi
+- if [ "$e2e" = true ]; then $gam info domain; fi
+- if [ "$e2e" = true ]; then $gam oauth info; fi
+- if [ "$e2e" = true ]; then $gam oauth refresh; fi
+- if [ "$e2e" = true ]; then $gam info user; fi
+- if [ "$e2e" = true ]; then export tstamp=$(date +%s%3N);
+  export newbase=travis-test-$jid-$tstamp;
+  export newuser=$newbase@pdl.jaylee.us;
+  export newgroup=$newbase-group@pdl.jaylee.us;
+  export newalias=$newbase-alias@pdl.jaylee.us;
+  export newbuilding=$newbase-building;
+  export newresource=$newbase-resource;
+  export GAM_THREADS=5; fi
+- if [ "$e2e" = true ]; then echo email > sample.csv;
+  for i in {01..20};
+    do echo $newbase-bulkuser-$i >> sample.csv;
+  done; fi
+- if [ "$e2e" = true ]; then $gam create user $newuser firstname Travis lastname $jid password random recoveryphone 12125121110 recoveryemail jay0lee@gmail.com travis.jid $jid; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "Travis test message"; fi
+- if [ "$e2e" = true ]; then $gam create group $newgroup name "Travis $jid group" description "This is a description" isarchived true; fi
+- if [ "$e2e" = true ]; then $gam user $newuser add license gsuitebusiness; fi
+- if [ "$e2e" = true ]; then $gam update group $newgroup add owner $gam_user; fi
+- if [ "$e2e" = true ]; then $gam update group $newgroup add member $newuser; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam create user ~~email~~ firstname "Travis Bulk" lastname ~~email~~ travis.jid $jid; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add license gsuitebusiness; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user $gam_user sendemail recipient ~~email~~@pdl.jaylee.us subject "test message $newbase" message "Travis test message"; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam update group $newgroup add member ~email; fi
+- if [ "$e2e" = true ]; then $gam info group $newgroup; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user check serviceaccount; fi
+- if [ "$e2e" = true ]; then $gam user $newuser imap on; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show imap; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user $newuser delegate to ~email; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show delegates; fi
+- if [ "$e2e" = true ]; then $gam user $newuser label "✔ unicode checkmark ✔"; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show labels > labels.txt; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user importemail subject "Travis import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user insertemail subject "Travis insert $newbase" file gam.py labels INBOX,UNREAD; fi # yep body is gam code
+- if [ "$e2e" = true ]; then $gam user $gam_user sendemail subject "Travis send $gam_user $newbase" file gam.py recipient admin@pdl.jaylee.us; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user draftemail subject "Travis draft $newbase" message "Draft message test"; fi
+- if [ "$e2e" = true ]; then $gam users "$gam_user $newbase-bulkuser-01 $newbase-bulkuser-02 $newbase-bulkuser-03" delete messages query in:anywhere maxtodelete 99999 doit; fi
+- if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-04 $newbase-bulkuser-05 $newbase-bulkuser-06" trash messages query in:anywhere maxtotrash 99999 doit; fi
+- if [ "$e2e" = true ]; then $gam users "$newbase-bulkuser-07 $newbase-bulkuser-08 $newbase-bulkuser-09" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit; fi
+- if [ "$e2e" = true ]; then $gam user $newuser delete label --ALL_LABELS--; fi
+- if [ "$e2e" = true ]; then $gam create feature name Whiteboard-$newbase; fi
+- if [ "$e2e" = true ]; then $gam create feature name VC-$newbase; fi
+- if [ "$e2e" = true ]; then $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."; fi
+- if [ "$e2e" = true ]; then $gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room; fi
+- if [ "$e2e" = true ]; then $gam info resource $newresource; fi
+- if [ "$e2e" = true ]; then $gam user $newuser show filelist; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi # clear ACLs
+- if [ "$e2e" = true ]; then $gam calendar $gam_user update read domain; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user update freebusy default; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user add editor $newuser; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user showacl; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user addevent summary "Travis test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
+- if [ "$e2e" = true ]; then $gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser; fi
+- if [ "$e2e" = true ]; then $gam create vaulthold matter "Travis matter $newbase" name "Travis hold $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print vaultmatters; fi
+- if [ "$e2e" = true ]; then $gam print vaultholds matter "Travis matter $newbase"; fi
+- if [ "$e2e" = true ]; then $gam create vaultexport matter "Travis matter $newbase" name "Travis export $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam info export id:~~matterId~~ id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add calendar id:$newresource; fi
+- if [ "$e2e" = true ]; then $gam delete resource $newresource; fi
+- if [ "$e2e" = true ]; then $gam delete feature Whiteboard-$newbase; fi
+- if [ "$e2e" = true ]; then $gam delete feature VC-$newbase; fi
+- if [ "$e2e" = true ]; then $gam delete building $newbuilding; fi
+- if [ "$e2e" = true ]; then $gam delete group $newgroup; fi
+- if [ "$e2e" = true ]; then $gam create alias $newalias user $newuser; fi
+- if [ "$e2e" = true ]; then $gam whatis $newuser; fi
+- if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
+- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam download export id:~~matterId~~ id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter "Travis matter $newbase"; fi
+- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action close; fi
+- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action delete; fi
+- if [ "$e2e" = true ]; then $gam delete user $newuser; fi
+- if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
+- if [ "$e2e" = true ]; then $gam print mobile; fi
+- if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$e2e" = true ]; then $gam report customer todrive; fi
+- if [ "$e2e" = true ]; then $gam report users fulldatarequired accounts,gmail fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
+- if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
+
+before_deploy:
+- export TRAVIS_TAG="preview"
+- unset LD_LIBRARY_PATH
+
+deploy:
+  provider: releases
+  token:
+    secure: bzambMcQwyv/o5c5GrKGCsZHgE5R85tg8sNFvPfpISz3+uosCjnBXas7wvCKzT75XUFi2ztfbYak6HdKf4sGnNHk0saEicB3slH+ghPyZbYzp76yvvduhFO2nWW3/F01tL+Yfqqt4/q8wFaWGjrC5km+6GLVyB4lWA/Uyu49qKnz02uSwyhBD/VFbO7DOQ65a1iWk9HngyMsu0Oi7HIbSjSLtxTHedNfOf3waW0NivTTxYXiYGX/MCu3GWhgIGj47a+H3A6FcQ/9QWvnKgnoixdgPBUz7kDb7ktsWwQsILPGStgH7iMuG49ZlXdEFmqwifBri2wvzmFEevBGZjHcupy1IGrNFRG+IUGKMotio+OkLHlLjuv7ZJtqCz/Vf5SNFgNyMSanx6jKEUJuYvndVg99IRXmYVwHFwPu5BAcJACpU6C0AfyGmmSqqwxCd46uXL62ynxNFpHuRfOqlDnmCTfZgjOciJSlDDpf+Xz9fF7+oCoeCi3mrcZVFjhd3tT6Oxw5HrsDtm0ZNld1cdLidaq8H6vOFgHMd0A9yNYZzTzXTvpmxzkXT4Zc7s+PYKN6z5fRZ+pJeckUjRXblvVEfs5HFSymavcOc5AkRwxpvOsTQMNmlnaJCBo5UNs0K/rVmRi5cFmaiwTcBCY0kTllOBJ4zWsfq8seiokWwNUNK2g=
+  file_glob: true
+  overwrite: true
+  file: gam-$GAMVERSION-*
+  skip_cleanup: true
+  draft: true
+  on:
+    repo: jay0lee/GAM
+    condition: $VMTYPE = build

README.md

@@ -1,4 +1,4 @@
-GAM is a command line tool for Google G Suite Administrators to manage domain and user settings quickly and easily.
+GAM is a command line tool for Google G Suite Administrators to manage domain and user settings quickly and easily. [![Build Status](https://travis-ci.org/jay0lee/GAM.svg?branch=master)](https://travis-ci.org/jay0lee/GAM)
 # Quick Start
 ## Linux / MacOS
 Open a terminal and run:
@@ -12,6 +12,8 @@ Download the MSI Installer from the [GitHub Releases] page. Install the MSI and
 The GAM documentation is hosted in the [GitHub Wiki]
 # Mailing List / Discussion group
 The GAM mailing list / discussion group is hosted on [Google Groups].  You can join the list and interact via email, or just post from the web itself.
+# IM Room
+[![Join the chat at https://gitter.im/jay0lee-GAM/community](https://badges.gitter.im/jay0lee-GAM/community.svg)](https://gitter.im/jay0lee-GAM/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 # Author
 GAM is maintained by <a href="mailto:jay0lee@gmail.com">Jay Lee</a>. Please direct "how do I?" questions to [Google Groups].
 

src/auth/__init__.py

@@ -0,0 +1,27 @@
+"""Authentication/Credentials general purpose and convenience methods."""
+
+from . import oauth
+from var import _FN_OAUTH2_TXT
+from var import GC_OAUTH2_TXT
+from var import GC_Values
+
+# TODO: Move logic that determines file name into this module. We should be able
+# to discover the file location without accessing a private member or waiting
+# for a global initialization.
+DEFAULT_OAUTH_STORAGE_FILE = _FN_OAUTH2_TXT
+
+
+def get_admin_credentials_filename():
+  """Gets the name of the file that stores the admin account credentials."""
+  # If the environment globals are loaded, use the set global value. It may have
+  # some custom name in it. Otherwise, just use the default name.
+  if GC_Values[GC_OAUTH2_TXT]:
+    return GC_Values[GC_OAUTH2_TXT]
+  else:
+    return DEFAULT_OAUTH_STORAGE_FILE
+
+
+def get_admin_credentials():
+  """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+  credential_file = get_admin_credentials_filename()
+  return oauth.Credentials.from_credentials_file(credential_file)

src/auth/oauth.py

@@ -0,0 +1,568 @@
+"""OAuth2.0 user credentials."""
+
+import datetime
+import json
+import os
+import re
+import threading
+from urllib.parse import urlencode
+
+from filelock import FileLock
+import google_auth_oauthlib.flow
+import google.oauth2.credentials
+import google.oauth2.id_token
+
+import fileutils
+import transport
+from var import GAM_INFO
+from var import GM_Globals
+from var import GM_WINDOWS
+
+MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
+                                        'browser:\n\n\t{url}\n')
+MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
+MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
+                                             ' visit:\n\n\t{url}\n\nIf your '
+                                             'browser is on a different machine'
+                                             ' then press CTRL+C and create a '
+                                             'file called nobrowser.txt in the '
+                                             'same folder as GAM.\n')
+MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
+                                ' close this browser window and return to GAM.')
+
+
+class CredentialsError(Exception):
+  """Base error class."""
+  pass
+
+
+class InvalidCredentialsFileError(CredentialsError):
+  """Error raised when a file cannot be opened into a credentials object."""
+  pass
+
+
+class EmptyCredentialsFileError(InvalidCredentialsFileError):
+  """Error raised when a credentials file contains no content."""
+  pass
+
+
+class InvalidClientSecretsFileFormatError(CredentialsError):
+  """Error raised when a client secrets file format is invalid."""
+  pass
+
+
+class InvalidClientSecretsFileError(CredentialsError):
+  """Error raised when client secrets file cannot be read."""
+  pass
+
+
+class Credentials(google.oauth2.credentials.Credentials):
+  """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
+
+  DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+  def __init__(self,
+               token,
+               refresh_token=None,
+               id_token=None,
+               token_uri=None,
+               client_id=None,
+               client_secret=None,
+               scopes=None,
+               quota_project_id=None,
+               expiry=None,
+               id_token_data=None,
+               filename=None):
+    """A thread-safe OAuth2.0 credentials object.
+
+    Credentials adds additional utility properties and methods to a
+    standard OAuth2.0 credentials object. When used to store credentials on
+    disk, it implements a file lock to avoid collision during writes.
+
+    Args:
+      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
+        information is provided.
+      refresh_token: String, The OAuth 2.0 refresh token. If specified,
+        credentials can be refreshed.
+      id_token: String, The Open ID Connect ID Token.
+      token_uri: String, The OAuth 2.0 authorization server's token endpoint
+        URI. Must be specified for refresh, can be left as None if the token can
+        not be refreshed.
+      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
+        can be left as None if the token can not be refreshed.
+      client_secret: String, The OAuth 2.0 client secret. Must be specified for
+        refresh, can be left as None if the token can not be refreshed.
+      scopes: Sequence[str], The scopes used to obtain authorization.
+        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
+          not request additional scopes after authorization. The scopes must be
+          derivable from the refresh token if refresh information is provided
+          (e.g. The refresh token scopes are a superset of this or contain a
+          wild card scope like
+            'https://www.googleapis.com/auth/any-api').
+      quota_project_id: String, The project ID used for quota and billing. This
+        project may be different from the project used to create the
+        credentials.
+      expiry: datetime.datetime, The time at which the provided token will
+        expire.
+      id_token_data: Oauth2.0 ID Token data which was previously fetched for
+        this access token against the google.oauth2.id_token library.
+      filename: String, Path to a file that will be used to store the
+        credentials. If provided, a lock file of the same name and a ".lock"
+          extension will be created for concurrency controls. Note: New
+            credentials are not saved to disk until write() or refresh() are
+            called.
+
+    Raises:
+      TypeError: If id_token_data is not the required dict type.
+    """
+    super(Credentials, self).__init__(
+        token=token,
+        refresh_token=refresh_token,
+        id_token=id_token,
+        token_uri=token_uri,
+        client_id=client_id,
+        client_secret=client_secret,
+        scopes=scopes,
+        quota_project_id=quota_project_id)
+
+    # Load data not restored by the super class
+    self.expiry = expiry
+    if id_token_data and not isinstance(id_token_data, dict):
+      raise TypeError(f'Expected type id_token_data dict but received '
+                      f'{type(id_token_data)}')
+    self._id_token_data = id_token_data.copy() if id_token_data else None
+
+    # If a filename is provided, use a lock file to control concurrent access
+    # to the resource. If no filename is provided, use a thread lock that has
+    # the same interface as FileLock in order to simplify the implementation.
+    if filename:
+      # Convert relative paths into absolute
+      self._filename = os.path.abspath(filename)
+      lock_file = os.path.abspath(f'{self._filename}.lock')
+      self._lock = FileLock(lock_file)
+    else:
+      self._filename = None
+      self._lock = _FileLikeThreadLock()
+
+  # Use a property to prevent external mutation of the filename.
+  @property
+  def filename(self):
+    return self._filename
+
+  @classmethod
+  def from_authorized_user_info(cls, info, filename=None):
+    """Generates Credentials from JSON containing authorized user info.
+
+    Args:
+      info: Dict, authorized user info in Google format.
+      filename: String, the filename used to store these credentials on disk. If
+        no filename is provided, the credentials will not be saved to disk.
+
+    Raises:
+      ValueError: If missing fields are detected in the info.
+    """
+    # We need all of these keys
+    keys_needed = set(('client_id', 'client_secret'))
+    # We need 1 or more of these keys
+    keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
+    missing = keys_needed.difference(info.keys())
+    has_one_of = set(info) & keys_need_one_of
+    if missing or not has_one_of:
+      raise ValueError(
+          'Authorized user info was not in the expected format, missing '
+          f'fields {", ".join(missing)} and one of '
+          f'{", ".join(keys_need_one_of)}.')
+
+    expiry = info.get('token_expiry')
+    if expiry:
+      # Convert the raw expiry to datetime
+      expiry = datetime.datetime.strptime(expiry, Credentials.DATETIME_FORMAT)
+    id_token_data = info.get('decoded_id_token')
+
+    # Provide backwards compatibility with field names when loading from JSON.
+    # Some field names may be different, depending on when/how the credentials
+    # were pickled.
+    return cls(
+        token=info.get('token', info.get('auth_token', '')),
+        refresh_token=info.get('refresh_token', ''),
+        id_token=info.get('id_token_jwt', info.get('id_token')),
+        token_uri=info.get('token_uri'),
+        client_id=info['client_id'],
+        client_secret=info['client_secret'],
+        scopes=info.get('scopes'),
+        quota_project_id=info.get('quota_project_id'),
+        expiry=expiry,
+        id_token_data=id_token_data,
+        filename=filename)
+
+  @classmethod
+  def from_google_oauth2_credentials(cls, credentials, filename=None):
+    """Generates Credentials from a google.oauth2.Credentials object."""
+    info = json.loads(credentials.to_json())
+    # Add properties which are not exported with the native to_json() output.
+    info['id_token'] = credentials.id_token
+    if credentials.expiry:
+      info['token_expiry'] = credentials.expiry.strftime(
+          Credentials.DATETIME_FORMAT)
+    info['quota_project_id'] = credentials.quota_project_id
+
+    return cls.from_authorized_user_info(info, filename=filename)
+
+  @classmethod
+  def from_credentials_file(cls, filename):
+    """Generates Credentials from a stored Credentials file.
+
+    The same file will be used to save the credentials when the access token is
+    refreshed.
+
+    Args:
+      filename: String, the name of a file containing JSON credentials to load.
+        The same filename will be used to save credentials back to disk.
+
+    Returns:
+      The credentials loaded from disk.
+
+    Raises:
+      InvalidCredentialsFileError: When the credentials file cannot be opened.
+      EmptyCredentialsFileError: When the provided file contains no credentials.
+    """
+    file_content = fileutils.read_file(
+        filename, continue_on_error=True, display_errors=False)
+    if file_content is None:
+      raise InvalidCredentialsFileError(f'File {filename} could not be opened')
+    info = json.loads(file_content)
+    if not info:
+      raise EmptyCredentialsFileError(
+          f'File {filename} contains no credential data')
+
+    try:
+      # We read the existing data from the passed in file, but we also want to
+      # save future data/tokens in the same place.
+      return cls.from_authorized_user_info(info, filename=filename)
+    except ValueError as e:
+      raise InvalidCredentialsFileError(str(e))
+
+  @classmethod
+  def from_client_secrets(cls,
+                          client_id,
+                          client_secret,
+                          scopes,
+                          access_type='offline',
+                          login_hint=None,
+                          filename=None,
+                          use_console_flow=False):
+    """Runs an OAuth Flow from client secrets to generate credentials.
+
+    Args:
+      client_id: String, The OAuth2.0 Client ID.
+      client_secret: String, The OAuth2.0 Client Secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      filename: String, the path to a file to use to save the credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Returns:
+      Credentials
+    """
+    client_config = {
+        'installed': {
+            'client_id': client_id,
+            'client_secret': client_secret,
+            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
+            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+            'token_uri': 'https://oauth2.googleapis.com/token',
+        }
+    }
+
+    flow = _ShortURLFlow.from_client_config(
+        client_config, scopes, autogenerate_code_verifier=True)
+    flow_kwargs = {'access_type': access_type}
+    if login_hint:
+      flow_kwargs['login_hint'] = login_hint
+
+    # TODO: Move code for browser detection somewhere in this file so that the
+    # messaging about `nobrowser.txt` is co-located with the logic that uses it.
+    if use_console_flow:
+      flow.run_console(
+          authorization_prompt_message=MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
+          authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
+          **flow_kwargs)
+    else:
+      flow.run_local_server(
+          authorization_prompt_message=MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
+          success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
+          **flow_kwargs)
+    return cls.from_google_oauth2_credentials(
+        flow.credentials, filename=filename)
+
+  @classmethod
+  def from_client_secrets_file(cls,
+                               client_secrets_file,
+                               scopes,
+                               access_type='offline',
+                               login_hint=None,
+                               credentials_file=None,
+                               use_console_flow=False):
+    """Runs an OAuth Flow from secrets stored on disk to generate credentials.
+
+    Args:
+      client_secrets_file: String, path to a file containing a client ID and
+        secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      credentials_file: String, the path to a file to use to save the
+        credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Raises:
+      InvalidClientSecretsFileError: If the client secrets file cannot be
+        opened.
+      InvalidClientSecretsFileFormatError: If the client secrets file does not
+        contain the required data or the data is malformed.
+
+    Returns:
+      Credentials
+    """
+    cs_data = fileutils.read_file(
+        client_secrets_file, continue_on_error=True, display_errors=False)
+    if not cs_data:
+      raise InvalidClientSecretsFileError(
+          f'File {client_secrets_file} could not be opened')
+    try:
+      cs_json = json.loads(cs_data)
+      client_id = cs_json['installed']['client_id']
+      # Chop off .apps.googleusercontent.com suffix as it's not needed
+      # and we need to keep things short for the Auth URL.
+      client_id = re.sub(r'\.apps\.googleusercontent\.com$', '', client_id)
+      client_secret = cs_json['installed']['client_secret']
+    except (ValueError, IndexError, KeyError):
+      raise InvalidClientSecretsFileFormatError(
+          f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
+      )
+
+    return cls.from_client_secrets(
+        client_id,
+        client_secret,
+        scopes,
+        access_type=access_type,
+        login_hint=login_hint,
+        filename=credentials_file,
+        use_console_flow=use_console_flow)
+
+  def _fetch_id_token_data(self):
+    """Fetches verification details from Google for the OAuth2.0 token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Raises:
+      CredentialsError: If no id_token is present.
+    """
+    if not self.id_token:
+      raise CredentialsError('Failed to fetch token data. No id_token present.')
+    request = transport.create_request()
+    self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
+        self.id_token, request)
+
+  def get_token_value(self, field):
+    """Retrieves data from the OAuth ID token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Args:
+      field: The name of the key/field to fetch
+
+    Returns:
+      The value associated with the given key or 'Unknown' if the key data can
+      not be found in the access token data.
+    """
+    if not self._id_token_data:
+      self._fetch_id_token_data()
+    # Maintain legacy GAM behavior here to return "Unknown" if the field is
+    # otherwise unpopulated.
+    return self._id_token_data.get(field, 'Unknown')
+
+  def to_json(self, strip=None):
+    """Creates a JSON representation of a Credentials.
+
+    Args:
+        strip: Sequence[str], Optional list of members to exclude from the
+          generated JSON.
+
+    Returns:
+        str: A JSON representation of this instance, suitable to pass to
+             from_json().
+    """
+    expiry = self.expiry.strftime(
+        Credentials.DATETIME_FORMAT) if self.expiry else None
+    prep = {
+        'token': self.token,
+        'refresh_token': self.refresh_token,
+        'token_uri': self.token_uri,
+        'client_id': self.client_id,
+        'client_secret': self.client_secret,
+        'id_token': self.id_token,
+        # Google auth doesn't currently give us scopes back on refresh.
+        # 'scopes': sorted(self.scopes),
+        'token_expiry': expiry,
+        'decoded_id_token': self._id_token_data,
+    }
+
+    # Remove empty entries
+    prep = {k: v for k, v in prep.items() if v is not None}
+
+    # Remove entries that explicitly need to be removed
+    if strip is not None:
+      prep = {k: v for k, v in prep.items() if k not in strip}
+
+    return json.dumps(prep, indent=2, sort_keys=True)
+
+  def refresh(self, request=None):
+    """Refreshes the credential's access token.
+
+    Args:
+      request: google.auth.transport.Request, The object used to make HTTP
+        requests. If not provided, a default request will be used.
+
+    Raises:
+      google.auth.exceptions.RefreshError: If the credentials could not be
+      refreshed.
+    """
+    with self._lock:
+      if request is None:
+        request = transport.create_request()
+      self._locked_refresh(request)
+      # Save the new tokens back to disk, if these credentials are disk-backed.
+      if self._filename:
+        self._locked_write()
+
+  def _locked_refresh(self, request):
+    """Refreshes the credential's access token while the file lock is held."""
+    assert self._lock.is_locked
+    super(Credentials, self).refresh(request)
+
+  def write(self):
+    """Writes credentials to disk."""
+    with self._lock:
+      self._locked_write()
+
+  def _locked_write(self):
+    """Writes credentials to disk while the file lock is held."""
+    assert self._lock.is_locked
+    if not self.filename:
+      # If no filename was provided to the constructor, these credentials cannot
+      # be saved to disk.
+      raise CredentialsError(
+          'The credentials have no associated filename and cannot be saved '
+          'to disk.')
+    fileutils.write_file(self._filename, self.to_json())
+
+  def delete(self):
+    """Deletes all files on disk related to these credentials."""
+    with self._lock:
+      # Only attempt to remove the file if the lock we're using is a FileLock.
+      if isinstance(self._lock, FileLock):
+        os.remove(self._filename)
+        if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
+          os.remove(self._lock.lock_file)
+
+  _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+
+  def revoke(self, http=None):
+    """Revokes this credential's access token with the server.
+
+    Args:
+      http: httplib2.Http compatible object for use as a transport. If no http
+        is provided, a default will be used.
+    """
+    with self._lock:
+      if http is None:
+        http = transport.create_http()
+      params = urlencode({'token': self.refresh_token})
+      revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
+      http.request(revoke_uri, 'GET')
+
+
+class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
+  """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
+
+  URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+  def authorization_url(self, http=None, **kwargs):
+    """Gets a shortened authorization URL."""
+    long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
+    if not http:
+      http = transport.create_http(timeout=10)
+    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
+    try:
+      payload = json.dumps({'long_url': long_url})
+      resp, content = http.request(
+          _ShortURLFlow.URL_SHORTENER_ENDPOINT,
+          'POST',
+          payload,
+          headers=headers)
+    except:
+      return long_url, state
+
+    if resp.status != 200:
+      return long_url, state
+
+    try:
+      if isinstance(content, bytes):
+        content = content.decode()
+      return json.loads(content).get('short_url', long_url), state
+    except:
+      return long_url, state
+
+
+class _FileLikeThreadLock(object):
+  """A threading.lock which has the same interface as filelock.Filelock."""
+
+  def __init__(self):
+    """A shell object that holds a threading.Lock.
+
+    Since we cannot inherit from built-in classes such as threading.Lock, we
+    just use a shell object and maintain a lock inside of it.
+    """
+    self._lock = threading.Lock()
+
+  def __enter__(self, *args, **kwargs):
+    return self._lock.__enter__(*args, **kwargs)
+
+  def __exit__(self, *args, **kwargs):
+    return self._lock.__exit__(*args, **kwargs)
+
+  def acquire(self, **kwargs):
+    return self._lock.acquire(**kwargs)
+
+  def release(self):
+    return self._lock.release()
+
+  @property
+  def is_locked(self):
+    return self._lock.locked()
+
+  @property
+  def lock_file(self):
+    return None

src/auth/oauth_test.py

@@ -0,0 +1,680 @@
+"""Tests for oauth."""
+
+import datetime
+import json
+import os
+import platform
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import google.oauth2.credentials
+
+from auth import oauth
+
+
+class CredentialsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_token = 'fake_token'
+    self.fake_refresh_token = 'fake_refresh_token'
+    self.fake_id_token = 'fake_id_token'
+    self.fake_token_uri = 'https://fake.token.uri'
+    self.fake_client_id = 'fake_client_id'
+    self.fake_client_secret = 'fake_client_secret'
+    self.fake_scopes = [
+        'fake_api.readonly',
+        'fake_other_api.write',
+    ]
+    self.fake_quota_project_id = 'fake_quota_project_id'
+    self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
+    self.fake_filename = 'fake_filename'
+    self.fake_token_data = {
+        'field': 'value',
+        'another-field': 'another-value',
+    }
+    self.info_with_only_required_fields = {
+        'refresh_token': self.fake_refresh_token,
+        'client_id': self.fake_client_id,
+        'client_secret': self.fake_client_secret,
+    }
+    super(CredentialsTest, self).setUp()
+
+  def tearDown(self):
+    # Remove any credential files that may have been created.
+    if os.path.exists(self.fake_filename):
+      os.remove(self.fake_filename)
+    if os.path.exists('%s.lock' % self.fake_filename):
+      os.remove('%s.lock' % self.fake_filename)
+    super(CredentialsTest, self).tearDown()
+
+  def test_from_authorized_user_info_only_required_info(self):
+    creds = oauth.Credentials.from_authorized_user_info(
+        self.info_with_only_required_fields)
+    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+    self.assertEqual(self.fake_client_id, creds.client_id)
+    self.assertEqual(self.fake_client_secret, creds.client_secret)
+    self.assertIsNone(creds.id_token)
+    self.assertIsNone(creds.expiry)
+    self.assertIsNone(creds.filename)
+
+  def test_from_authorized_user_info_all_info_provided(self):
+    info = {
+        'token':
+            self.fake_token,
+        'refresh_token':
+            self.fake_refresh_token,
+        'id_token':
+            self.fake_id_token,
+        'token_uri':
+            self.fake_token_uri,
+        'client_id':
+            self.fake_client_id,
+        'client_secret':
+            self.fake_client_secret,
+        'token_expiry':
+            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+        'id_token_data':
+            self.fake_token_data,
+    }
+    creds = oauth.Credentials.from_authorized_user_info(info)
+    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+    self.assertEqual(self.fake_client_id, creds.client_id)
+    self.assertEqual(self.fake_client_secret, creds.client_secret)
+    self.assertEqual(self.fake_id_token, creds.id_token)
+    self.assertEqual(self.fake_token_uri, creds.token_uri)
+    self.assertEqual(self.fake_token_expiry, creds.expiry)
+    self.assertIsNone(creds.filename)
+
+  def test_from_authorized_user_info_missing_required_info(self):
+    info_with_missing_fields = {'token': self.fake_token}
+    with self.assertRaises(ValueError):
+      oauth.Credentials.from_authorized_user_info(info_with_missing_fields)
+
+  def test_from_authorized_user_info_no_expiry_in_info(self):
+    info_with_no_token_expiry = self.info_with_only_required_fields.copy()
+    self.assertIsNone(info_with_no_token_expiry.get('expiry'))
+    creds = oauth.Credentials.from_authorized_user_info(
+        info_with_no_token_expiry)
+    self.assertIsNone(creds.expiry)
+
+  def test_init_saves_filename(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+  def test_init_loads_decoded_id_token_data(self, mock_verify_token):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data)
+    self.assertEqual(
+        self.fake_token_data.get('field'), creds.get_token_value('field'))
+    # Verify the fetching method was not called, since the token
+    # data was supposed to be loaded from the passed in info.
+    self.assertEqual(mock_verify_token.call_count, 0)
+
+  def test_credentials_uses_file_lock_when_filename_provided(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    self.assertIsInstance(creds._lock, oauth.FileLock)
+    self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
+
+  def test_credentials_uses_thread_lock_when_filename_not_provided(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=None)
+    self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
+    self.assertIsNone(creds.filename)
+
+  def test_from_oauth2credentials(self):
+    google_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    creds = oauth.Credentials.from_google_oauth2_credentials(
+        google_creds, filename=self.fake_filename)
+    self.assertEqual(google_creds.token, creds.token)
+    self.assertEqual(google_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(google_creds.client_id, creds.client_id)
+    self.assertEqual(google_creds.client_secret, creds.client_secret)
+    self.assertEqual(google_creds.id_token, creds.id_token)
+    self.assertEqual(google_creds.expiry, creds.expiry)
+    self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+    self.assertIn('could not be opened', str(e.exception))
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_no_serialized_data_in_file_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({})
+    with self.assertRaises(oauth.EmptyCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_missing_any_token_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({
+        # This data is missing a token key/value pair
+        'client_id': self.fake_client_id,
+        'client_secret': self.fake_client_secret,
+    })
+    with self.assertRaises(oauth.InvalidCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_missing_required_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({
+        # This data is missing a client_secret key/value pair
+        'client_id': self.fake_client_id,
+        'refresh_token': self.fake_refresh_token,
+    })
+    with self.assertRaises(oauth.InvalidCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_console_flow(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        use_console_flow=True)
+    self.assertTrue(mock_flow.return_value.run_console.called)
+    self.assertFalse(mock_flow.return_value.run_local_server.called)
+    self.assertEqual(flow_creds.token, creds.token)
+    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(flow_creds.client_id, creds.client_id)
+    self.assertEqual(flow_creds.client_secret, creds.client_secret)
+    self.assertEqual(flow_creds.id_token, creds.id_token)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_local_server_flow(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        use_console_flow=False)
+    self.assertFalse(mock_flow.return_value.run_console.called)
+    self.assertTrue(mock_flow.return_value.run_local_server.called)
+    self.assertEqual(flow_creds.token, creds.token)
+    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(flow_creds.client_id, creds.client_id)
+    self.assertEqual(flow_creds.client_secret, creds.client_secret)
+    self.assertEqual(flow_creds.id_token, creds.id_token)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_uses_login_hint(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        login_hint='someone@domain.com')
+
+    run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
+    self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
+
+  def test_from_client_secrets_uses_shortened_url_flow(self):
+    with patch.object(oauth._ShortURLFlow, 'from_client_config') as mock_flow:
+      flow_creds = google.oauth2.credentials.Credentials(
+          token=self.fake_token,
+          refresh_token=self.fake_refresh_token,
+          client_id=self.fake_client_id,
+          client_secret=self.fake_client_secret,
+          id_token=self.fake_id_token)
+      mock_flow.return_value.credentials = flow_creds
+      oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                            self.fake_client_secret,
+                                            self.fake_scopes)
+    self.assertTrue(mock_flow.called)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        filename=self.fake_filename)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    with self.assertRaises(oauth.InvalidClientSecretsFileError):
+      oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                 self.fake_scopes)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_client_secrets_file_missing_required_json_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({})
+    with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
+      oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                 self.fake_scopes)
+    self.assertIn('Could not extract Client ID or Client Secret',
+                  str(e.exception))
+
+  @patch.object(oauth.Credentials, 'from_client_secrets')
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_client_secrets_file_strips_domain_from_client_id(
+      self, mock_read_file, mock_creds_from_client_secrets):
+    mock_read_file.return_value = json.dumps({
+        'installed': {
+            'client_id': self.fake_client_id + '.apps.googleusercontent.com',
+            'client_secret': self.fake_client_secret,
+        }
+    })
+
+    oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                               self.fake_scopes)
+    self.assertEqual(self.fake_client_id,
+                     mock_creds_from_client_secrets.call_args[0][0])
+
+  def test_get_token_value_known_token_field(self):
+    token_data = {'known-field': 'known-value'}
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token_data=token_data)
+    self.assertEqual('known-value', creds.get_token_value('known-field'))
+
+  def test_get_token_value_unknown_field_returns_unknown(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token_data=self.fake_token_data)
+    self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
+
+  def test_to_json_contains_all_required_fields(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data,
+        token_uri=self.fake_token_uri,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        scopes=self.fake_scopes,
+        quota_project_id=self.fake_quota_project_id,
+        expiry=self.fake_token_expiry)
+    json_string = creds.to_json()
+    json_data = json.loads(json_string)
+    keys = json_data.keys()
+    self.assertIn('token', keys)
+    self.assertEqual(self.fake_token, json_data['token'])
+    self.assertIn('refresh_token', keys)
+    self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
+    self.assertIn('id_token', keys)
+    self.assertEqual(self.fake_id_token, json_data['id_token'])
+    self.assertIn('token_uri', keys)
+    self.assertEqual(self.fake_token_uri, json_data['token_uri'])
+    self.assertIn('client_id', keys)
+    self.assertEqual(self.fake_client_id, json_data['client_id'])
+    self.assertIn('client_secret', keys)
+    self.assertEqual(self.fake_client_secret, json_data['client_secret'])
+    self.assertNotIn('scopes', keys)  # Scopes are not currently saved
+    self.assertIn('token_expiry', keys)
+    self.assertEqual(
+        self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+        json_data['token_expiry'])
+    self.assertIn('decoded_id_token', keys)
+    self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
+
+  def test_credentials_to_json_and_back(self):
+    original_creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data,
+        token_uri=self.fake_token_uri,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        scopes=self.fake_scopes,
+        quota_project_id=self.fake_quota_project_id,
+        expiry=self.fake_token_expiry)
+    pickled_creds = original_creds.to_json()
+    serialized_json = json.loads(pickled_creds)
+    unpickled_creds = oauth.Credentials.from_authorized_user_info(
+        serialized_json)
+    self.assertEqual(original_creds.token, unpickled_creds.token)
+    self.assertEqual(original_creds.refresh_token,
+                     unpickled_creds.refresh_token)
+    self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
+    self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
+    self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
+    self.assertEqual(original_creds.client_secret,
+                     unpickled_creds.client_secret)
+    self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  def test_refresh_calls_super_refresh(self, mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    request = MagicMock()
+
+    creds.refresh(request)
+    self.assertTrue(mock_super_refresh.called)
+    self.assertEqual(request, mock_super_refresh.call_args[0][0])
+
+  def test_refresh_locks_resource_during_refresh(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    lock = creds._lock
+
+    def check_lock_is_locked(*unused_args, **unused_kwargs):
+      self.assertTrue(lock.is_locked)
+
+    # We need to mock the superclass refresh so it doesn't actually try to
+    # refresh our fake token.
+    # At the same time, we'll make sure the lock is held during the refresh.
+    with patch.object(oauth.google.oauth2.credentials.Credentials,
+                      'refresh') as mock_refresh:
+      mock_refresh.side_effect = check_lock_is_locked
+      creds.refresh(request=MagicMock())
+
+    # Make sure our side effect was actually performed.
+    self.assertTrue(mock_refresh.called)
+    # The lock should be released after refresh
+    self.assertFalse(lock.is_locked)
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  @patch.object(oauth.fileutils, 'write_file')
+  def test_refresh_writes_new_credentials_to_disk_after_refresh(
+      self, mock_write_file, mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+
+    def update_access_token(unused_request):
+      creds.token = 'refreshed_access_token'
+
+    mock_super_refresh.side_effect = update_access_token
+
+    self.assertIsNone(creds.token)
+    creds.refresh(request=MagicMock())
+    self.assertEqual('refreshed_access_token', creds.token,
+                     'Access token was not refreshed')
+    text_written_to_file = mock_write_file.call_args[0][1]
+    self.assertIsNotNone(text_written_to_file, 'Nothing was written to file')
+    saved_json = json.loads(text_written_to_file)
+    self.assertEqual('refreshed_access_token', saved_json['token'],
+                     'Refreshed access token was not saved to disk')
+
+  def test_write_writes_credentials_to_disk(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+
+    self.assertFalse(os.path.exists(self.fake_filename))
+    creds.write()
+    self.assertTrue(os.path.exists(self.fake_filename))
+
+  def test_write_raises_error_when_no_credentials_file_is_set(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+
+    self.assertIsNone(creds.filename)
+    with self.assertRaises(oauth.CredentialsError):
+      creds.write()
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  @patch.object(oauth.fileutils, 'write_file')
+  def test_write_locks_resource_during_write(self, mock_write_file,
+                                             unused_mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    lock = creds._lock
+
+    def check_lock_is_locked(*unused_args, **unused_kwargs):
+      self.assertTrue(creds._lock.is_locked)
+
+    mock_write_file.side_effect = check_lock_is_locked
+
+    self.assertFalse(lock.is_locked)
+    creds.refresh(request=MagicMock())
+    self.assertFalse(lock.is_locked)
+    self.assertTrue(mock_write_file.called)
+
+  def test_delete_removes_credentials_file(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    creds.write()
+    self.assertTrue(os.path.exists(self.fake_filename))
+    creds.delete()
+    self.assertFalse(os.path.exists(self.fake_filename))
+
+  @unittest.skipIf(
+      platform.system() == 'Windows',
+      reason=('On Windows, Filelock deletes the lock file each time the lock '
+              'is released. Delete does not remove it.'))
+  def test_delete_removes_lock_file(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    lock_file = '%s.lock' % creds.filename
+    creds.write()
+    self.assertTrue(os.path.exists(lock_file))
+    creds.delete()
+    self.assertFalse(os.path.exists(lock_file))
+
+  def test_delete_is_noop_when_not_using_filelock(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    self.assertIsNone(creds.filename)
+    creds.delete()  # This should not raise an exception.
+
+  def test_revoke_requests_credential_revoke(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    mock_http = MagicMock()
+
+    creds.revoke(http=mock_http)
+
+    uri = mock_http.request.call_args[0][0]
+    self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
+    params = uri[uri.index('?'):]
+    self.assertIn('token=%s' % creds.refresh_token, params)
+    self.assertEqual('GET', mock_http.request.call_args[0][1])
+
+
+class ShortUrlFlowTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_client_id = 'fake_client_id'
+    self.fake_client_secret = 'fake_client_secret'
+    self.fake_scopes = [
+        'fake_api.readonly',
+        'fake_other_api.write',
+    ]
+    self.fake_client_config = {
+        'installed': {
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
+            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+            'token_uri': 'https://oauth2.googleapis.com/token',
+        }
+    }
+    self.long_url = 'http://example.com/some/long/url'
+    self.short_url = 'http://ex.co/short'
+    super(ShortUrlFlowTest, self).setUp()
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = json.dumps({'short_url': self.short_url})
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.short_url, url)
+    self.assertEqual('fake_state', state)
+
+    # Verify request() was called with the expected arguments.
+    self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
+                     mock_http.request.call_args[0][0])
+    self.assertEqual('POST', mock_http.request.call_args[0][1])
+    self.assertIn(self.long_url, mock_http.request.call_args[0][2])
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_request_error(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_http.request.side_effect = Exception()
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 404  # Use a status that is not 200
+    content = json.dumps({'short_url': self.short_url})
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = None
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = json.dumps({})  # This json content contains no "short-url" key
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/controlflow.py

@@ -0,0 +1,99 @@
+"""Methods related to the central control flow of an application."""
+import random
+import sys
+import time
+
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from var import MESSAGE_INVALID_JSON
+
+
+def system_error_exit(return_code, message):
+  """Raises a system exit with the given return code and message.
+
+  Args:
+    return_code: Int, the return code to yield when the system exits.
+    message: An error message to print before the system exits.
+  """
+  if message:
+    display.print_error(message)
+  sys.exit(return_code)
+
+
+def invalid_argument_exit(argument, command):
+  '''Indicate that the argument is not valid for the command.
+
+  Args:
+    argument: the invalid argument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'{argument} is not a valid argument for "{command}"')
+
+def missing_argument_exit(argument, command):
+  '''Indicate that the argument is missing for the command.
+
+  Args:
+    argument: the missingagrument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'missing argument {argument} for "{command}"')
+
+def expected_argument_exit(name, expected, argument):
+  '''Indicate that the argument does not have an expected value for the command.
+
+  Args:
+    name: the field name
+    expected: the expected values
+    argument: the invalid argument
+  '''
+  system_error_exit(
+      2,
+      f'{name} must be one of {expected}; got {argument}')
+
+def csv_field_error_exit(field_name, field_names):
+  """Raises a system exit when a CSV field is malformed.
+
+  Args:
+    field_name: The CSV field name for which a header does not exist in the
+      existing CSV headers.
+    field_names: The known list of CSV headers.
+  """
+  system_error_exit(
+      2,
+      MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                     ','.join(field_names)))
+
+
+def invalid_json_exit(file_name):
+  """Raises a sysyem exit when invalid JSON content is encountered."""
+  system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+
+
+def wait_on_failure(current_attempt_num,
+                    total_num_retries,
+                    error_message,
+                    error_print_threshold=3):
+  """Executes an exponential backoff-style system sleep.
+
+  Args:
+    current_attempt_num: Int, the current number of retries.
+    total_num_retries: Int, the total number of times the current action will be
+      retried.
+    error_message: String, a message to be displayed that will give more context
+      around why the action is being retried.
+    error_print_threshold: Int, the number of attempts which will have their
+      error messages suppressed. Any current_attempt_num greater than
+      error_print_threshold will print the prescribed error.
+  """
+  wait_on_fail = min(2**current_attempt_num,
+                     60) + float(random.randint(1, 1000)) / 1000
+  if current_attempt_num > error_print_threshold:
+    sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+        f'{int(wait_on_fail)} seconds, Retry: '
+        f'{current_attempt_num}/{total_num_retries}\n'))
+    sys.stderr.flush()
+  time.sleep(wait_on_fail)

src/controlflow_test.py

@@ -0,0 +1,106 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+  def test_system_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(1, 'exit message')
+
+  def test_system_error_exit_raises_systemexit_with_return_code(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertEqual(context_manager.exception.code, 100)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+  def test_csv_field_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+
+  def test_csv_field_error_exit_exits_code_2(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    self.assertEqual(context_manager.exception.code, 2)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('aField', printed_message)
+    self.assertIn('unusedField1', printed_message)
+    self.assertIn('unusedField2', printed_message)
+
+  def test_invalid_json_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+
+  def test_invalid_json_exit_exit_exits_code_17(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.invalid_json_exit('filename')
+    self.assertEqual(context_manager.exception.code, 17)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('filename', printed_message)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+    sleep_calls = mock_sleep.call_args_list
+    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+    total_attempts = 20
+    for attempt in range(1, total_attempts + 1):
+      controlflow.wait_on_failure(
+          attempt,
+          total_attempts,
+          'Attempt #%s' % attempt,
+          # Suppress messages while we make a lot of attempts.
+          error_print_threshold=total_attempts + 1)
+      # Wait time may be between 60 and 61 secs, due to rand addition.
+      self.assertLessEqual(mock_sleep.call_args[0][0], 61)
+
+  # Prevent the system from actually sleeping and thus slowing down the test.
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
+    message = 'An error message to display'
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+    self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_only_prints_after_threshold(self, unused_mock_sleep):
+    total_attempts = 5
+    threshold = 3
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      for attempt in range(1, total_attempts + 1):
+        controlflow.wait_on_failure(
+            attempt,
+            total_attempts,
+            'Attempt #%s' % attempt,
+            error_print_threshold=threshold)
+    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

src/display.py

@@ -0,0 +1,234 @@
+"""Methods related to display of information to the user."""
+
+import csv
+import io
+import sys
+import webbrowser
+
+import dateutil
+import googleapiclient.http
+
+#TODO: get rid of these hacks
+import __main__
+from var import *
+import controlflow
+import gapi
+
+
+def current_count(i, count):
+  return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
+
+def current_count_nl(i, count):
+  return f' ({i}/{count})\n' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
+
+def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
+  fields = fieldsChoiceMap[fieldName.lower()]
+  if isinstance(fields, list):
+    fieldsList.extend(fields)
+  else:
+    fieldsList.append(fields)
+
+# Write a CSV file
+def add_titles_to_csv_file(addTitles, titles):
+  for title in addTitles:
+    if title not in titles:
+      titles.append(title)
+
+def add_row_titles_to_csv_file(row, csvRows, titles):
+  csvRows.append(row)
+  for title in row:
+    if title not in titles:
+      titles.append(title)
+
+# fieldName is command line argument
+# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
+#ARGUMENT_TO_PROPERTY_MAP = {
+#  u'admincreated': [u'adminCreated'],
+#  u'aliases': [u'aliases', u'nonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles, titles):
+  for ftList in fieldNameMap[fieldName]:
+    if ftList not in fieldsTitles:
+      fieldsList.append(ftList)
+      fieldsTitles[ftList] = ftList
+      add_titles_to_csv_file([ftList], titles)
+
+# fieldName is command line argument
+# fieldNameTitleMap maps fieldName to API field name and CSV file header
+#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
+#  u'admincreated': [u'adminCreated', u'Admin_Created'],
+#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList, fieldsTitles, titles):
+  ftList = fieldNameTitleMap[fieldName]
+  for i in range(0, len(ftList), 2):
+    if ftList[i] not in fieldsTitles:
+      fieldsList.append(ftList[i])
+      fieldsTitles[ftList[i]] = ftList[i+1]
+      add_titles_to_csv_file([ftList[i+1]], titles)
+
+def sort_csv_titles(firstTitle, titles):
+  restoreTitles = []
+  for title in firstTitle:
+    if title in titles:
+      titles.remove(title)
+      restoreTitles.append(title)
+  titles.sort()
+  for title in restoreTitles[::-1]:
+    titles.insert(0, title)
+
+def QuotedArgumentList(items):
+  return ' '.join([item if item and (item.find(' ') == -1) and (item.find(',') == -1) else '"'+item+'"' for item in items])
+
+def write_csv_file(csvRows, titles, list_type, todrive):
+  def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
+    if not rowDate or not isinstance(rowDate, str):
+      return False
+    try:
+      rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
+      if dateMode:
+        rowDate = datetime.datetime(rowTime.year, rowTime.month, rowTime.day).isoformat()+'Z'
+    except ValueError:
+      rowDate = NEVER_TIME
+    if op == '<':
+      return rowDate < filterDate
+    if op == '<=':
+      return rowDate <= filterDate
+    if op == '>':
+      return rowDate > filterDate
+    if op == '>=':
+      return rowDate >= filterDate
+    if op == '!=':
+      return rowDate != filterDate
+    return rowDate == filterDate
+
+  def rowCountFilterMatch(rowCount, op, filterCount):
+    if isinstance(rowCount, str):
+      if not rowCount.isdigit():
+        return False
+      rowCount = int(rowCount)
+    elif not isinstance(rowCount, int):
+      return False
+    if op == '<':
+      return rowCount < filterCount
+    if op == '<=':
+      return rowCount <= filterCount
+    if op == '>':
+      return rowCount > filterCount
+    if op == '>=':
+      return rowCount >= filterCount
+    if op == '!=':
+      return rowCount != filterCount
+    return rowCount == filterCount
+  def rowBooleanFilterMatch(rowBoolean, filterBoolean):
+    if not isinstance(rowBoolean, bool):
+      return False
+    return rowBoolean == filterBoolean
+
+  def headerFilterMatch(title):
+    for filterStr in GC_Values[GC_CSV_HEADER_FILTER]:
+      if filterStr.match(title):
+        return True
+    return False
+
+  if GC_Values[GC_CSV_ROW_FILTER]:
+    for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
+      if column not in titles:
+        sys.stderr.write(f'WARNING: Row filter column "{column}" is not in output columns\n')
+        continue
+      if filterVal[0] == 'regex':
+        csvRows = [row for row in csvRows if filterVal[1].search(str(row.get(column, '')))]
+      elif filterVal[0] == 'notregex':
+        csvRows = [row for row in csvRows if not filterVal[1].search(str(row.get(column, '')))]
+      elif filterVal[0] in ['date', 'time']:
+        csvRows = [row for row in csvRows if rowDateTimeFilterMatch(filterVal[0] == 'date', row.get(column, ''), filterVal[1], filterVal[2])]
+      elif filterVal[0] == 'count':
+        csvRows = [row for row in csvRows if rowCountFilterMatch(row.get(column, 0), filterVal[1], filterVal[2])]
+      else: #boolean
+        csvRows = [row for row in csvRows if rowBooleanFilterMatch(row.get(column, False), filterVal[1])]
+  if GC_Values[GC_CSV_HEADER_FILTER]:
+    titles = [t for t in titles if headerFilterMatch(t)]
+    if not titles:
+      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER\n')
+      return
+  csv.register_dialect('nixstdout', lineterminator='\n')
+  if todrive:
+    write_to = io.StringIO()
+  else:
+    write_to = sys.stdout
+  writer = csv.DictWriter(write_to, fieldnames=titles, dialect='nixstdout', extrasaction='ignore', quoting=csv.QUOTE_MINIMAL)
+  try:
+    writer.writerow(dict((item, item) for item in writer.fieldnames))
+    writer.writerows(csvRows)
+  except IOError as e:
+    controlflow.system_error_exit(6, e)
+  if todrive:
+    admin_email = __main__._getValueFromOAuth('email')
+    _, drive = __main__.buildDrive3GAPIObject(admin_email)
+    if not drive:
+      print(f'''\nGAM is not authorized to create Drive files. Please run:
+gam user {admin_email} check serviceaccount
+and follow recommend steps to authorize GAM for Drive access.''')
+      sys.exit(5)
+    result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
+    columns = len(titles)
+    rows = len(csvRows)
+    cell_count = rows * columns
+    data_size = len(write_to.getvalue())
+    max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
+    if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
+      print(f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}')
+      mimeType = 'text/csv'
+    else:
+      mimeType = MIMETYPE_GA_SPREADSHEET
+    body = {'description': QuotedArgumentList(sys.argv),
+            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
+            'mimeType': mimeType}
+    result = gapi.call(drive.files(), 'create', fields='webViewLink',
+                       body=body,
+                       media_body=googleapiclient.http.MediaInMemoryUpload(write_to.getvalue().encode(),
+                                                                           mimetype='text/csv'))
+    file_url = result['webViewLink']
+    if GC_Values[GC_NO_BROWSER]:
+      msg_txt = f'Drive file uploaded to:\n {file_url}'
+      msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
+      __main__.send_email(msg_subj, msg_txt)
+      print(msg_txt)
+    else:
+      webbrowser.open(file_url)
+
+def print_error(message):
+  """Prints a one-line error message to stderr in a standard format."""
+  sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
+
+
+def print_warning(message):
+  """Prints a one-line warning message to stderr in a standard format."""
+  sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
+
+def print_json(object_value, spacing=''):
+  """Prints Dict or Array to screen in clean human-readable format.."""
+  if isinstance(object_value, list):
+    if len(object_value) == 1 and isinstance(object_value[0], (str, int, bool)):
+      sys.stdout.write(f'{object_value[0]}\n')
+      return
+    if spacing:
+      sys.stdout.write('\n')
+    for i, a_value in enumerate(object_value):
+      if isinstance(a_value, (str, int, bool)):
+        sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
+      else:
+        sys.stdout.write(f' {spacing}{i+1}) ')
+        print_json(a_value, f' {spacing}')
+  elif isinstance(object_value, dict):
+    for key in ['kind', 'etag', 'etags']:
+      object_value.pop(key, None)
+    for another_object, another_value in object_value.items():
+      sys.stdout.write(f' {spacing}{another_object}: ')
+      print_json(another_value, f' {spacing}')
+  else:
+    sys.stdout.write(f'{object_value}\n')

src/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+import display
+from var import ERROR_PREFIX
+from var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+  def test_print_error_prints_to_stderr(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  def test_print_error_prints_error_prefix(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(ERROR_PREFIX), printed_message.find(message),
+        'The error prefix does not appear before the error message')
+
+  def test_print_error_ends_message_with_newline(self):
+    message = 'test error'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The error message does not end in a newline.')
+
+  def test_print_warning_prints_to_stderr(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  def test_print_warning_prints_error_prefix(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(WARNING_PREFIX), printed_message.find(message),
+        'The warning prefix does not appear before the error message')
+
+  def test_print_warning_ends_message_with_newline(self):
+    message = 'test warning'
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The warning message does not end in a newline.')

src/email-settings-v2.json

@@ -1,151 +0,0 @@
-{
- "kind": "discovery#restDescription",
- "discoveryVersion": "v1",
- "id": "email-settings:v2",
- "name": "email-settings",
- "version": "v2",
- "revision": "20161013",
- "title": "Email Settings API",
- "description": "Lets you manage Google Apps Email Settings",
- "ownerDomain": "google.com",
- "ownerName": "Google",
- "icons": {
-  "x16": "http://www.google.com/images/icons/product/search-16.gif",
-  "x32": "http://www.google.com/images/icons/product/search-32.gif"
- },
- "documentationLink": "https://developers.google.com/admin-sdk/email-settings",
- "protocol": "rest",
- "baseUrl": "https://apps-apis.google.com/",
- "rootUrl": "https://apps-apis.google.com/",
- "servicePath": "/a/feeds/emailsettings/2.0/",
- "parameters": {
-  "v": {
-   "type": "string",
-   "description": "GData Version",
-   "default": "2.0",
-   "enum": [
-    "2.0"
-   ],
-   "enumDescriptions": [
-    "GData 2.0"
-   ],
-   "location": "query"
-  },
-  "alt": {
-   "type": "string",
-   "description": "Data format for the response.",
-   "default": "json",
-   "enum": [
-    "json"
-   ],
-   "enumDescriptions": [
-    "Responses with Content-Type of application/json"
-   ],
-   "location": "query"
-  },
-  "quotaUser": {
-   "type": "string",
-   "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.",
-   "location": "query"
-  },
-  "prettyPrint": {
-   "type": "boolean",
-   "description": "Returns response with indentations and line breaks.",
-   "default": "true",
-   "location": "query"
-  }
- },
- "auth": {
-  "oauth2": {
-   "scopes": {
-    "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": {
-     "description": "Manage email settings"
-    }
-   }
-  }
- },
- "schemas": {
-  "Delegate": {
-   "id": "Delegate",
-   "type": "object",
-   "description": "a delegate.",
-   "properties": {
-    "apps$property": {
-     "type": "object",
-     "properties": {
-      "name": {
-       "type": "string",
-       "description": "property name"
-      },
-      "value": {
-        "type": "string",
-        "description": "organization name value"
-      }
-     }
-    }
-   }
-  },
-  "Delegates": {
-   "id": "feed",
-   "type": "object",
-   "description": "List of delegates.",
-   "properties": {
-    "entry": {
-     "type": "object",
-     "description": "list of delegates",
-     "items": {
-      "$ref": "Delegate"
-     }
-    }
-   }
-  }
- },
- "resources": {
-  "delegates": {
-   "methods": {
-    "get": {
-     "id": "email-settings.delegates.get",
-     "path": "{domainName}/{delegator}/delegation",
-     "httpMethod": "GET",
-     "parameters": {
-      "domainName": {
-       "type": "string",
-       "required": "true",
-       "location": "path"
-      },
-      "delegator": {
-       "type": "string",
-       "required": "true",
-       "location": "path"
-      }
-     },
-     "response": {
-      "$ref": "Delegates"
-     }
-    },
-    "delete": {
-     "id": "email-settings.delegates.delete",
-     "path": "{domainName}/{delegator}/delegation/{delegate}",
-     "httpMethod": "DELETE",
-     "parameters": {
-      "domainName": {
-       "type": "string",
-       "required": "true",
-       "location": "path"
-      },
-      "delegator": {
-       "type": "string",
-       "required": "true",
-       "location": "path"
-      },
-      "delegate": {
-       "type": "string",
-       "required": "true",
-       "location": "path"
-      }
-     }
-    }
-   }
-  }   
- }
-}

src/fileutils.py

@@ -0,0 +1,179 @@
+"""Common file operations."""
+
+import io
+import os
+import sys
+
+import controlflow
+import display
+from var import GM_Globals
+from var import GM_SYS_ENCODING
+from var import UTF8_SIG
+
+
+def _open_file(filename, mode, encoding=None, newline=None):
+  """Opens a file with no error handling."""
+  # Determine which encoding to use
+  if 'b' in mode:
+    encoding = None
+  elif not encoding:
+    encoding = GM_Globals[GM_SYS_ENCODING]
+  elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
+    encoding = UTF8_SIG
+
+  return open(
+      os.path.expanduser(filename), mode, newline=newline, encoding=encoding)
+
+
+def open_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              strip_utf_bom=False):
+  """Opens a file.
+
+  Args:
+    filename: String, the name of the file to open, or '-' to use stdin/stdout,
+      to read/write, depending on the mode param, respectively.
+    mode: String, the common file mode to open the file with. Default is read.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    strip_utf_bom: Boolean, True if the file being opened should seek past the
+      UTF Byte Order Mark before being returned.
+        See more: https://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+
+  Returns:
+    The opened file.
+  """
+  try:
+    if filename == '-':
+      # Read from stdin, rather than a file
+      if 'r' in mode:
+        return io.StringIO(str(sys.stdin.read()))
+      return sys.stdout
+
+    # Open a file on disk
+    f = _open_file(filename, mode, newline=newline, encoding=encoding)
+    if strip_utf_bom:
+      utf_bom = u'\ufeff'
+      has_bom = False
+
+      if 'b' in mode:
+        has_bom = f.read(3).decode('UTF-8') == utf_bom
+      elif f.encoding and not f.encoding.lower().startswith('utf'):
+        # Convert UTF BOM into ISO-8859-1 via Bytes
+        utf8_bom_bytes = utf_bom.encode('UTF-8')
+        iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
+            'iso-8859-1')
+        has_bom = f.read(3).encode('iso-8859-1', 'replace') == iso_8859_1_bom
+      else:
+        has_bom = f.read(1) == utf_bom
+
+      if not has_bom:
+        f.seek(0)
+
+    return f
+
+  except IOError as e:
+    controlflow.system_error_exit(6, e)
+
+
+def close_file(f, force_flush=False):
+  """Closes a file.
+
+  Args:
+    f: The file to close
+    force_flush: Flush file to disk emptying Python and OS caches. See:
+       https://stackoverflow.com/a/13762137/1503886
+
+  Returns:
+    Boolean, True if the file was successfully closed. False if an error
+        was encountered while closing.
+  """
+  if force_flush:
+    f.flush()
+    os.fsync(f.fileno())
+  try:
+    f.close()
+    return True
+  except IOError as e:
+    display.print_error(e)
+    return False
+
+
+def read_file(filename,
+              mode='r',
+              encoding=None,
+              newline=None,
+              continue_on_error=False,
+              display_errors=True):
+  """Reads a file from disk.
+
+  Args:
+    filename: String, the path of the file to open from disk, or "-" to read
+      from stdin.
+    mode: String, the mode in which to open the file.
+    encoding: String, the name of the encoding used to decode or encode the
+      file. This should only be used in text mode.
+    newline: See param description in
+        https://docs.python.org/3.7/library/functions.html#open
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    The contents of the file, or stdin if filename == "-". Returns None if
+    an error is encountered and continue_on_errors is True.
+  """
+  try:
+    if filename == '-':
+      # Read from stdin, rather than a file.
+      return str(sys.stdin.read())
+
+    with _open_file(filename, mode, newline=newline, encoding=encoding) as f:
+      return f.read()
+
+  except IOError as e:
+    if continue_on_error:
+      if display_errors:
+        display.print_warning(e)
+      return None
+    controlflow.system_error_exit(6, e)
+  except (LookupError, UnicodeDecodeError, UnicodeError) as e:
+    controlflow.system_error_exit(2, str(e))
+
+
+def write_file(filename,
+               data,
+               mode='w',
+               continue_on_error=False,
+               display_errors=True):
+  """Writes data to a file.
+
+  Args:
+    filename: String, the path of the file to write to disk.
+    data: Serializable data to write to the file.
+    mode: String, the mode in which to open the file and write to it.
+    continue_on_error: Boolean, If True, suppresses any IO errors and returns to
+      the caller without any externalities.
+    display_errors: Boolean, If True, prints error messages when errors are
+      encountered and continue_on_error is True.
+
+  Returns:
+    Boolean, True if the write operation succeeded, or False if not.
+  """
+  try:
+    with _open_file(filename, mode) as f:
+      f.write(data)
+    return True
+
+  except IOError as e:
+    if continue_on_error:
+      if display_errors:
+        display.print_error(e)
+      return False
+    else:
+      controlflow.system_error_exit(6, e)

src/fileutils_test.py

@@ -0,0 +1,234 @@
+"""Tests for fileutils."""
+
+import io
+import os
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import fileutils
+
+
+class FileutilsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_path = '/some/path/to/file'
+    super(FileutilsTest, self).setUp()
+
+  @patch.object(fileutils.sys, 'stdin')
+  def test_open_file_stdin(self, mock_stdin):
+    mock_stdin.read.return_value = 'some stdin content'
+    f = fileutils.open_file('-', mode='r')
+    self.assertIsInstance(f, fileutils.io.StringIO)
+    self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
+
+  def test_open_file_stdout(self):
+    f = fileutils.open_file('-', mode='w')
+    self.assertEqual(fileutils.sys.stdout, f)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_opens_correct_path(self, mock_open):
+    f = fileutils.open_file(self.fake_path)
+    self.assertEqual(self.fake_path, mock_open.call_args[0][0])
+    self.assertEqual(mock_open.return_value, f)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_expands_user_file_path(self, mock_open):
+    file_path = '~/some/path/containing/tilde/shortcut/to/home'
+    fileutils.open_file(file_path)
+    opened_path = mock_open.call_args[0][0]
+    home_path = os.environ.get('HOME')
+    self.assertIsNotNone(home_path)
+    self.assertIn(home_path, opened_path)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_opens_correct_mode(self, mock_open):
+    fileutils.open_file(self.fake_path)
+    self.assertEqual('r', mock_open.call_args[0][1])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_encoding_for_binary(self, mock_open):
+    fileutils.open_file(self.fake_path, mode='b')
+    self.assertIsNone(mock_open.call_args[1]['encoding'])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_default_system_encoding(self, mock_open):
+    fileutils.open_file(self.fake_path)
+    self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
+                     mock_open.call_args[1]['encoding'])
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_utf8_encoding_specified(self, mock_open):
+    fileutils.open_file(self.fake_path, encoding='UTF-8')
+    self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
+
+  def test_open_file_strips_utf_bom_in_utf(self):
+    bom_prefixed_data = u'\ufefffoobar'
+    fake_file = io.StringIO(bom_prefixed_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      self.assertEqual('foobar', f.read())
+
+  def test_open_file_strips_utf_bom_in_non_utf(self):
+    bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
+
+    # We need to trick the method under test into believing that a StringIO
+    # instance is a file with an encoding. Since StringIO does not usually have,
+    # an encoding, we'll mock it and add our own encoding, but send the other
+    # methods in use (read and seek) back to the real StringIO object.
+    real_stringio = io.StringIO(bom_prefixed_data)
+    mock_file = MagicMock(spec=io.StringIO)
+    mock_file.read.side_effect = real_stringio.read
+    mock_file.seek.side_effect = real_stringio.seek
+    mock_file.encoding = 'iso-8859-1'
+
+    mock_open = MagicMock(spec=open, return_value=mock_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      self.assertEqual('foobar', f.read())
+
+  def test_open_file_strips_utf_bom_in_binary(self):
+    bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
+    fake_file = io.BytesIO(bom_prefixed_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, mode='rb', strip_utf_bom=True)
+      self.assertEqual(b'foobar', f.read())
+
+  def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
+    no_bom_data = 'This data has no BOM'
+    fake_file = io.StringIO(no_bom_data)
+    mock_open = MagicMock(spec=open, return_value=fake_file)
+
+    with patch.object(fileutils, 'open', mock_open):
+      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+      # Since there was no opening BOM, we should be back at the beginning of
+      # the file.
+      self.assertEqual(fake_file.tell(), 0)
+      self.assertEqual(f.read(), no_bom_data)
+
+  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+  def test_open_file_exits_on_io_error(self, mock_open):
+    mock_open.side_effect = IOError('Fake IOError')
+    with self.assertRaises(SystemExit) as context:
+      fileutils.open_file(self.fake_path)
+    self.assertEqual(context.exception.code, 6)
+
+  def test_close_file_closes_file_successfully(self):
+    mock_file = MagicMock()
+    self.assertTrue(fileutils.close_file(mock_file))
+    self.assertEqual(mock_file.close.call_count, 1)
+
+  def test_close_file_with_error(self):
+    mock_file = MagicMock()
+    mock_file.close.side_effect = IOError()
+    self.assertFalse(fileutils.close_file(mock_file))
+    self.assertEqual(mock_file.close.call_count, 1)
+
+  @patch.object(fileutils.sys, 'stdin')
+  def test_read_file_from_stdin(self, mock_stdin):
+    mock_stdin.read.return_value = 'some stdin content'
+    self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_default_params(self, mock_open_file):
+    fake_content = 'some fake content'
+    mock_open_file.return_value.__enter__().read.return_value = fake_content
+    self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
+    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+    self.assertEqual(mock_open_file.call_args[0][1], 'r')
+    self.assertIsNone(mock_open_file.call_args[1]['newline'])
+
+  @patch.object(fileutils.display, 'print_warning')
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_continues_on_errors_without_displaying(
+      self, mock_open_file, mock_print_warning):
+    mock_open_file.side_effect = IOError()
+    contents = fileutils.read_file(
+        self.fake_path, continue_on_error=True, display_errors=False)
+    self.assertIsNone(contents)
+    self.assertFalse(mock_print_warning.called)
+
+  @patch.object(fileutils.display, 'print_warning')
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_displays_errors(self, mock_open_file, mock_print_warning):
+    mock_open_file.side_effect = IOError()
+    fileutils.read_file(
+        self.fake_path, continue_on_error=True, display_errors=True)
+    self.assertTrue(mock_print_warning.called)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_6_when_continue_on_error_is_false(
+      self, mock_open_file):
+    mock_open_file.side_effect = IOError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path, continue_on_error=False)
+    self.assertEqual(context.exception.code, 6)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
+    mock_open_file.return_value.__enter__().read.side_effect = LookupError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
+    mock_open_file.return_value.__enter__().read.side_effect = UnicodeError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
+    fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0, 1,
+                                           'testing only')
+    mock_open_file.return_value.__enter__().read.side_effect = fake_decode_error
+    with self.assertRaises(SystemExit) as context:
+      fileutils.read_file(self.fake_path)
+    self.assertEqual(context.exception.code, 2)
+
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_writes_data_to_file(self, mock_open_file):
+    fake_data = 'some fake data'
+    fileutils.write_file(self.fake_path, fake_data)
+    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+    self.assertEqual(mock_open_file.call_args[0][1], 'w')
+
+    opened_file = mock_open_file.return_value.__enter__()
+    self.assertTrue(opened_file.write.called)
+    self.assertEqual(opened_file.write.call_args[0][0], fake_data)
+
+  @patch.object(fileutils.display, 'print_error')
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_continues_on_errors_without_displaying(
+      self, mock_open_file, mock_print_error):
+    mock_open_file.side_effect = IOError()
+    status = fileutils.write_file(
+        self.fake_path,
+        'foo data',
+        continue_on_error=True,
+        display_errors=False)
+    self.assertFalse(status)
+    self.assertFalse(mock_print_error.called)
+
+  @patch.object(fileutils.display, 'print_error')
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
+    mock_open_file.side_effect = IOError()
+    fileutils.write_file(
+        self.fake_path, 'foo data', continue_on_error=True, display_errors=True)
+    self.assertTrue(mock_print_error.called)
+
+  @patch.object(fileutils, '_open_file')
+  def test_write_file_exits_code_6_when_continue_on_error_is_false(
+      self, mock_open_file):
+    mock_open_file.side_effect = IOError()
+    with self.assertRaises(SystemExit) as context:
+      fileutils.write_file(self.fake_path, 'foo data', continue_on_error=False)
+    self.assertEqual(context.exception.code, 6)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gam-install.sh

@@ -8,8 +8,9 @@ GAM installation script.
 OPTIONS:
    -h      show help.
    -d      Directory where gam folder will be installed. Default is \$HOME/bin/
-   -a      Architecture to install (i386, x86_64, arm). Default is to detect your arch with "uname -m".
+   -a      Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m".
    -o      OS we are running (linux, macos). Default is to detect your OS with "uname -s".
+   -b      OS version. Default is to detect on MacOS and Linux.
    -l      Just upgrade GAM to latest version. Skips project creation and auth.
    -p      Profile update (true, false). Should script add gam command to environment. Default is true.
    -u      Admin user email address to use with GAM. Default is to prompt.
@@ -21,18 +22,23 @@ EOF
 target_dir="$HOME/bin"
 gamarch=$(uname -m)
 gamos=$(uname -s)
+osversion=""
 update_profile=true
 upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-while getopts "hd:a:o:lp:u:r:v:" OPTION
+gam_glibc_vers="2.27 2.23 2.19 2.15"
+gam_macos_vers="10.14.4 10.13.6 10.12.6"
+
+while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
      case $OPTION in
          h) usage; exit;;
          d) target_dir="$OPTARG";;
          a) gamarch="$OPTARG";;
          o) gamos="$OPTARG";;
+         b) osversion="$OPTARG";;
          l) upgrade_only=true;;
          p) update_profile="$OPTARG";;
          u) adminuser="$OPTARG";;
@@ -42,8 +48,11 @@ do
      esac
 done
 
+# remove possible / from end of target_dir
+target_dir=${target_dir%/}
+
 update_profile() {
-	[ -f "$1" ] || return 1
+	[ $2 -eq 1 ] || [ -f "$1" ] || return 1
 
 	grep -F "$alias_line" "$1" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
@@ -72,28 +81,66 @@ echo -e "\x1B[1;33m$1"
 echo -e '\x1B[0m'
 }
 
+version_gt()
+{
+# MacOS < 10.13 doesn't support sort -V
+echo "" | sort -V > /dev/null 2>&1
+vsort_failed=$?
+if [ "${1}" = "${2}" ]; then
+  true
+elif (( $vsort_failed != 0 )); then
+  false
+else
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+fi
+}
+
 case $gamos in
   [lL]inux)
     gamos="linux"
+    if [ "$osversion" == "" ]; then
+      this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    else
+      this_glibc_ver=$osversion
+    fi
+    echo "This Linux distribution uses glibc $this_glibc_ver"
+    useglibc="legacy"
+    for gam_glibc_ver in $gam_glibc_vers; do
+      if version_gt $this_glibc_ver $gam_glibc_ver; then
+        useglibc="glibc$gam_glibc_ver"
+        echo_green "Using GAM compiled against $useglibc"
+        break
+      fi
+    done
     case $gamarch in
-      x86_64) gamfile="linux-x86_64.tar.xz";;
-      i?86) gamfile="linux-i686.tar.xz";;
-      arm*) gamfile="linux-armv7l.tar.xz";;
+      x86_64) gamfile="linux-x86_64-$useglibc.tar.xz";;
+      arm64|aarch64) gamfile="linux-arm64-$useglibc.tar.xz";;
       *)
-        echo_red "ERROR: this installer currently only supports i386, x86_64 and arm Linux. Looks like you're running on $gamarch. Exiting."
+        echo_red "ERROR: this installer currently only supports x86_64 and arm64 Linux. Looks like you're running on $gamarch. Exiting."
         exit
     esac
     ;;
   [Mm]ac[Oo][sS]|[Dd]arwin)
-    osver=$(sw_vers -productVersion | awk -F'.' '{print $2}')
-    if (( $osver < 10 )); then
-      echo_red "ERROR: GAM currently requires MacOS 10.10 or newer. You are running MacOS 10.$osver. Please upgrade." 
-      exit
-    else
-      echo_green "Good, you're running MacOS 10.$osver..."
-    fi
     gamos="macos"
-    gamfile="macos.tar.xz"
+    if [ "$osversion" == "" ]; then
+      this_macos_ver=$(sw_vers -productVersion)
+    else
+      this_macos_ver=$osversion
+    fi
+    echo "You are running MacOS $this_macos_ver"
+    use_macos_ver=""
+    for gam_macos_ver in $gam_macos_vers; do
+      if version_gt $this_macos_ver $gam_macos_ver; then
+        use_macos_ver="MacOS$gam_macos_ver"
+        echo_green "Using GAM compiled on $use_macos_ver"
+        break
+      fi
+    done
+    if [ "$use_macos_ver" == "" ]; then
+      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
+      exit
+    fi
+    gamfile="macos-x86_64-$use_macos_ver.tar.xz"
     ;;
   *)
     echo_red "Sorry, this installer currently only supports Linux and MacOS. Looks like you're runnning on $gamos. Exiting."
@@ -130,16 +177,26 @@ if type(release) is list:
       continue
     release = a_release
     break
-for asset in release['assets']:
-  if asset[sys.argv[1]].endswith('$gamfile'):
-    print(asset[sys.argv[1]])
-    break"
+try:
+  for asset in release['assets']:
+    if asset[attrib].endswith('$gamfile'):
+      print(asset[attrib])
+      break
+  else:
+    print('ERROR: Attribute: {0} for $gamfile version {1} not found'.format(attrib, gamversion))
+except KeyError:
+  print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
-pycmd="python"
+pycmd="python3"
 $pycmd -V >/dev/null 2>&1
 rc=$?
 if (( $rc != 0 )); then
-  pycmd="python3"
+  pycmd="python"
+fi
+$pycmd -V >/dev/null 2>&1
+rc=$?
+if (( $rc != 0 )); then
+  pycmd="python2"
 fi
 $pycmd -V >/dev/null 2>&1
 rc=$?
@@ -149,7 +206,15 @@ if (( $rc != 0 )); then
 fi
 
 browser_download_url=$(echo "$release_json" | $pycmd -c "$pycode" browser_download_url $gamversion)
+if [[ ${browser_download_url:0:5} = "ERROR" ]]; then
+  echo_red "${browser_download_url}"
+  exit
+fi
 name=$(echo "$release_json" | $pycmd -c "$pycode" name $gamversion)
+if [[ ${name:0:5} = "ERROR" ]]; then
+  echo_red "${name}"
+  exit
+fi
 # Temp dir for archive
 #temp_archive_dir=$(mktemp -d)
 temp_archive_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
@@ -169,9 +234,21 @@ else
   echo_green "Finished extracting GAM archive."
 fi
 
+# Update profile to add gam command
+if [ "$update_profile" = true ]; then
+  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
+  if [ "$gamos" == "linux" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0
+  elif [ "$gamos" == "macos" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0 || update_profile "$HOME/.profile" 1
+  fi
+else
+  echo_yellow "skipping profile update."
+fi
+
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
-  "$target_dir/gam/gam" version
+  "$target_dir/gam/gam" version extended
   rc=$?
   if (( $rc != 0 )); then
     echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -182,18 +259,6 @@ if [ "$upgrade_only" = true ]; then
   exit
 fi
 
-# Update profile to add gam command
-if [ "$update_profile" = true ]; then
-  alias_line="alias gam=\"$target_dir/gam/gam\""
-  if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile"
-  elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile"
-  fi
-else
-  echo_yellow "skipping profile update."
-fi
-
 while true; do
   read -p "Can you run a full browser on this machine? (usually Y for MacOS, N for Linux if you SSH into this machine) " yn
   case $yn in
@@ -294,7 +359,7 @@ while $project_created; do
 done
 
 echo_green "Here's information about your new GAM installation:"
-"$target_dir/gam/gam" version
+"$target_dir/gam/gam" version extended
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."

src/gam-setup.bat

@@ -1,5 +1,15 @@
+:neworupgrade
 @echo(
-@set /p adminemail= "Please enter your G Suite admin email address: "
+@set /p nu= "Is this a new install or an upgrade? [n or u] "
+@if /I "%nu%"=="u" (
+@  echo GAM installation and setup complete!
+@  goto alldone
+   )
+@if /I not "%nu%"=="n" (
+@  echo(
+@  echo Please answer n or u.
+@  goto neworupgrade
+   )
 
 :createproject
 @echo(
@@ -16,10 +26,12 @@
 @  echo Please answer y or n.
 @  goto createproject
    )
+@echo(
+@set /p adminemail= "Please enter your G Suite admin email address: "
 @gam create project %adminemail%
 @if not ERRORLEVEL 1 goto projectdone
 @echo(
-@echo Projection creation failed. Trying again. Say n to skip projection creation.
+@echo Project creation failed. Trying again. Say n to skip project creation.
 @goto createproject
 :projectdone
 

src/gapi/__init__.py

@@ -0,0 +1,328 @@
+"""Methods related to execution of GAPI requests."""
+
+import sys
+
+import googleapiclient.errors
+import google.auth.exceptions
+import httplib2
+
+import controlflow
+import display
+from gapi import errors
+import transport
+from var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+                 GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
+                 MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
+                 MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
+
+
+def call(service,
+         function,
+         silent_errors=False,
+         soft_errors=False,
+         throw_reasons=None,
+         retry_reasons=None,
+         **kwargs):
+  """Executes a single request on a Google service function.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    silent_errors: Bool, If True, error messages are suppressed when
+      encountered.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A response object for the corresponding Google API call.
+  """
+  if throw_reasons is None:
+    throw_reasons = []
+  if retry_reasons is None:
+    retry_reasons = []
+
+  method = getattr(service, function)
+  retries = 10
+  parameters = dict(
+      list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+  for n in range(1, retries + 1):
+    try:
+      return method(**parameters).execute()
+    except googleapiclient.errors.HttpError as e:
+      http_status, reason, message = errors.get_gapi_error_detail(
+          e,
+          soft_errors=soft_errors,
+          silent_errors=silent_errors,
+          retry_on_http_error=n < 3)
+      if http_status == -1:
+        # The error detail indicated that we should retry this request
+        # We'll refresh credentials and make another pass
+        service._http.request.credentials.refresh(transport.create_http())
+        continue
+      if http_status == 0:
+        return None
+
+      is_known_error_reason = reason in [r.value for r in errors.ErrorReason]
+      if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
+        if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
+          raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
+              message)
+        raise e
+      if (n != retries) and (is_known_error_reason and errors.ErrorReason(
+          reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
+        controlflow.wait_on_failure(n, retries, reason)
+        continue
+      if soft_errors:
+        display.print_error(f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}')
+        return None
+      controlflow.system_error_exit(
+          int(http_status), f'{http_status}: {message} - {reason}')
+    except google.auth.exceptions.RefreshError as e:
+      handle_oauth_token_error(
+          e, soft_errors or
+          errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+      if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
+        raise errors.GapiServiceNotAvailableError(str(e))
+      display.print_error(f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
+      return None
+    except ValueError as e:
+      if hasattr(service._http, 'cache') and service._http.cache is not None:
+        service._http.cache = None
+        continue
+      controlflow.system_error_exit(4, str(e))
+    except (httplib2.ServerNotFoundError, RuntimeError) as e:
+      if n != retries:
+        service._http.connections = {}
+        controlflow.wait_on_failure(n, retries, str(e))
+        continue
+      controlflow.system_error_exit(4, str(e))
+    except TypeError as e:
+      controlflow.system_error_exit(4, str(e))
+
+
+def get_items(service,
+              function,
+              items='items',
+              throw_reasons=None,
+              retry_reasons=None,
+              **kwargs):
+  """Gets a single page of items from a Google service function that is paged.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the service
+      method's response object.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    The list of items in the first page of a response.
+  """
+  results = call(
+      service,
+      function,
+      throw_reasons=throw_reasons,
+      retry_reasons=retry_reasons,
+      **kwargs)
+  if results:
+    return results.get(items, [])
+  return []
+
+
+def _get_max_page_size_for_api_call(service, function, **kwargs):
+  """Gets the maximum number of results supported for a single API call.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of the service method to check for max page size.
+    **kwargs: Additional params that will be passed to the request method.
+
+  Returns:
+    Int, A value from discovery if it exists, otherwise value from
+        MAX_RESULTS_API_EXCEPTIONS, otherwise None
+  """
+  method = getattr(service, function)
+  api_id = method(**kwargs).methodId
+  for resource in service._rootDesc.get('resources', {}).values():
+    for a_method in resource.get('methods', {}).values():
+      if a_method.get('id') == api_id:
+        if not a_method.get('parameters') or a_method['parameters'].get(
+            'pageSize') or not a_method['parameters'].get('maxResults'):
+          # Make sure API call supports maxResults. For now we don't care to
+          # set pageSize since all known pageSize API calls have
+          # default pageSize == max pageSize.
+          return None
+        known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
+        max_results = a_method['parameters']['maxResults'].get(
+            'maximum', known_api_max)
+        return {'maxResults': max_results}
+
+  return None
+
+
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+def got_total_items_msg(items, eol):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+def got_total_items_first_last_msg(items):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}'+'\n'
+
+def get_all_pages(service,
+                  function,
+                  items='items',
+                  page_message=None,
+                  message_attribute=None,
+                  soft_errors=False,
+                  throw_reasons=None,
+                  retry_reasons=None,
+                  **kwargs):
+  """Aggregates and returns all pages of a Google service function response.
+
+  All pages of items are aggregated and returned as a single list.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the method's
+      response object. The items in this field will be aggregated across all
+      pages and returned.
+    page_message: String, a message to be displayed to the user during paging.
+      Template strings allow for dynamic content to be inserted during paging.
+        Supported template strings:
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
+            pages.
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
+            display a unique property of the first item in the current page.
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
+            display a unique property of the last item in the current page.
+    message_attribute: String, the name of a signature field within a single
+      returned item which identifies that unique item. This field is used with
+      `page_message` to templatize a paging status message.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A list of all items received from all paged responses.
+  """
+  if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
+    page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
+    if page_key:
+      kwargs.update(page_key)
+  all_items = []
+  page_token = None
+  total_items = 0
+  while True:
+    page = call(
+        service,
+        function,
+        soft_errors=soft_errors,
+        throw_reasons=throw_reasons,
+        retry_reasons=retry_reasons,
+        pageToken=page_token,
+        **kwargs)
+    if page:
+      page_token = page.get('nextPageToken')
+      page_items = page.get(items, [])
+      num_page_items = len(page_items)
+      total_items += num_page_items
+      all_items.extend(page_items)
+    else:
+      page_token = None
+      num_page_items = 0
+
+    # Show a paging message to the user that indicates paging progress
+    if page_message:
+      show_message = page_message.replace(TOTAL_ITEMS_MARKER, str(total_items))
+      if message_attribute:
+        first_item = page_items[0] if num_page_items > 0 else {}
+        last_item = page_items[-1] if num_page_items > 1 else first_item
+        show_message = show_message.replace(FIRST_ITEM_MARKER, str(first_item.get(message_attribute, '')))
+        show_message = show_message.replace(LAST_ITEM_MARKER, str(last_item.get(message_attribute, '')))
+      sys.stderr.write('\r')
+      sys.stderr.flush()
+      sys.stderr.write(show_message)
+
+    if not page_token:
+      # End the paging status message and return all items.
+      if page_message and (page_message[-1] != '\n'):
+        sys.stderr.write('\r\n')
+        sys.stderr.flush()
+      return all_items
+
+
+# TODO: Make this private once all execution related items that use this method
+# have been brought into this file
+def handle_oauth_token_error(e, soft_errors):
+  """On a token error, exits the application and writes a message to stderr.
+
+  Args:
+    e: google.auth.exceptions.RefreshError, The error to handle.
+    soft_errors: Boolean, if True, suppresses any applicable errors and instead
+      returns to the caller.
+  """
+  token_error = str(e).replace('.', '')
+  if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
+      'Invalid response'):
+    if soft_errors:
+      return
+    if not GM_Globals[GM_CURRENT_API_USER]:
+      display.print_error(
+          MESSAGE_API_ACCESS_DENIED.format(
+              GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+              ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+      controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
+    else:
+      controlflow.system_error_exit(
+          19,
+          MESSAGE_SERVICE_NOT_APPLICABLE.format(
+              GM_Globals[GM_CURRENT_API_USER]))
+  controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
+
+def get_enum_values_minus_unspecified(values):
+  return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gapi/__init___test.py

@@ -0,0 +1,502 @@
+"""Tests for gapi."""
+
+import json
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import gapi
+from gapi import errors
+
+
+def create_http_error(status, reason, message):
+  """Creates a HttpError object similar to most Google API Errors.
+
+  Args:
+    status: Int, the error's HTTP response status number.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    googleapiclient.errors.HttpError
+  """
+  response = {
+      'status': status,
+      'content-type': 'application/json',
+  }
+  content = {
+      'error': {
+          'code': status,
+          'errors': [{
+              'reason': str(reason),
+              'message': message,
+          }]
+      }
+  }
+  content_bytes = json.dumps(content).encode('UTF-8')
+  return gapi.googleapiclient.errors.HttpError(response, content_bytes)
+
+
+class GapiTest(unittest.TestCase):
+
+  def setUp(self):
+    SetGlobalVariables()
+    self.mock_service = MagicMock()
+    self.mock_method_name = 'mock_method'
+    self.mock_method = getattr(self.mock_service, self.mock_method_name)
+
+    self.simple_3_page_response = [
+        {
+            'items': [{
+                'position': 'page1,item1'
+            }, {
+                'position': 'page1,item2'
+            }, {
+                'position': 'page1,item3'
+            }],
+            'nextPageToken': 'page2'
+        },
+        {
+            'items': [{
+                'position': 'page2,item1'
+            }, {
+                'position': 'page2,item2'
+            }, {
+                'position': 'page2,item3'
+            }],
+            'nextPageToken': 'page3'
+        },
+        {
+            'items': [{
+                'position': 'page3,item1'
+            }, {
+                'position': 'page3,item2'
+            }, {
+                'position': 'page3,item3'
+            }],
+        },
+    ]
+    self.empty_items_response = {'items': []}
+
+    super(GapiTest, self).setUp()
+
+  def test_call_returns_basic_200_response(self):
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, self.mock_method().execute.return_value)
+
+  def test_call_passes_target_method_params(self):
+    gapi.call(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    self.assertEqual(self.mock_method.call_count, 1)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(method_kwargs.get('my_param_1'), 1)
+    self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+  @patch.object(gapi.errors, 'get_gapi_error_detail')
+  def test_call_retries_with_soft_errors(self, mock_error_detail):
+    mock_error_detail.return_value = (-1, 'aReason', 'some message')
+
+    # Make the request fail first, then return the proper response on the retry.
+    fake_http_error = create_http_error(403, 'aReason', 'unused message')
+    fake_200_response = MagicMock()
+    self.mock_method.return_value.execute.side_effect = [
+        fake_http_error, fake_200_response
+    ]
+
+    response = gapi.call(
+        self.mock_service, self.mock_method_name, soft_errors=True)
+    self.assertEqual(response, fake_200_response)
+    self.assertEqual(
+        self.mock_service._http.request.credentials.refresh.call_count, 1)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+
+  def test_call_throws_for_provided_reason(self):
+    throw_reason = errors.ErrorReason.USER_NOT_FOUND
+    fake_http_error = create_http_error(404, throw_reason, 'forced throw')
+    self.mock_method.return_value.execute.side_effect = fake_http_error
+
+    gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
+    with self.assertRaises(gam_exception):
+      gapi.call(
+          self.mock_service,
+          self.mock_method_name,
+          throw_reasons=[throw_reason])
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_request_for_default_retry_reasons(
+      self, mock_wait_on_failure):
+
+    # Test using one of the default retry reasons
+    default_throw_reason = errors.ErrorReason.BACKEND_ERROR
+    self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
+
+    fake_http_error = create_http_error(404, default_throw_reason, 'message')
+    fake_200_response = MagicMock()
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_http_error, fake_200_response
+    ]
+
+    response = gapi.call(
+        self.mock_service, self.mock_method_name, retry_reasons=[])
+    self.assertEqual(response, fake_200_response)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+    # Make sure a backoff technique was used for retry.
+    self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_requests_for_provided_retry_reasons(
+      self, unused_mock_wait_on_failure):
+
+    retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
+    fake_retrieable_error1 = create_http_error(400, retry_reason1,
+                                               'Forced Error 1')
+    retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
+    fake_retrieable_error2 = create_http_error(400, retry_reason2,
+                                               'Forced Error 2')
+    non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
+    fake_non_retriable_error = create_http_error(
+        400, non_retriable_reason,
+        'This error should not cause the request to be retried')
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_retrieable_error1, fake_retrieable_error2, fake_non_retriable_error
+    ]
+
+    with self.assertRaises(SystemExit):
+      # The third call should raise the SystemExit when non_retriable_error is
+      # raised.
+      gapi.call(
+          self.mock_service,
+          self.mock_method_name,
+          retry_reasons=[retry_reason1, retry_reason2])
+
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
+
+  def test_call_exits_on_oauth_token_error(self):
+    # An error with any OAUTH2_TOKEN_ERROR
+    fake_token_error = gapi.google.auth.exceptions.RefreshError(
+        errors.OAUTH2_TOKEN_ERRORS[0])
+    self.mock_method.return_value.execute.side_effect = fake_token_error
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_exits_on_nonretriable_error(self):
+    error_reason = 'unknownReason'
+    fake_http_error = create_http_error(500, error_reason,
+                                        'Testing unretriable errors')
+    self.mock_method.return_value.execute.side_effect = fake_http_error
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_exits_on_request_valueerror(self):
+    self.mock_method.return_value.execute.side_effect = ValueError()
+
+    with self.assertRaises(SystemExit):
+      gapi.call(self.mock_service, self.mock_method_name)
+
+  def test_call_clears_bad_http_cache_on_request_failure(self):
+    self.mock_service._http.cache = 'something that is not None'
+    fake_200_response = MagicMock()
+    self.mock_method.return_value.execute.side_effect = [
+        ValueError(), fake_200_response
+    ]
+
+    self.assertIsNotNone(self.mock_service._http.cache)
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, fake_200_response)
+    # Assert the cache was cleared
+    self.assertIsNone(self.mock_service._http.cache)
+
+  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+  # we're not actually testing over a network connection
+  @patch.object(gapi.controlflow, 'wait_on_failure')
+  def test_call_retries_requests_with_backoff_on_servernotfounderror(
+      self, mock_wait_on_failure):
+    fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
+    fake_200_response = MagicMock()
+    # Fail once, then succeed on retry
+    self.mock_method.return_value.execute.side_effect = [
+        fake_servernotfounderror, fake_200_response
+    ]
+
+    http_connections = self.mock_service._http.connections
+    response = gapi.call(self.mock_service, self.mock_method_name)
+    self.assertEqual(response, fake_200_response)
+    # HTTP cached connections should be cleared on receiving this error
+    self.assertNotEqual(http_connections, self.mock_service._http.connections)
+    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+    # Make sure a backoff technique was used for retry.
+    self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+  def test_get_items_calls_correct_service_function(self):
+    gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertTrue(self.mock_method.called)
+
+  def test_get_items_returns_one_page(self):
+    fake_response = {'items': [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertEqual(page, fake_response['items'])
+
+  def test_get_items_non_default_page_field_name(self):
+    field_name = 'things'
+    fake_response = {field_name: [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_items(
+        self.mock_service, self.mock_method_name, items=field_name)
+    self.assertEqual(page, fake_response[field_name])
+
+  def test_get_items_passes_additional_kwargs_to_service(self):
+    gapi.get_items(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    self.assertEqual(self.mock_method.call_count, 1)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(1, method_kwargs.get('my_param_1'))
+    self.assertEqual(2, method_kwargs.get('my_param_2'))
+
+  def test_get_items_returns_empty_list_when_no_items_returned(self):
+    non_items_response = {'noItemsInThisResponse': {}}
+    self.mock_method.return_value.execute.return_value = non_items_response
+    page = gapi.get_items(self.mock_service, self.mock_method_name)
+    self.assertIsInstance(page, list)
+    self.assertEqual(0, len(page))
+
+  def test_get_all_pages_returns_all_items(self):
+    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
+    page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
+    page_3 = {'items': ['3-1', '3-2', '3-3']}
+    self.mock_method.return_value.execute.side_effect = [page_1, page_2, page_3]
+    response_items = gapi.get_all_pages(self.mock_service,
+                                        self.mock_method_name)
+    self.assertListEqual(response_items,
+                         page_1['items'] + page_2['items'] + page_3['items'])
+
+  def test_get_all_pages_includes_next_pagetoken_in_request(self):
+    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
+    page_2 = {'items': ['2-1', '2-2', '2-3']}
+    self.mock_method.return_value.execute.side_effect = [page_1, page_2]
+
+    gapi.get_all_pages(self.mock_service, self.mock_method_name, pageSize=100)
+    self.assertEqual(self.mock_method.call_count, 2)
+    call_2_kwargs = self.mock_method.call_args_list[1][1]
+    self.assertIn('pageToken', call_2_kwargs)
+    self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
+
+  def test_get_all_pages_uses_default_max_page_size(self):
+    sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
+    sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
+    self.mock_method.return_value.methodId = sample_api_id
+    self.mock_service._rootDesc = {
+        'resources': {
+            'someResource': {
+                'methods': {
+                    'someMethod': {
+                        'id': sample_api_id,
+                        'parameters': {
+                            'maxResults': {
+                                'maximum': sample_api_max_results
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+    gapi.get_all_pages(self.mock_service, self.mock_method_name)
+    request_method_kwargs = self.mock_method.call_args[1]
+    self.assertIn('maxResults', request_method_kwargs)
+    self.assertEqual(request_method_kwargs['maxResults'],
+                     gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
+
+  def test_get_all_pages_max_page_size_overrided(self):
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+    gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, pageSize=123456)
+    request_method_kwargs = self.mock_method.call_args[1]
+    self.assertIn('pageSize', request_method_kwargs)
+    self.assertEqual(123456, request_method_kwargs['pageSize'])
+
+  def test_get_all_pages_prints_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    self.assertIn(paging_message, messages_written)
+
+  def test_get_all_pages_prints_paging_message_inline(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+
+    # Make sure a return carriage was written between two pages
+    paging_message_call_positions = [
+        i for i, message in enumerate(messages_written)
+        if message == paging_message
+    ]
+    self.assertGreater(len(paging_message_call_positions), 1)
+    printed_between_page_messages = messages_written[
+        paging_message_call_positions[0]:paging_message_call_positions[1]]
+    self.assertIn('\r', printed_between_page_messages)
+
+  def test_get_all_pages_ends_paging_message_with_newline(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'A simple string displayed during paging'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    last_page_message_index = len(
+        messages_written) - messages_written[::-1].index(paging_message)
+    last_carriage_return_index = len(
+        messages_written) - messages_written[::-1].index('\r\n')
+    self.assertGreater(last_carriage_return_index, last_page_message_index)
+
+  def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'Total number of items discovered: %%total_items%%'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_item_count = len(self.simple_3_page_response[0]['items'])
+    page_1_message = paging_message.replace('%%total_items%%',
+                                            str(page_1_item_count))
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_item_count = len(self.simple_3_page_response[1]['items'])
+    page_2_message = paging_message.replace(
+        '%%total_items%%', str(page_1_item_count + page_2_item_count))
+    self.assertIn(page_2_message, messages_written)
+
+    page_3_item_count = len(self.simple_3_page_response[2]['items'])
+    page_3_message = paging_message.replace(
+        '%%total_items%%',
+        str(page_1_item_count + page_2_item_count + page_3_item_count))
+    self.assertIn(page_3_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%total_items', message)
+
+  def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'First item in page: %%first_item%%'
+
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_message = paging_message.replace(
+        '%%first_item%%',
+        self.simple_3_page_response[0]['items'][0]['position'])
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_message = paging_message.replace(
+        '%%first_item%%',
+        self.simple_3_page_response[1]['items'][0]['position'])
+    self.assertIn(page_2_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%first_item', message)
+
+  def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
+    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+    paging_message = 'Last item in page: %%last_item%%'
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
+
+    messages_written = [
+        call_args[0][0] for call_args in mock_write.call_args_list
+    ]
+    page_1_message = paging_message.replace(
+        '%%last_item%%',
+        self.simple_3_page_response[0]['items'][-1]['position'])
+    self.assertIn(page_1_message, messages_written)
+
+    page_2_message = paging_message.replace(
+        '%%last_item%%',
+        self.simple_3_page_response[1]['items'][-1]['position'])
+    self.assertIn(page_2_message, messages_written)
+
+    # Assert that the template text is always replaced.
+    for message in messages_written:
+      self.assertNotIn('%%last_item', message)
+
+  def test_get_all_pages_prints_all_attributes_in_paging_message(self):
+    pass
+
+  def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
+    self.mock_method.return_value.execute.return_value = self.empty_items_response
+    gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
+    method_kwargs = self.mock_method.call_args[1]
+    self.assertEqual(method_kwargs.get('my_param_1'), 1)
+    self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+  @patch.object(gapi, 'call')
+  def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
+    throw_for = MagicMock()
+    retry_for = MagicMock()
+    mock_call.return_value = self.empty_items_response
+    gapi.get_all_pages(
+        self.mock_service,
+        self.mock_method_name,
+        throw_reasons=throw_for,
+        retry_reasons=retry_for)
+    method_kwargs = mock_call.call_args[1]
+    self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
+    self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
+
+  def test_get_all_pages_non_default_items_field_name(self):
+    field_name = 'things'
+    fake_response = {field_name: [{}, {}, {}]}
+    self.mock_method.return_value.execute.return_value = fake_response
+    page = gapi.get_all_pages(
+        self.mock_service, self.mock_method_name, items=field_name)
+    self.assertEqual(page, fake_response[field_name])
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gapi/calendar.py

@@ -0,0 +1,893 @@
+import csv
+import sys
+import uuid
+
+# TODO: get rid of these hacks
+import __main__
+from var import *
+
+import controlflow
+import display
+import fileutils
+import gapi
+import utils
+
+
+def normalizeCalendarId(calname, checkPrimary=False):
+    if checkPrimary and calname.lower() == 'primary':
+        return calname
+    if not GC_Values[GC_DOMAIN]:
+        GC_Values[GC_DOMAIN] = __main__._getValueFromOAuth('hd')
+    return __main__.convertUIDtoEmailAddress(calname,
+                                             email_types=['user', 'resource'])
+
+
+def buildCalendarGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+    return (calendarId, __main__.buildGAPIServiceObject('calendar',
+                                                        calendarId))
+
+
+def buildCalendarDataGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+
+    # Try to impersonate the calendar owner. If we fail, fall back to using
+    # admin for authentication. Resource calendars cannot be impersonated,
+    # so we need to access them as the admin.
+    cal = None
+    if not calname.endswith('.calendar.google.com'):
+        cal = __main__.buildGAPIServiceObject('calendar', calendarId, False)
+    if cal is None:
+        _, cal = buildCalendarGAPIObject(__main__._getValueFromOAuth('email'))
+    return (calendarId, cal)
+
+def printShowACLs(csvFormat):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    toDrive = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if csvFormat and myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            action = ['showacl', 'printacl'][csvFormat]
+            message = f"gam calendar <email> {action}"
+            controlflow.invalid_argument_exit(sys.argv[i], message)
+    acls = gapi.get_all_pages(
+        cal.acl(), 'list', 'items', calendarId=calendarId)
+    i = 0
+    if csvFormat:
+        titles = []
+        rows = []
+    else:
+        count = len(acls)
+    for rule in acls:
+        i += 1
+        if csvFormat:
+            row = utils.flatten_json(rule, None)
+            for key in row:
+                if key not in titles:
+                    titles.append(key)
+            rows.append(row)
+        else:
+            formatted_acl = formatACLRule(rule)
+            current_count = display.current_count(i, count)
+            print(f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
+    if csvFormat:
+        display.write_csv_file(
+            rows, titles, f'{calendarId} Calendar ACLs', toDrive)
+
+
+def _getCalendarACLScope(i, body):
+    body['scope'] = {}
+    myarg = sys.argv[i].lower()
+    body['scope']['type'] = myarg
+    i += 1
+    if myarg in ['user', 'group']:
+        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
+            sys.argv[i], noUid=True)
+        i += 1
+    elif myarg == 'domain':
+        if i < len(sys.argv) and \
+           sys.argv[i].lower().replace('_', '') != 'sendnotifications':
+            body['scope']['value'] = sys.argv[i].lower()
+            i += 1
+        else:
+            body['scope']['value'] = GC_Values[GC_DOMAIN]
+    elif myarg != 'default':
+        body['scope']['type'] = 'user'
+        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
+            myarg, noUid=True)
+    return i
+
+
+CALENDAR_ACL_ROLES_MAP = {
+    'editor': 'writer',
+    'freebusy': 'freeBusyReader',
+    'freebusyreader': 'freeBusyReader',
+    'owner': 'owner',
+    'read': 'reader',
+    'reader': 'reader',
+    'writer': 'writer',
+    'none': 'none',
+}
+
+
+def addACL(function):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    myarg = sys.argv[4].lower().replace('_', '')
+    if myarg not in CALENDAR_ACL_ROLES_MAP:
+        controlflow.expected_argument_exit(
+            "Role", ", ".join(CALENDAR_ACL_ROLES_MAP), myarg)
+    body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]}
+    i = _getCalendarACLScope(5, body)
+    sendNotifications = True
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'sendnotifications':
+            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam calendar <email> {function.lower()}")
+    print(f'Calendar: {calendarId}, {function} ACL: {formatACLRule(body)}')
+    gapi.call(cal.acl(), 'insert', calendarId=calendarId,
+              body=body, sendNotifications=sendNotifications)
+
+
+def delACL():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    if sys.argv[4].lower() == 'id':
+        ruleId = sys.argv[5]
+        print(f'Removing rights for {ruleId} to {calendarId}')
+        gapi.call(cal.acl(), 'delete', calendarId=calendarId, ruleId=ruleId)
+    else:
+        body = {'role': 'none'}
+        _getCalendarACLScope(5, body)
+        print(f'Calendar: {calendarId}, Delete ACL: {formatACLScope(body)}')
+        gapi.call(cal.acl(), 'insert', calendarId=calendarId,
+                  body=body, sendNotifications=False)
+
+
+def wipeData():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    gapi.call(cal.calendars(), 'clear', calendarId=calendarId)
+
+
+def printEvents():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    q = showDeleted = showHiddenInvitations = timeMin = \
+        timeMax = timeZone = updatedMin = None
+    toDrive = False
+    titles = []
+    csvRows = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            q = sys.argv[i+1]
+            i += 2
+        elif myarg == 'includedeleted':
+            showDeleted = True
+            i += 1
+        elif myarg == 'includehidden':
+            showHiddenInvitations = True
+            i += 1
+        elif myarg == 'after':
+            timeMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'before':
+            timeMax = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i+1]
+            i += 2
+        elif myarg == 'updated':
+            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam calendar <email> printevents")
+    page_message = gapi.got_total_items_msg(f'Events for {calendarId}', '')
+    results = gapi.get_all_pages(cal.events(), 'list', 'items',
+                                 page_message=page_message,
+                                 calendarId=calendarId, q=q,
+                                 showDeleted=showDeleted,
+                                 showHiddenInvitations=showHiddenInvitations,
+                                 timeMin=timeMin, timeMax=timeMax,
+                                 timeZone=timeZone,
+                                 updatedMin=updatedMin)
+    for result in results:
+        row = {'calendarId': calendarId}
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(result, flattened=row), csvRows, titles)
+    display.sort_csv_titles(['calendarId', 'id', 'summary', 'status'], titles)
+    display.write_csv_file(csvRows, titles, 'Calendar Events', toDrive)
+
+
+def formatACLScope(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]})'
+    return f'(Scope: {rule["scope"]["type"]})'
+
+
+def formatACLRule(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]}, ' \
+               f'Role: {rule["role"]})'
+    return f'(Scope: {rule["scope"]["type"]}, Role: {rule["role"]})'
+
+
+def getSendUpdates(myarg, i, cal):
+    if myarg == 'notifyattendees':
+        sendUpdates = 'all'
+        i += 1
+    elif myarg == 'sendnotifications':
+        sendUpdates = 'all' if __main__.getBoolean(sys.argv[i+1], myarg) else 'none'
+        i += 2
+    else:  # 'sendupdates':
+        sendUpdatesMap = {}
+        for val in cal._rootDesc['resources']['events']['methods']['delete'][
+                'parameters']['sendUpdates']['enum']:
+            sendUpdatesMap[val.lower()] = val
+        sendUpdates = sendUpdatesMap.get(sys.argv[i+1].lower(), False)
+        if not sendUpdates:
+            controlflow.expected_argument_exit(
+                "sendupdates", ", ".join(sendUpdatesMap), sys.argv[i+1])
+        i += 2
+    return (sendUpdates, i)
+
+
+def moveOrDeleteEvent(moveOrDelete):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    sendUpdates = None
+    doit = False
+    kwargs = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg in ['id', 'eventid']:
+            eventId = sys.argv[i+1]
+            i += 2
+        elif myarg in ['query', 'eventquery']:
+            controlflow.system_error_exit(
+                2, f'query is no longer supported for {moveOrDelete}event. ' \
+                   f'Use "gam calendar <email> printevents query <query> | ' \
+                   f'gam csv - gam {moveOrDelete}event id ~id" instead.')
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        elif moveOrDelete == 'move' and myarg == 'destination':
+            kwargs['destination'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam calendar <email> {moveOrDelete}event")
+    if doit:
+        print(f' going to {moveOrDelete} eventId {eventId}')
+        gapi.call(cal.events(), moveOrDelete, calendarId=calendarId,
+                  eventId=eventId, sendUpdates=sendUpdates, **kwargs)
+    else:
+        print(
+            f' would {moveOrDelete} eventId {eventId}. Add doit to command ' \
+            f'to actually {moveOrDelete} event')
+
+
+def infoEvent():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    eventId = sys.argv[4]
+    result = gapi.call(cal.events(), 'get',
+                       calendarId=calendarId, eventId=eventId)
+    display.print_json(result)
+
+
+def addOrUpdateEvent(action):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    # only way for non-Google calendars to get updates is via email
+    timeZone = None
+    kwargs = {}
+    body = {}
+    if action == 'add':
+        i = 4
+        func = 'insert'
+    else:
+        eventId = sys.argv[4]
+        kwargs = {'eventId': eventId}
+        i = 5
+        func = 'patch'
+        requires_full_update = ['attendee', 'optionalattendee',
+                                'removeattendee', 'replacedescription']
+        for arg in sys.argv[i:]:
+            if arg.replace('_', '').lower() in requires_full_update:
+                func = 'update'
+                body = gapi.call(cal.events(), 'get',
+                                 calendarId=calendarId, eventId=eventId)
+                break
+    sendUpdates, body = getEventAttributes(i, calendarId, cal, body, action)
+    result = gapi.call(cal.events(), func, conferenceDataVersion=1,
+                       supportsAttachments=True, calendarId=calendarId,
+                       sendUpdates=sendUpdates, body=body, fields='id',
+                       **kwargs)
+    print(f'Event {result["id"]} {action} finished')
+
+
+def _remove_attendee(attendees, remove_email):
+    return [attendee for attendee in attendees
+            if not attendee['email'].lower() == remove_email]
+
+
+def getEventAttributes(i, calendarId, cal, body, action):
+    # Default to external only so non-Google
+    # calendars are notified of changes
+    sendUpdates = 'externalOnly'
+    action = 'update' if body else 'add'
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg == 'attendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({'email': sys.argv[i+1]})
+            i += 2
+        elif myarg == 'removeattendee' and action == 'update':
+            remove_email = sys.argv[i+1].lower()
+            body['attendees'] = _remove_attendee(body['attendees'],
+                                                 remove_email)
+            i += 2
+        elif myarg == 'optionalattendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append(
+                {'email': sys.argv[i+1], 'optional': True})
+            i += 2
+        elif myarg == 'anyonecanaddself':
+            body['anyoneCanAddSelf'] = True
+            i += 1
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'replacedescription':
+            search = sys.argv[i+1]
+            replace = sys.argv[i+2]
+            body['description'] = re.sub(search, replace, body['description'])
+            i += 3
+        elif myarg == 'start':
+            if sys.argv[i+1].lower() == 'allday':
+                body['start'] = {'date': __main__.getYYYYMMDD(sys.argv[i+2])}
+                i += 3
+            else:
+                start_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                body['start'] = {'dateTime': start_time}
+                i += 2
+        elif myarg == 'end':
+            if sys.argv[i+1].lower() == 'allday':
+                body['end'] = {'date': __main__.getYYYYMMDD(sys.argv[i+2])}
+                i += 3
+            else:
+                end_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                body['end'] = {'dateTime': end_time}
+                i += 2
+        elif myarg == 'guestscantinviteothers':
+            body['guestsCanInviteOthers'] = False
+            i += 1
+        elif myarg == 'guestscaninviteothers':
+            body['guestsCanInviteTohters'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscaninviteothers')
+            i += 2
+        elif myarg == 'guestscantseeothers':
+            body['guestsCanSeeOtherGuests'] = False
+            i += 1
+        elif myarg == 'guestscanseeothers':
+            body['guestsCanSeeOtherGuests'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscanseeothers')
+            i += 2
+        elif myarg == 'guestscanmodify':
+            body['guestsCanModify'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscanmodify')
+            i += 2
+        elif myarg == 'id':
+            if action == 'update':
+                controlflow.invalid_argument_exit(
+                    'id', 'gam calendar <calendar> updateevent')
+            body['id'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'available':
+            body['transparency'] = 'transparent'
+            i += 1
+        elif myarg == 'transparency':
+            validTransparency = ['opaque', 'transparent']
+            if sys.argv[i+1].lower() in validTransparency:
+                body['transparency'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    'transparency',
+                    ", ".join(validTransparency), sys.argv[i+1])
+            i += 2
+        elif myarg == 'visibility':
+            validVisibility = ['default', 'public', 'private']
+            if sys.argv[i+1].lower() in validVisibility:
+                body['visibility'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    "visibility", ", ".join(validVisibility), sys.argv[i+1])
+            i += 2
+        elif myarg == 'tentative':
+            body['status'] = 'tentative'
+            i += 1
+        elif myarg == 'status':
+            validStatus = ['confirmed', 'tentative', 'cancelled']
+            if sys.argv[i+1].lower() in validStatus:
+                body['status'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    'visibility', ', '.join(validStatus), sys.argv[i+1])
+            i += 2
+        elif myarg == 'source':
+            body['source'] = {'title': sys.argv[i+1], 'url': sys.argv[i+2]}
+            i += 3
+        elif myarg == 'noreminders':
+            body['reminders'] = {'useDefault': False}
+            i += 1
+        elif myarg == 'reminder':
+            minutes = \
+            __main__.getInteger(sys.argv[i+1], myarg, minVal=0,
+                                maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+            reminder = {'minutes': minutes, 'method': sys.argv[i+2]}
+            body.setdefault(
+                'reminders', {'overrides': [], 'useDefault': False})
+            body['reminders']['overrides'].append(reminder)
+            i += 3
+        elif myarg == 'recurrence':
+            body.setdefault('recurrence', [])
+            body['recurrence'].append(sys.argv[i+1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i+1]
+            i += 2
+        elif myarg == 'privateproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['private'][sys.argv[i+1]] = sys.argv[i+2]
+            i += 3
+        elif myarg == 'sharedproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['shared'][sys.argv[i+1]] = sys.argv[i+2]
+            i += 3
+        elif myarg == 'colorindex':
+            body['colorId'] = __main__.getInteger(
+                sys.argv[i+1], myarg, CALENDAR_EVENT_MIN_COLOR_INDEX,
+                CALENDAR_EVENT_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'hangoutsmeet':
+            body['conferenceData'] = {'createRequest': {
+                'requestId': f'{str(uuid.uuid4())}'}}
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {action}event')
+    if ('recurrence' in body) and (('start' in body) or ('end' in body)):
+        if not timeZone:
+            timeZone = gapi.call(cal.calendars(), 'get',
+                                 calendarId=calendarId,
+                                 fields='timeZone')['timeZone']
+        if 'start' in body:
+            body['start']['timeZone'] = timeZone
+        if 'end' in body:
+            body['end']['timeZone'] = timeZone
+    return (sendUpdates, body)
+
+
+def modifySettings():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'timezone':
+            body['timeZone'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam calendar <email> modify")
+    gapi.call(cal.calendars(), 'patch', calendarId=calendarId, body=body)
+
+
+def changeAttendees(users):
+    do_it = True
+    i = 5
+    allevents = False
+    start_date = end_date = None
+    while len(sys.argv) > i:
+        myarg = sys.argv[i].lower()
+        if myarg == 'csv':
+            csv_file = sys.argv[i+1]
+            i += 2
+        elif myarg == 'dryrun':
+            do_it = False
+            i += 1
+        elif myarg == 'start':
+            start_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'end':
+            end_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'allevents':
+            allevents = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam <users> update calattendees")
+    attendee_map = {}
+    f = fileutils.open_file(csv_file)
+    csvFile = csv.reader(f)
+    for row in csvFile:
+        attendee_map[row[0].lower()] = row[1].lower()
+    fileutils.close_file(f)
+    for user in users:
+        sys.stdout.write(f'Checking user {user}\n')
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        page_token = None
+        while True:
+            events_page = gapi.call(cal.events(), 'list', calendarId=user,
+                                    pageToken=page_token, timeMin=start_date,
+                                    timeMax=end_date, showDeleted=False,
+                                    showHiddenInvitations=False)
+            print(f'Got {len(events_page.get("items", []))}')
+            for event in events_page.get('items', []):
+                if event['status'] == 'cancelled':
+                    # print u' skipping cancelled event'
+                    continue
+                try:
+                    event_summary = event['summary']
+                except (KeyError, UnicodeEncodeError, UnicodeDecodeError):
+                    event_summary = event['id']
+                try:
+                    organizer = event['organizer']['email'].lower()
+                    if not allevents and organizer != user:
+                        #print(f' skipping not-my-event {event_summary}')
+                        continue
+                except KeyError:
+                    pass  # no email for organizer
+                needs_update = False
+                try:
+                    for attendee in event['attendees']:
+                        try:
+                            if attendee['email'].lower() in attendee_map:
+                                old_email = attendee['email'].lower()
+                                new_email = attendee_map[attendee['email'].lower(
+                                )]
+                                print(f' SWITCHING attendee {old_email} to ' \
+                                    f'{new_email} for {event_summary}')
+                                event['attendees'].remove(attendee)
+                                event['attendees'].append({'email': new_email})
+                                needs_update = True
+                        except KeyError:  # no email for that attendee
+                            pass
+                except KeyError:
+                    continue  # no attendees
+                if needs_update:
+                    body = {}
+                    body['attendees'] = event['attendees']
+                    print(f'UPDATING {event_summary}')
+                    if do_it:
+                        gapi.call(cal.events(), 'patch', calendarId=user,
+                                  eventId=event['id'],
+                                  sendNotifications=False, body=body)
+                    else:
+                        print(' not pulling the trigger.')
+                # else:
+                #  print(f' no update needed for {event_summary}')
+            try:
+                page_token = events_page['nextPageToken']
+            except KeyError:
+                break
+
+
+def deleteCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    for user in users:
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        gapi.call(cal.calendarList(), 'delete',
+                  soft_errors=True, calendarId=calendarId)
+
+
+CALENDAR_REMINDER_MAX_MINUTES = 40320
+
+CALENDAR_MIN_COLOR_INDEX = 1
+CALENDAR_MAX_COLOR_INDEX = 24
+
+CALENDAR_EVENT_MIN_COLOR_INDEX = 1
+CALENDAR_EVENT_MAX_COLOR_INDEX = 11
+
+
+def getCalendarAttributes(i, body, function):
+    colorRgbFormat = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'selected':
+            body['selected'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg == 'hidden':
+            body['hidden'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg == 'summary':
+            body['summaryOverride'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'colorindex':
+            body['colorId'] = __main__.getInteger(
+                sys.argv[i+1], myarg, minVal=CALENDAR_MIN_COLOR_INDEX,
+                maxVal=CALENDAR_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'backgroundcolor':
+            body['backgroundColor'] = __main__.getColor(sys.argv[i+1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'foregroundcolor':
+            body['foregroundColor'] = __main__.getColor(sys.argv[i+1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'reminder':
+            body.setdefault('defaultReminders', [])
+            method = sys.argv[i+1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_REMINDER_METHODS:
+                    controlflow.expected_argument_exit("Method", ", ".join(
+                        CALENDAR_REMINDER_METHODS+CLEAR_NONE_ARGUMENT), method)
+                minutes = __main__.getInteger(
+                    sys.argv[i+2], myarg, minVal=0,
+                    maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+                body['defaultReminders'].append(
+                    {'method': method, 'minutes': minutes})
+                i += 3
+            else:
+                i += 2
+        elif myarg == 'notification':
+            body.setdefault('notificationSettings', {'notifications': []})
+            method = sys.argv[i+1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_NOTIFICATION_METHODS:
+                    controlflow.expected_argument_exit("Method", ", ".join(
+                        CALENDAR_NOTIFICATION_METHODS+CLEAR_NONE_ARGUMENT), method)
+                eventType = sys.argv[i+2].lower()
+                if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP:
+                    controlflow.expected_argument_exit("Event", ", ".join(
+                        CALENDAR_NOTIFICATION_TYPES_MAP), eventType)
+                notice = {'method': method,
+                          'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]}
+                body['notificationSettings']['notifications'].append(notice)
+                i += 3
+            else:
+                i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam {function} calendar")
+    return colorRgbFormat
+
+
+def addCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    body = {'id': calendarId, 'selected': True, 'hidden': False}
+    colorRgbFormat = getCalendarAttributes(6, body, 'add')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f'Subscribing {user} to calendar {calendarId}{current_count}')
+        gapi.call(cal.calendarList(), 'insert', soft_errors=True,
+                  body=body, colorRgbFormat=colorRgbFormat)
+
+
+def updateCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    body = {}
+    colorRgbFormat = getCalendarAttributes(6, body, 'update')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f"Updating {user}'s subscription to calendar ' \
+              f'{calendarId}{current_count}")
+        calId = calendarId if calendarId != 'primary' else user
+        gapi.call(cal.calendarList(), 'patch', soft_errors=True,
+                  calendarId=calId, body=body, colorRgbFormat=colorRgbFormat)
+
+
+def _showCalendar(userCalendar, j, jcount):
+    current_count = display.current_count(j, jcount)
+    summary = userCalendar.get("summaryOverride", userCalendar["summary"])
+    print(f'  Calendar: {userCalendar["id"]}{current_count}')
+    print(f'    Summary: {summary}')
+    print(f'    Description: {userCalendar.get("description", "")}')
+    print(f'    Access Level: {userCalendar["accessRole"]}')
+    print(f'    Timezone: {userCalendar["timeZone"]}')
+    print(f'    Location: {userCalendar.get("location", "")}')
+    print(f'    Hidden: {userCalendar.get("hidden", "False")}')
+    print(f'    Selected: {userCalendar.get("selected", "False")}')
+    print(f'    Color ID: {userCalendar["colorId"]}, ' \
+          f'Background Color: {userCalendar["backgroundColor"]}, ' \
+          f'Foreground Color: {userCalendar["foregroundColor"]}')
+    print(f'    Default Reminders:')
+    for reminder in userCalendar.get('defaultReminders', []):
+        print(f'      Method: {reminder["method"]}, ' \
+              f'Minutes: {reminder["minutes"]}')
+    print('    Notifications:')
+    if 'notificationSettings' in userCalendar:
+        notifications = userCalendar['notificationSettings'].get(
+            'notifications', [])
+        for notification in notifications:
+            print(f'      Method: {notification["method"]}, ' \
+                  f'Type: {notification["type"]}')
+
+
+def infoCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.call(cal.calendarList(), 'get',
+                           soft_errors=True,
+                           calendarId=calendarId)
+        if result:
+            print(f'User: {user}, Calendar:{display.current_count(i, count)}')
+            _showCalendar(result, 1, 1)
+
+
+def printShowCalendars(users, csvFormat):
+    if csvFormat:
+        todrive = False
+        titles = []
+        csvRows = []
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, f"gam <users> {['show', 'print'][csvFormat]} calendars")
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.get_all_pages(
+            cal.calendarList(), 'list', 'items', soft_errors=True)
+        jcount = len(result)
+        if not csvFormat:
+            print(f'User: {user}, Calendars:{display.current_count(i, count)}')
+            if jcount == 0:
+                continue
+            j = 0
+            for userCalendar in result:
+                j += 1
+                _showCalendar(userCalendar, j, jcount)
+        else:
+            if jcount == 0:
+                continue
+            for userCalendar in result:
+                row = {'primaryEmail': user}
+                display.add_row_titles_to_csv_file(utils.flatten_json(
+                    userCalendar, flattened=row), csvRows, titles)
+    if csvFormat:
+        display.sort_csv_titles(['primaryEmail', 'id'], titles)
+        display.write_csv_file(csvRows, titles, 'Calendars', todrive)
+
+
+def showCalSettings(users):
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        feed = gapi.get_all_pages(
+            cal.settings(), 'list', 'items', soft_errors=True)
+        if feed:
+            current_count = display.current_count(i, count)
+            print(f'User: {user}, Calendar Settings:{current_count}')
+            settings = {}
+            for setting in feed:
+                settings[setting['id']] = setting['value']
+            for attr, value in sorted(settings.items()):
+                print(f'  {attr}: {value}')
+
+
+def transferSecCals(users):
+    target_user = sys.argv[5]
+    remove_source_user = sendNotifications = True
+    i = 6
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'keepuser':
+            remove_source_user = False
+            i += 1
+        elif myarg == 'sendnotifications':
+            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam <users> transfer seccals")
+    if remove_source_user:
+        target_user, target_cal = buildCalendarGAPIObject(target_user)
+        if not target_cal:
+            return
+    for user in users:
+        user, source_cal = buildCalendarGAPIObject(user)
+        if not source_cal:
+            continue
+        calendars = gapi.get_all_pages(source_cal.calendarList(), 'list',
+                                       'items', soft_errors=True,
+                                       minAccessRole='owner', showHidden=True,
+                                       fields='items(id),nextPageToken')
+        for calendar in calendars:
+            calendarId = calendar['id']
+            if calendarId.find('@group.calendar.google.com') != -1:
+                body = {'role': 'owner',
+                        'scope': {'type': 'user', 'value': target_user}}
+                gapi.call(source_cal.acl(), 'insert', calendarId=calendarId,
+                          body=body, sendNotifications=sendNotifications)
+                if remove_source_user:
+                    body = {'role': 'none',
+                            'scope': {'type': 'user', 'value': user}}
+                    gapi.call(target_cal.acl(), 'insert',
+                              calendarId=calendarId, body=body,
+                              sendNotifications=sendNotifications)

src/gapi/directory/__init__.py

@@ -0,0 +1,5 @@
+import __main__
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('directory')

src/gapi/directory/cros.py

@@ -0,0 +1,795 @@
+import datetime
+
+from var import *
+import __main__
+import controlflow
+import display
+import fileutils
+import gapi
+import gapi.directory
+import utils
+
+
+def doUpdateCros():
+    cd = gapi.directory.buildGAPIObject()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    update_body = {}
+    action_body = {}
+    orgUnitPath = None
+    ack_wipe = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'user':
+            update_body['annotatedUser'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            update_body['annotatedLocation'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'notes':
+            update_body['notes'] = sys.argv[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg in ['tag', 'asset', 'assetid']:
+            update_body['annotatedAssetId'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['ou', 'org']:
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'action':
+            action = sys.argv[i+1].lower().replace('_', '').replace('-', '')
+            deprovisionReason = None
+            if action in ['deprovisionsamemodelreplace',
+                          'deprovisionsamemodelreplacement']:
+                action = 'deprovision'
+                deprovisionReason = 'same_model_replacement'
+            elif action in ['deprovisiondifferentmodelreplace',
+                            'deprovisiondifferentmodelreplacement']:
+                action = 'deprovision'
+                deprovisionReason = 'different_model_replacement'
+            elif action in ['deprovisionretiringdevice']:
+                action = 'deprovision'
+                deprovisionReason = 'retiring_device'
+            elif action not in ['disable', 'reenable']:
+                controlflow.system_error_exit(2, f'expected action of ' \
+                    f'deprovision_same_model_replace, ' \
+                    f'deprovision_different_model_replace, ' \
+                    f'deprovision_retiring_device, disable or reenable,'
+                    f' got {action}')
+            action_body = {'action': action}
+            if deprovisionReason:
+                action_body['deprovisionReason'] = deprovisionReason
+            i += 2
+        elif myarg == 'acknowledgedevicetouchrequirement':
+            ack_wipe = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam update cros")
+    i = 0
+    count = len(devices)
+    if action_body:
+        if action_body['action'] == 'deprovision' and not ack_wipe:
+            print(f'WARNING: Refusing to deprovision {count} devices because '
+                  'acknowledge_device_touch_requirement not specified. ' \
+                  'Deprovisioning a device means the device will have to ' \
+                  'be physically wiped and re-enrolled to be managed by ' \
+                  'your domain again. This requires physical access to ' \
+                  'the device and is very time consuming to perform for ' \
+                  'each device. Please add ' \
+                  '"acknowledge_device_touch_requirement" to the GAM ' \
+                  'command if you understand this and wish to proceed ' \
+                  'with the deprovision. Please also be aware that ' \
+                  'deprovisioning can have an effect on your device ' \
+                  'license count. See ' \
+                  'https://support.google.com/chrome/a/answer/3523633 '\
+                  'for full details.')
+            sys.exit(3)
+        for deviceId in devices:
+            i += 1
+            cur_count = __main__.currentCount(i, count)
+            print(f' performing action {action} for {deviceId}{cur_count}')
+            gapi.call(cd.chromeosdevices(), function='action',
+                      customerId=GC_Values[GC_CUSTOMER_ID],
+                      resourceId=deviceId, body=action_body)
+    else:
+        if update_body:
+            for deviceId in devices:
+                i += 1
+                current_count = __main__.currentCount(i, count)
+                print(f' updating {deviceId}{current_count}')
+                gapi.call(cd.chromeosdevices(), 'update',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          deviceId=deviceId, body=update_body)
+        if orgUnitPath:
+            # split moves into max 50 devices per batch
+            for l in range(0, len(devices), 50):
+                move_body = {'deviceIds': devices[l:l+50]}
+                print(f' moving {len(move_body["deviceIds"])} devices to ' \
+                      f'{orgUnitPath}')
+                gapi.call(cd.chromeosdevices(), 'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath, body=move_body)
+
+
+def doGetCrosInfo():
+    cd = gapi.directory.buildGAPIObject()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    downloadfile = None
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    projection = None
+    fieldsList = []
+    noLists = False
+    startDate = endDate = None
+    listLimit = 0
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nolists':
+            noLists = True
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=-1)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            fieldsList = []
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg])
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i+1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field])
+                    if field in CROS_ACTIVE_TIME_RANGES_ARGUMENTS + \
+                                CROS_DEVICE_FILES_ARGUMENTS + \
+                                CROS_RECENT_USERS_ARGUMENTS:
+                        projection = 'FULL'
+                        noLists = False
+                else:
+                    controlflow.invalid_argument_exit(
+                        field, "gam info cros fields")
+            i += 2
+        elif myarg == 'downloadfile':
+            downloadfile = sys.argv[i+1]
+            if downloadfile.lower() == 'latest':
+                downloadfile = downloadfile.lower()
+            i += 2
+        elif myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i+1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam info cros")
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = ','.join(set(fieldsList)).replace('.', '/')
+    else:
+        fields = None
+    i = 0
+    device_count = len(devices)
+    for deviceId in devices:
+        i += 1
+        cros = gapi.call(cd.chromeosdevices(), 'get',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         deviceId=deviceId, projection=projection,
+                         fields=fields)
+        print(f'CrOS Device: {deviceId} ({i} of {device_count})')
+        if 'notes' in cros:
+            cros['notes'] = cros['notes'].replace('\n', '\\n')
+        if 'autoUpdateExpiration' in cros:
+            cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                cros['autoUpdateExpiration'])
+        _checkTPMVulnerability(cros)
+        for up in CROS_SCALAR_PROPERTY_PRINT_ORDER:
+            if up in cros:
+                if isinstance(cros[up], str):
+                    print(f'  {up}: {cros[up]}')
+                else:
+                    sys.stdout.write(f'  {up}:')
+                    display.print_json(cros[up], '  ')
+        if not noLists:
+            activeTimeRanges = _filterTimeRanges(
+                cros.get('activeTimeRanges', []), startDate, endDate)
+            lenATR = len(activeTimeRanges)
+            if lenATR:
+                print('  activeTimeRanges')
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    active_date = activeTimeRange["date"]
+                    active_time = activeTimeRange["activeTime"]
+                    duration = utils.formatMilliSeconds(active_time)
+                    minutes = active_time // 60000
+                    print(f'    date: {active_date}')
+                    print(f'      activeTime: {active_time}')
+                    print(f'      duration: {duration}')
+                    print(f'      minutes: {minutes}')
+            recentUsers = cros.get('recentUsers', [])
+            lenRU = len(recentUsers)
+            if lenRU:
+                print('  recentUsers')
+                num_ranges = min(lenRU, listLimit or lenRU)
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get("email")
+                    if not useremail:
+                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    print(f'    type: {recentUser["type"]}')
+                    print(f'      email: {useremail}')
+            deviceFiles = _filterCreateReportTime(
+                cros.get('deviceFiles', []), 'createTime', startDate, endDate)
+            lenDF = len(deviceFiles)
+            if lenDF:
+                num_ranges = min(lenDF, listLimit or lenDF)
+                print('  deviceFiles')
+                for deviceFile in deviceFiles[:num_ranges]:
+                    device_type = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    print(f'    {device_type}: {create_time}')
+            if downloadfile:
+                deviceFiles = cros.get('deviceFiles', [])
+                lenDF = len(deviceFiles)
+                if lenDF:
+                    if downloadfile == 'latest':
+                        deviceFile = deviceFiles[-1]
+                    else:
+                        for deviceFile in deviceFiles:
+                            if deviceFile['createTime'] == downloadfile:
+                                break
+                        else:
+                            print(f'ERROR: file {downloadfile} not ' \
+                                  f'available to download.')
+                            deviceFile = None
+                    if deviceFile:
+                        created = deviceFile["createTime"]
+                        downloadfile = f'cros-logs-{deviceId}-{created}.zip'
+                        downloadfilename = os.path.join(targetFolder,
+                                                        downloadfile)
+                        dl_url = deviceFile['downloadUrl']
+                        _, content = cd._http.request(dl_url)
+                        fileutils.write_file(downloadfilename, content,
+                                             mode='wb',
+                                             continue_on_error=True)
+                        print(f'Downloaded: {downloadfilename}')
+                elif downloadfile:
+                    print('ERROR: no files to download.')
+            cpuStatusReports = _filterCreateReportTime(
+                cros.get('cpuStatusReports', []),
+                'reportTime',
+                startDate,
+                endDate)
+            lenCSR = len(cpuStatusReports)
+            if lenCSR:
+                print('  cpuStatusReports')
+                num_ranges = min(lenCSR, listLimit or lenCSR)
+                for cpuStatusReport in cpuStatusReports[:num_ranges]:
+                    print(f'    reportTime: {cpuStatusReport["reportTime"]}')
+                    print('      cpuTemperatureInfo')
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        temp_label = tempInfo['label'].strip()
+                        temperature = tempInfo['temperature']
+                        print(f'        {temp_label}: {temperature}')
+                    pct_info = cpuStatusReport["cpuUtilizationPercentageInfo"]
+                    util = ",".join([str(x) for x in pct_info])
+                    print(f'      cpuUtilizationPercentageInfo: {util}')
+            diskVolumeReports = cros.get('diskVolumeReports', [])
+            lenDVR = len(diskVolumeReports)
+            if lenDVR:
+                print('  diskVolumeReports')
+                print('    volumeInfo')
+                num_ranges = min(lenDVR, listLimit or lenDVR)
+                for diskVolumeReport in diskVolumeReports[:num_ranges]:
+                    volumeInfo = diskVolumeReport['volumeInfo']
+                    for volume in volumeInfo:
+                        vid = volume['volumeId']
+                        vstorage_free = volume['storageFree']
+                        vstorage_total = volume['storageTotal']
+                        print(f'      volumeId: {vid}')
+                        print(f'        storageFree: {vstorage_free}')
+                        print(f'        storageTotal: {vstorage_total}')
+            systemRamFreeReports = _filterCreateReportTime(
+                cros.get('systemRamFreeReports', []),
+                'reportTime', startDate, endDate)
+            lenSRFR = len(systemRamFreeReports)
+            if lenSRFR:
+                print('  systemRamFreeReports')
+                num_ranges = min(lenSRFR, listLimit or lenSRFR)
+                for systemRamFreeReport in systemRamFreeReports[:num_ranges]:
+                    report_time = systemRamFreeReport["reportTime"]
+                    free_info = systemRamFreeReport["systemRamFreeInfo"]
+                    free_ram = ",".join(free_info)
+                    print(f'    reportTime: {report_time}')
+                    print(f'      systemRamFreeInfo: {free_ram}')
+
+
+def doPrintCrosActivity():
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    titles = ['deviceId', 'annotatedAssetId',
+              'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    csvRows = []
+    fieldsList = ['deviceId', 'annotatedAssetId',
+                  'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    startDate = endDate = None
+    selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False
+    listLimit = 0
+    delimiter = ','
+    orgUnitPath = None
+    queries = [None]
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectActiveTimeRanges = True
+            i += 1
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectDeviceFiles = True
+            i += 1
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectRecentUsers = True
+            i += 1
+        elif myarg == 'both':
+            selectActiveTimeRanges = selectRecentUsers = True
+            i += 1
+        elif myarg == 'all':
+            selectActiveTimeRanges = selectDeviceFiles = True
+            selectRecentUsers = True
+            i += 1
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print crosactivity")
+    if not selectActiveTimeRanges and \
+       not selectDeviceFiles and \
+       not selectRecentUsers:
+        selectActiveTimeRanges = selectRecentUsers = True
+    if selectRecentUsers:
+        fieldsList.append('recentUsers')
+        display.add_titles_to_csv_file(['recentUsers.email', ], titles)
+    if selectActiveTimeRanges:
+        fieldsList.append('activeTimeRanges')
+        titles_to_add = ['activeTimeRanges.date',
+                         'activeTimeRanges.duration',
+                         'activeTimeRanges.minutes']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    if selectDeviceFiles:
+        fieldsList.append('deviceFiles')
+        titles_to_add = ['deviceFiles.type', 'deviceFiles.createTime']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    fields = f'nextPageToken,chromeosdevices({",".join(fieldsList)})'
+    for query in queries:
+        __main__.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection='FULL',
+                                      fields=fields, orgUnitPath=orgUnitPath)
+        for cros in all_cros:
+            row = {}
+            skip_attribs = ['recentUsers', 'activeTimeRanges', 'deviceFiles']
+            for attrib in cros:
+                if attrib not in skip_attribs:
+                    row[attrib] = cros[attrib]
+            if selectActiveTimeRanges:
+                activeTimeRanges = _filterTimeRanges(
+                    cros.get('activeTimeRanges', []), startDate, endDate)
+                lenATR = len(activeTimeRanges)
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['activeTimeRanges.date'] = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    newrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    newrow['activeTimeRanges.minutes'] = \
+                        activeTimeRange['activeTime']//60000
+                    csvRows.append(new_row)
+            if selectRecentUsers:
+                recentUsers = cros.get('recentUsers', [])
+                lenRU = len(recentUsers)
+                num_ranges = min(lenRU, listLimit or lenRU)
+                recent_users = []
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get("email")
+                    if not useremail:
+                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    recent_users.append(useremail)
+                row['recentUsers.email'] = delimiter.join(recent_users)
+                csvRows.append(row)
+            if selectDeviceFiles:
+                deviceFiles = _filterCreateReportTime(
+                    cros.get('deviceFiles', []),
+                    'createTime', startDate, endDate)
+                lenDF = len(deviceFiles)
+                num_ranges = min(lenDF, listLimit or lenDF)
+                for deviceFile in deviceFiles[:num_ranges]:
+                    new_row = row.copy()
+                    new_row['deviceFiles.type'] = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    new_row['deviceFiles.createTime'] = create_time
+                    csvRows.append(new_row)
+    display.write_csv_file(csvRows, titles, 'CrOS Activity', todrive)
+
+
+def _checkTPMVulnerability(cros):
+    if 'tpmVersionInfo' in cros and \
+       'firmwareVersion' in cros['tpmVersionInfo']:
+        firmware_version = cros['tpmVersionInfo']['firmwareVersion']
+        if firmware_version in CROS_TPM_VULN_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'VULNERABLE'
+        elif firmware_version in CROS_TPM_FIXED_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'UPDATED'
+        else:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'NOT IMPACTED'
+
+
+def doPrintCrosDevices():
+    def _getSelectedLists(myarg):
+        if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectedLists['activeTimeRanges'] = True
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectedLists['recentUsers'] = True
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectedLists['deviceFiles'] = True
+        elif myarg in CROS_CPU_STATUS_REPORTS_ARGUMENTS:
+            selectedLists['cpuStatusReports'] = True
+        elif myarg in CROS_DISK_VOLUME_REPORTS_ARGUMENTS:
+            selectedLists['diskVolumeReports'] = True
+        elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS:
+            selectedLists['systemRamFreeReports'] = True
+
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    display.add_field_to_csv_file(
+        'deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles)
+    projection = orderBy = sortOrder = orgUnitPath = None
+    queries = [None]
+    noLists = sortHeaders = False
+    selectedLists = {}
+    startDate = endDate = None
+    listLimit = 0
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'nolists':
+            noLists = True
+            selectedLists = {}
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i+1].lower().replace('_', '')
+            validOrderBy = ['location', 'user', 'lastsync',
+                            'notes', 'serialnumber', 'status', 'supportenddate']
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit(
+                    "orderby", ", ".join(validOrderBy), orderBy)
+            if orderBy == 'location':
+                orderBy = 'annotatedLocation'
+            elif orderBy == 'user':
+                orderBy = 'annotatedUser'
+            elif orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'serialnumber':
+                orderBy = 'serialNumber'
+            elif orderBy == 'supportenddate':
+                orderBy = 'supportEndDate'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            sortHeaders = True
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            sortHeaders = True
+            fieldsList = []
+            i += 1
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in CROS_LISTS_ARGUMENTS:
+            _getSelectedLists(myarg)
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_fields_list(
+                myarg, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i+1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_LISTS_ARGUMENTS:
+                    _getSelectedLists(field)
+                elif field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    display.add_field_to_fields_list(
+                        field, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+                else:
+                    controlflow.invalid_argument_exit(
+                        field, "gam print cros fields")
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam print cros")
+    if selectedLists:
+        noLists = False
+        projection = 'FULL'
+        for selectList in selectedLists:
+            display.add_field_to_fields_list(
+                selectList, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = f'nextPageToken,chromeosdevices({",".join(set(fieldsList))})'.replace(
+            '.', '/')
+    else:
+        fields = None
+    for query in queries:
+        __main__.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                      'chromeosdevices',
+                                      page_message=page_message, query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection=projection,
+                                      orgUnitPath=orgUnitPath,
+                                      orderBy=orderBy, sortOrder=sortOrder,
+                                      fields=fields)
+        for cros in all_cros:
+            _checkTPMVulnerability(cros)
+        if not noLists and not selectedLists:
+            for cros in all_cros:
+                if 'notes' in cros:
+                    cros['notes'] = cros['notes'].replace('\n', '\\n')
+                if 'autoUpdateExpiration' in cros:
+                    cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                        cros['autoUpdateExpiration'])
+                for cpuStatusReport in cros.get('cpuStatusReports', []):
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        tempInfo['label'] = tempInfo['label'].strip()
+                display.add_row_titles_to_csv_file(utils.flatten_json(
+                    cros, listLimit=listLimit), csvRows, titles)
+            continue
+        for cros in all_cros:
+            if 'notes' in cros:
+                cros['notes'] = cros['notes'].replace('\n', '\\n')
+            if 'autoUpdateExpiration' in cros:
+                cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                    cros['autoUpdateExpiration'])
+            row = {}
+            for attrib in cros:
+                if attrib not in set(['kind', 'etag', 'tpmVersionInfo',
+                                      'recentUsers', 'activeTimeRanges',
+                                      'deviceFiles', 'cpuStatusReports',
+                                      'diskVolumeReports',
+                                      'systemRamFreeReports']):
+                    row[attrib] = cros[attrib]
+            if selectedLists.get('activeTimeRanges'):
+                timergs = cros.get('activeTimeRanges', [])
+            else:
+                timergs = []
+            activeTimeRanges = _filterTimeRanges(timergs, startDate, endDate)
+            if selectedLists.get('recentUsers'):
+                recentUsers = cros.get('recentUsers', [])
+            else:
+                recentUsers = []
+            if selectedLists.get('deviceFiles'):
+                device_files = cros.get('deviceFiles', [])
+            else:
+                device_files = []
+            deviceFiles = _filterCreateReportTime(device_files, 'createTime',
+                                                  startDate, endDate)
+            if selectedLists.get('cpuStatusReports'):
+                cpu_reports = cros.get('cpuStatusReports', [])
+            else:
+                cpu_reports = []
+            cpuStatusReports = _filterCreateReportTime(cpu_reports,
+                                                       'reportTime',
+                                                       startDate, endDate)
+            if selectedLists.get('diskVolumeReports'):
+                diskVolumeReports = cros.get('diskVolumeReports', [])
+            else:
+                diskVolumeReports = []
+            if selectedLists.get('systemRamFreeReports'):
+                ram_reports = cros.get('systemRamFreeReports', [])
+            else:
+                ram_reports = []
+            systemRamFreeReports = _filterCreateReportTime(ram_reports,
+                                                           'reportTime',
+                                                           startDate,
+                                                           endDate)
+            if noLists or (not activeTimeRanges and \
+                           not recentUsers and \
+                           not deviceFiles and \
+                           not cpuStatusReports and \
+                           not diskVolumeReports and \
+                           not systemRamFreeReports):
+                display.add_row_titles_to_csv_file(row, csvRows, titles)
+                continue
+            lenATR = len(activeTimeRanges)
+            lenRU = len(recentUsers)
+            lenDF = len(deviceFiles)
+            lenCSR = len(cpuStatusReports)
+            lenDVR = len(diskVolumeReports)
+            lenSRFR = len(systemRamFreeReports)
+            max_len = max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR)
+            for i in range(min(max_len, listLimit or max_len)):
+                nrow = row.copy()
+                if i < lenATR:
+                    nrow['activeTimeRanges.date'] = \
+                        activeTimeRanges[i]['date']
+                    nrow['activeTimeRanges.activeTime'] = \
+                        str(activeTimeRanges[i]['activeTime'])
+                    active_time = activeTimeRanges[i]['activeTime']
+                    nrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    nrow['activeTimeRanges.minutes'] = active_time // 60000
+                if i < lenRU:
+                    nrow['recentUsers.type'] = recentUsers[i]['type']
+                    nrow['recentUsers.email'] = recentUsers[i].get('email')
+                    if not nrow['recentUsers.email']:
+                        if nrow['recentUsers.type'] == 'USER_TYPE_UNMANAGED':
+                            nrow['recentUsers.email'] = 'UnmanagedUser'
+                        else:
+                            nrow['recentUsers.email'] = 'Unknown'
+                if i < lenDF:
+                    nrow['deviceFiles.type'] = deviceFiles[i]['type']
+                    nrow['deviceFiles.createTime'] = \
+                        deviceFiles[i]['createTime']
+                if i < lenCSR:
+                    nrow['cpuStatusReports.reportTime'] = \
+                        cpuStatusReports[i]['reportTime']
+                    tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo',
+                                                        [])
+                    for tempInfo in tempInfos:
+                        temperature = tempInfo['temperature']
+                        label = tempInfo["label"].strip()
+                        base = 'cpuStatusReports.cpuTemperatureInfo.'
+                        nrow[f'{base}{label}'] = tempInfo['temperature']
+                    cpu_field = 'cpuUtilizationPercentageInfo'
+                    cpu_reports = cpuStatusReports[i][cpu_field]
+                    cpu_pcts = [str(x) for x in cpu_reports]
+                    nrow[f'cpuStatusReports.{cpu_field}'] = ','.join(cpu_pcts)
+                if i < lenDVR:
+                    volumeInfo = diskVolumeReports[i]['volumeInfo']
+                    j = 0
+                    vfield = 'diskVolumeReports.volumeInfo.'
+                    for volume in volumeInfo:
+                        nrow[f'{vfield}{j}.volumeId'] = \
+                            volume['volumeId']
+                        nrow[f'{vfield}{j}.storageFree'] = \
+                            volume['storageFree']
+                        nrow[f'{vfield}{j}.storageTotal'] = \
+                            volume['storageTotal']
+                        j += 1
+                if i < lenSRFR:
+                    nrow['systemRamFreeReports.reportTime'] = \
+                        systemRamFreeReports[i]['reportTime']
+                    ram_reports = systemRamFreeReports[i]['systemRamFreeInfo']
+                    ram_info = [str(x) for x in ram_reports]
+                    nrow['systenRamFreeReports.systemRamFreeInfo'] = \
+                        ','.join(ram_info)
+                display.add_row_titles_to_csv_file(nrow, csvRows, titles)
+    if sortHeaders:
+        display.sort_csv_titles(['deviceId', ], titles)
+    display.write_csv_file(csvRows, titles, 'CrOS', todrive)
+
+
+def getCrOSDeviceEntity(i, cd):
+    myarg = sys.argv[i].lower()
+    if myarg == 'cros_sn':
+        return i+2, __main__.getUsersToModify('cros_sn', sys.argv[i+1])
+    if myarg == 'query':
+        return i+2, __main__.getUsersToModify('crosquery', sys.argv[i+1])
+    if myarg[:6] == 'query:':
+        query = sys.argv[i][6:]
+        if query[:12].lower() == 'orgunitpath:':
+            kwargs = {'orgUnitPath': query[12:]}
+        else:
+            kwargs = {'query': query}
+        fields = 'nextPageToken,chromeosdevices(deviceId)'
+        devices = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                     'chromeosdevices',
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     fields=fields, **kwargs)
+        return i+1, [device['deviceId'] for device in devices]
+    return i+1, sys.argv[i].replace(',', ' ').split()
+
+
+def _getFilterDate(dateStr):
+    return datetime.datetime.strptime(dateStr, YYYYMMDD_FORMAT)
+
+
+def _filterTimeRanges(activeTimeRanges, startDate, endDate):
+    if startDate is None and endDate is None:
+        return activeTimeRanges
+    filteredTimeRanges = []
+    for timeRange in activeTimeRanges:
+        activityDate = datetime.datetime.strptime(
+            timeRange['date'], YYYYMMDD_FORMAT)
+        if ((startDate is None) or \
+            (activityDate >= startDate)) and \
+           ((endDate is None) or \
+            (activityDate <= endDate)):
+            filteredTimeRanges.append(timeRange)
+    return filteredTimeRanges
+
+
+def _filterCreateReportTime(items, timeField, startTime, endTime):
+    if startTime is None and endTime is None:
+        return items
+    filteredItems = []
+    time_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    for item in items:
+        timeValue = datetime.datetime.strptime(item[timeField], time_format)
+        if ((startTime is None) or \
+            (timeValue >= startTime)) and \
+           ((endTime is None) or \
+            (timeValue <= endTime)):
+            filteredItems.append(item)
+    return filteredItems

src/gapi/directory/customer.py

@@ -0,0 +1,113 @@
+import datetime
+
+from var import *
+import controlflow
+import gapi
+import gapi.directory
+import gapi.reports
+
+
+def doGetCustomerInfo():
+    cd = gapi.directory.buildGAPIObject()
+    customer_info = gapi.call(cd.customers(), 'get',
+                              customerKey=GC_Values[GC_CUSTOMER_ID])
+    print(f'Customer ID: {customer_info["id"]}')
+    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    result = gapi.call(cd.domains(), 'get', customer=customer_info['id'],
+                       domainName=customer_info['customerDomain'],
+                       fields='verified')
+    print(f'Primary Domain Verified: {result["verified"]}')
+    # If customer has changed primary domain customerCreationTime is date
+    # of current primary being added, not customer create date.
+    # We should also get all domains and use oldest date
+    customer_creation = customer_info['customerCreationTime']
+    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    oldest = datetime.datetime.strptime(customer_creation, date_format)
+    domains = gapi.get_items(cd.domains(), 'list', 'domains',
+                             customer=GC_Values[GC_CUSTOMER_ID],
+                             fields='domains(creationTime)')
+    for domain in domains:
+        creation_timestamp = int(domain['creationTime'])/1000
+        domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
+        if domain_creation < oldest:
+            oldest = domain_creation
+    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
+    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    print(f'Default Language: {customer_language}')
+    if 'postalAddress' in customer_info:
+        print('Address:')
+        for field in ADDRESS_FIELDS_PRINT_ORDER:
+            if field in customer_info['postalAddress']:
+                print(f' {field}: {customer_info["postalAddress"][field]}')
+    if 'phoneNumber' in customer_info:
+        print(f'Phone: {customer_info["phoneNumber"]}')
+    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    user_counts_map = {
+        'accounts:num_users': 'Total Users',
+        'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',
+        'accounts:gsuite_basic_used_licenses': 'G Suite Basic Users',
+        'accounts:gsuite_enterprise_total_licenses': 'G Suite Enterprise ' \
+        'Licenses',
+        'accounts:gsuite_enterprise_used_licenses': 'G Suite Enterprise ' \
+        'Users',
+        'accounts:gsuite_unlimited_total_licenses': 'G Suite Business ' \
+        'Licenses',
+        'accounts:gsuite_unlimited_used_licenses': 'G Suite Business Users'
+    }
+    parameters = ','.join(list(user_counts_map))
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    rep = gapi.reports.buildGAPIObject()
+    usage = None
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    while True:
+        try:
+            usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                                       'usageReports',
+                                       throw_reasons=throw_reasons,
+                                       customerId=customerId, date=tryDate,
+                                       parameters=parameters)
+            break
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = gapi.reports._adjust_date(str(e))
+    if not usage:
+        print('No user count data available.')
+        return
+    print(f'User counts as of {tryDate}:')
+    for item in usage[0]['parameters']:
+        api_name = user_counts_map.get(item['name'])
+        api_value = int(item.get('intValue', 0))
+        if api_name and api_value:
+            print(f'  {api_name}: {api_value:,}')
+
+
+def doUpdateCustomer():
+    cd = gapi.directory.buildGAPIObject()
+    body = {}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ADDRESS_FIELDS_ARGUMENT_MAP:
+            body.setdefault('postalAddress', {})
+            arg = ADDRESS_FIELDS_ARGUMENT_MAP[myarg]
+            body['postalAddress'][arg] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['adminsecondaryemail', 'alternateemail']:
+            body['alternateEmail'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['phone', 'phonenumber']:
+            body['phoneNumber'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'language':
+            body['language'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam update customer")
+    if not body:
+        controlflow.system_error_exit(2, 'no arguments specified for "gam '
+                                         'update customer"')
+    gapi.call(cd.customers(), 'patch', customerKey=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print('Updated customer')

src/gapi/directory/resource.py

@@ -0,0 +1,487 @@
+import sys
+import uuid
+
+import __main__
+from var import *
+import controlflow
+import display
+import gapi.directory
+import utils
+
+
+def printBuildings():
+    to_drive = False
+    cd = gapi.directory.buildGAPIObject()
+    titles = []
+    csvRows = []
+    fieldsList = ['buildingId']
+    # buildings.list() currently doesn't support paging
+    # but should soon, attempt to use it now so we
+    # won't break when it's turned on.
+    fields = 'nextPageToken,buildings(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Building']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        # Allows shorter arguments like "name" instead of "buildingname"
+        elif 'building'+myarg in possible_fields:
+            fieldsList.append(possible_fields['building'+myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print buildings")
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    for building in buildings:
+        building.pop('etags', None)
+        building.pop('etag', None)
+        building.pop('kind', None)
+        if 'buildingId' in building:
+            building['buildingId'] = f'id:{building["buildingId"]}'
+        if 'floorNames' in building:
+            building['floorNames'] = ','.join(building['floorNames'])
+        building = utils.flatten_json(building)
+        for item in building:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(building)
+    display.sort_csv_titles('buildingId', titles)
+    display.write_csv_file(csvRows, titles, 'Buildings', to_drive)
+
+
+def printResourceCalendars():
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    query = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg == 'allfields':
+            fieldsList = []
+            fieldsTitles = {}
+            titles = []
+            for field in RESCAL_ALLFIELDS:
+                display.add_field_to_csv_file(field,
+                                              RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                              fieldsList, fieldsTitles,
+                                              titles)
+            i += 1
+        elif myarg in RESCAL_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_csv_file(myarg,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print resources")
+    if not fieldsList:
+        for field in RESCAL_DFLTFIELDS:
+            display.add_field_to_csv_file(field,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+    fields = f'nextPageToken,items({",".join(set(fieldsList))})'
+    if 'buildingId' in fieldsList:
+        display.add_field_to_csv_file('buildingName', {'buildingName': [
+            'buildingName', ]}, fieldsList, fieldsTitles, titles)
+    __main__.printGettingAllItems('Resource Calendars', None)
+    page_message = gapi.got_total_items_first_last_msg('Resource Calendars')
+    resources = gapi.get_all_pages(cd.resources().calendars(), 'list',
+                                   'items', page_message=page_message,
+                                   message_attribute='resourceId',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   query=query, fields=fields)
+    for resource in resources:
+        if 'featureInstances' in resource:
+            features = [a_feature['feature']['name'] for \
+                a_feature in resource['featureInstances']]
+            resource['featureInstances'] = ','.join(features)
+        if 'buildingId' in resource:
+            resource['buildingName'] = getBuildingNameById(
+                cd, resource['buildingId'])
+            resource['buildingId'] = f'id:{resource["buildingId"]}'
+        resUnit = {}
+        for field in fieldsList:
+            resUnit[fieldsTitles[field]] = resource.get(field, '')
+        csvRows.append(resUnit)
+    display.sort_csv_titles(
+        ['resourceId', 'resourceName', 'resourceEmail'], titles)
+    display.write_csv_file(csvRows, titles, 'Resources', todrive)
+
+
+RESCAL_DFLTFIELDS = ['id', 'name', 'email',]
+RESCAL_ALLFIELDS = ['id', 'name', 'email', 'description', 'type',
+                    'buildingid', 'category', 'capacity', 'features', 'floor',
+                    'floorsection', 'generatedresourcename',
+                    'uservisibledescription',]
+
+RESCAL_ARGUMENT_TO_PROPERTY_MAP = {
+    'description': ['resourceDescription'],
+    'building': ['buildingId', ],
+    'buildingid': ['buildingId', ],
+    'capacity': ['capacity', ],
+    'category': ['resourceCategory', ],
+    'email': ['resourceEmail'],
+    'feature': ['featureInstances', ],
+    'features': ['featureInstances', ],
+    'floor': ['floorName', ],
+    'floorname': ['floorName', ],
+    'floorsection': ['floorSection', ],
+    'generatedresourcename': ['generatedResourceName', ],
+    'id': ['resourceId'],
+    'name': ['resourceName'],
+    'type': ['resourceType'],
+    'userdescription': ['userVisibleDescription', ],
+    'uservisibledescription': ['userVisibleDescription', ],
+}
+
+
+def printFeatures():
+    to_drive = False
+    cd = gapi.directory.buildGAPIObject()
+    titles = []
+    csvRows = []
+    fieldsList = ['name']
+    fields = 'nextPageToken,features(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Feature']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        elif 'feature'+myarg in possible_fields:
+            fieldsList.append(possible_fields['feature'+myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print features")
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    features = gapi.get_all_pages(cd.resources().features(), 'list',
+                                  'features',
+                                  customer=GC_Values[GC_CUSTOMER_ID],
+                                  fields=fields)
+    for feature in features:
+        feature.pop('etags', None)
+        feature.pop('etag', None)
+        feature.pop('kind', None)
+        feature = utils.flatten_json(feature)
+        for item in feature:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(feature)
+    display.sort_csv_titles('name', titles)
+    display.write_csv_file(csvRows, titles, 'Features', to_drive)
+
+
+def _getBuildingAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'id':
+            body['buildingId'] = args[i+1]
+            i += 2
+        elif myarg == 'name':
+            body['buildingName'] = args[i+1]
+            i += 2
+        elif myarg in ['lat', 'latitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['latitude'] = args[i+1]
+            i += 2
+        elif myarg in ['long', 'lng', 'longitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['longitude'] = args[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = args[i+1]
+            i += 2
+        elif myarg == 'floors':
+            body['floorNames'] = args[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, "gam create|update building")
+    return body
+
+
+def createBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    body = {'floorNames': ['1'],
+            'buildingId': str(uuid.uuid4()),
+            'buildingName': sys.argv[3]}
+    body = _getBuildingAttributes(sys.argv[4:], body)
+    print(f'Creating building {body["buildingId"]}...')
+    gapi.call(cd.resources().buildings(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def _makeBuildingIdNameMap(cd):
+    fields = 'nextPageToken,buildings(buildingId,buildingName)'
+    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {}
+    GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {}
+    for building in buildings:
+        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][building['buildingId']
+                                               ] = building['buildingName']
+        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building['buildingName']
+                                               ] = building['buildingId']
+
+
+def getBuildingByNameOrId(cd, which_building, minLen=1):
+    if not which_building or \
+       (minLen == 0 and which_building in ['id:', 'uid:']):
+        if minLen == 0:
+            return ''
+        controlflow.system_error_exit(3, 'Building id/name is empty')
+    cg = UID_PATTERN.match(which_building)
+    if cg:
+        return cg.group(1)
+    if GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] is None:
+        _makeBuildingIdNameMap(cd)
+    # Exact name match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID]:
+        return GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][which_building]
+    # No exact name match, check for case insensitive name matches
+    which_building_lower = which_building.lower()
+    ci_matches = []
+    for buildingName, buildingId in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID].items():
+        if buildingName.lower() == which_building_lower:
+            ci_matches.append(
+                {'buildingName': buildingName, 'buildingId': buildingId})
+    # One match, return ID
+    if len(ci_matches) == 1:
+        return ci_matches[0]['buildingId']
+    # No or multiple name matches, try ID
+    # Exact ID match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        return which_building
+    # No exact ID match, check for case insensitive id match
+    for buildingId in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        # Match, return ID
+        if buildingId.lower() == which_building_lower:
+            return buildingId
+    # Multiple name  matches
+    if len(ci_matches) > 1:
+        message = 'Multiple buildings with same name:\n'
+        for building in ci_matches:
+            message += f'  Name:{building["buildingName"]}  ' \
+                       f'id:{building["buildingId"]}\n'
+        message += '\nPlease specify building name by exact case or by id.'
+        controlflow.system_error_exit(3, message)
+    # No matches
+    else:
+        controlflow.system_error_exit(3, f'No such building {which_building}')
+
+
+def getBuildingNameById(cd, buildingId):
+    if GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] is None:
+        _makeBuildingIdNameMap(cd)
+    return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, 'UNKNOWN')
+
+
+def updateBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    body = _getBuildingAttributes(sys.argv[4:])
+    print(f'Updating building {buildingId}...')
+    gapi.call(cd.resources().buildings(), 'patch',
+              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId,
+              body=body)
+
+
+def getBuildingInfo():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    building = gapi.call(cd.resources().buildings(), 'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         buildingId=buildingId)
+    if 'buildingId' in building:
+        building['buildingId'] = f'id:{building["buildingId"]}'
+    if 'floorNames' in building:
+        building['floorNames'] = ','.join(building['floorNames'])
+    if 'buildingName' in building:
+        sys.stdout.write(building.pop('buildingName'))
+    display.print_json(building)
+
+
+def deleteBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    print(f'Deleting building {buildingId}...')
+    gapi.call(cd.resources().buildings(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId)
+
+
+def _getFeatureAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = args[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, "gam create|update feature")
+    return body
+
+
+def createFeature():
+    cd = gapi.directory.buildGAPIObject()
+    body = _getFeatureAttributes(sys.argv[3:])
+    print(f'Creating feature {body["name"]}...')
+    gapi.call(cd.resources().features(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def updateFeature():
+    # update does not work for name and name is only field to be updated
+    # if additional writable fields are added to feature in the future
+    # we'll add support for update as well as rename
+    cd = gapi.directory.buildGAPIObject()
+    oldName = sys.argv[3]
+    body = {'newName': sys.argv[5:]}
+    print(f'Updating feature {oldName}...')
+    gapi.call(cd.resources().features(), 'rename',
+              customer=GC_Values[GC_CUSTOMER_ID], oldName=oldName,
+              body=body)
+
+
+def deleteFeature():
+    cd = gapi.directory.buildGAPIObject()
+    featureKey = sys.argv[3]
+    print(f'Deleting feature {featureKey}...')
+    gapi.call(cd.resources().features(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], featureKey=featureKey)
+
+
+def _getResourceCalendarAttributes(cd, args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['resourceName'] = args[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['resourceDescription'] = args[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'type':
+            body['resourceType'] = args[i+1]
+            i += 2
+        elif myarg in ['building', 'buildingid']:
+            body['buildingId'] = getBuildingByNameOrId(
+                cd, args[i+1], minLen=0)
+            i += 2
+        elif myarg in ['capacity']:
+            body['capacity'] = __main__.getInteger(args[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg in ['feature', 'features']:
+            features = args[i+1].split(',')
+            body['featureInstances'] = []
+            for feature in features:
+                instance = {'feature': {'name': feature}}
+                body['featureInstances'].append(instance)
+            i += 2
+        elif myarg in ['floor', 'floorname']:
+            body['floorName'] = args[i+1]
+            i += 2
+        elif myarg in ['floorsection']:
+            body['floorSection'] = args[i+1]
+            i += 2
+        elif myarg in ['category']:
+            body['resourceCategory'] = args[i+1].upper()
+            if body['resourceCategory'] == 'ROOM':
+                body['resourceCategory'] = 'CONFERENCE_ROOM'
+            i += 2
+        elif myarg in ['uservisibledescription', 'userdescription']:
+            body['userVisibleDescription'] = args[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                args[i], "gam create|update resource")
+    return body
+
+
+def createResourceCalendar():
+    cd = gapi.directory.buildGAPIObject()
+    body = {'resourceId': sys.argv[3],
+            'resourceName': sys.argv[4]}
+    body = _getResourceCalendarAttributes(cd, sys.argv[5:], body)
+    print(f'Creating resource {body["resourceId"]}...')
+    gapi.call(cd.resources().calendars(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def updateResourceCalendar():
+    cd = gapi.directory.buildGAPIObject()
+    resId = sys.argv[3]
+    body = _getResourceCalendarAttributes(cd, sys.argv[4:])
+    # Use patch since it seems to work better.
+    # update requires name to be set.
+    gapi.call(cd.resources().calendars(), 'patch',
+              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId,
+              body=body, fields='')
+    print(f'updated resource {resId}')
+
+
+def getResourceCalendarInfo():
+    cd = gapi.directory.buildGAPIObject()
+    resId = sys.argv[3]
+    resource = gapi.call(cd.resources().calendars(), 'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         calendarResourceId=resId)
+    if 'featureInstances' in resource:
+        features = []
+        for a_feature in resource.pop('featureInstances'):
+            features.append(a_feature['feature']['name'])
+        resource['features'] = ', '.join(features)
+    if 'buildingId' in resource:
+        resource['buildingName'] = getBuildingNameById(
+            cd, resource['buildingId'])
+        resource['buildingId'] = f'id:{resource["buildingId"]}'
+    display.print_json(resource)
+
+
+def deleteResourceCalendar():
+    resId = sys.argv[3]
+    cd = gapi.directory.buildGAPIObject()
+    print(f'Deleting resource calendar {resId}')
+    gapi.call(cd.resources().calendars(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId)

src/gapi/errors.py

@@ -0,0 +1,362 @@
+"""GAPI and OAuth Token related errors methods."""
+
+from enum import Enum
+import json
+
+import controlflow
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import UTF8
+
+
+class GapiAbortedError(Exception):
+  pass
+
+
+class GapiAuthErrorError(Exception):
+  pass
+
+
+class GapiBadGatewayError(Exception):
+  pass
+
+
+class GapiBadRequestError(Exception):
+  pass
+
+
+class GapiConditionNotMetError(Exception):
+  pass
+
+
+class GapiCyclicMembershipsNotAllowedError(Exception):
+  pass
+
+
+class GapiDomainCannotUseApisError(Exception):
+  pass
+
+
+class GapiDomainNotFoundError(Exception):
+  pass
+
+
+class GapiDuplicateError(Exception):
+  pass
+
+
+class GapiFailedPreconditionError(Exception):
+  pass
+
+
+class GapiForbiddenError(Exception):
+  pass
+
+
+class GapiGatewayTimeoutError(Exception):
+  pass
+
+
+class GapiGroupNotFoundError(Exception):
+  pass
+
+
+class GapiInvalidError(Exception):
+  pass
+
+
+class GapiInvalidArgumentError(Exception):
+  pass
+
+
+class GapiInvalidMemberError(Exception):
+  pass
+
+
+class GapiMemberNotFoundError(Exception):
+  pass
+
+
+class GapiNotFoundError(Exception):
+  pass
+
+
+class GapiNotImplementedError(Exception):
+  pass
+
+
+class GapiPermissionDeniedError(Exception):
+  pass
+
+
+class GapiResourceNotFoundError(Exception):
+  pass
+
+
+class GapiServiceNotAvailableError(Exception):
+  pass
+
+
+class GapiUserNotFoundError(Exception):
+  pass
+
+
+# GAPI Error Reasons
+class ErrorReason(Enum):
+  """The reason why a non-200 HTTP response was returned from a GAPI."""
+  ABORTED = 'aborted'
+  AUTH_ERROR = 'authError'
+  BACKEND_ERROR = 'backendError'
+  BAD_GATEWAY = 'badGateway'
+  BAD_REQUEST = 'badRequest'
+  CONDITION_NOT_MET = 'conditionNotMet'
+  CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
+  DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
+  DOMAIN_NOT_FOUND = 'domainNotFound'
+  DUPLICATE = 'duplicate'
+  FAILED_PRECONDITION = 'failedPrecondition'
+  FORBIDDEN = 'forbidden'
+  FOUR_O_THREE = '403'
+  GATEWAY_TIMEOUT = 'gatewayTimeout'
+  GROUP_NOT_FOUND = 'groupNotFound'
+  INTERNAL_ERROR = 'internalError'
+  INVALID = 'invalid'
+  INVALID_ARGUMENT = 'invalidArgument'
+  INVALID_MEMBER = 'invalidMember'
+  MEMBER_NOT_FOUND = 'memberNotFound'
+  NOT_FOUND = 'notFound'
+  NOT_IMPLEMENTED = 'notImplemented'
+  PERMISSION_DENIED = 'permissionDenied'
+  QUOTA_EXCEEDED = 'quotaExceeded'
+  RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
+  RESOURCE_NOT_FOUND = 'resourceNotFound'
+  SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+  SERVICE_LIMIT = 'serviceLimit'
+  SYSTEM_ERROR = 'systemError'
+  USER_NOT_FOUND = 'userNotFound'
+  USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+  FOUR_TWO_NINE = '429'
+  DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
+
+  def __str__(self):
+    return str(self.value)
+
+
+# Common sets of GAPI error reasons
+DEFAULT_RETRY_REASONS = [
+  ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
+  ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
+  ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT,
+  ErrorReason.INTERNAL_ERROR, ErrorReason.FOUR_TWO_NINE,
+  ]
+GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
+GROUP_GET_THROW_REASONS = [
+  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
+  ErrorReason.BAD_REQUEST
+  ]
+GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
+MEMBERS_THROW_REASONS = [
+  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
+  ErrorReason.FORBIDDEN
+  ]
+MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
+
+# A map of GAPI error reasons to the corresponding GAM Python Exception
+ERROR_REASON_TO_EXCEPTION = {
+  ErrorReason.ABORTED:
+    GapiAbortedError,
+  ErrorReason.AUTH_ERROR:
+    GapiAuthErrorError,
+  ErrorReason.BAD_GATEWAY:
+    GapiBadGatewayError,
+  ErrorReason.BAD_REQUEST:
+    GapiBadRequestError,
+  ErrorReason.CONDITION_NOT_MET:
+    GapiConditionNotMetError,
+  ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
+    GapiCyclicMembershipsNotAllowedError,
+  ErrorReason.DOMAIN_CANNOT_USE_APIS:
+    GapiDomainCannotUseApisError,
+  ErrorReason.DOMAIN_NOT_FOUND:
+    GapiDomainNotFoundError,
+  ErrorReason.DUPLICATE:
+    GapiDuplicateError,
+  ErrorReason.FAILED_PRECONDITION:
+    GapiFailedPreconditionError,
+  ErrorReason.FORBIDDEN:
+    GapiForbiddenError,
+  ErrorReason.GATEWAY_TIMEOUT:
+    GapiGatewayTimeoutError,
+  ErrorReason.GROUP_NOT_FOUND:
+    GapiGroupNotFoundError,
+  ErrorReason.INVALID:
+    GapiInvalidError,
+  ErrorReason.INVALID_ARGUMENT:
+    GapiInvalidArgumentError,
+  ErrorReason.INVALID_MEMBER:
+    GapiInvalidMemberError,
+  ErrorReason.MEMBER_NOT_FOUND:
+    GapiMemberNotFoundError,
+  ErrorReason.NOT_FOUND:
+    GapiNotFoundError,
+  ErrorReason.NOT_IMPLEMENTED:
+    GapiNotImplementedError,
+  ErrorReason.PERMISSION_DENIED:
+    GapiPermissionDeniedError,
+  ErrorReason.RESOURCE_NOT_FOUND:
+    GapiResourceNotFoundError,
+  ErrorReason.SERVICE_NOT_AVAILABLE:
+    GapiServiceNotAvailableError,
+  ErrorReason.USER_NOT_FOUND:
+    GapiUserNotFoundError,
+  }
+
+# OAuth Token Errors
+OAUTH2_TOKEN_ERRORS = [
+  'access_denied',
+  'access_denied: Requested client not authorized',
+  'internal_failure: Backend Error',
+  'internal_failure: None',
+  'invalid_grant',
+  'invalid_grant: Bad Request',
+  'invalid_grant: Invalid email or User ID',
+  'invalid_grant: Not a valid email',
+  'invalid_grant: Invalid JWT: No valid verifier found for issuer',
+  'invalid_request: Invalid impersonation prn email address',
+  'unauthorized_client: Client is unauthorized to retrieve access tokens '
+  'using this method',
+  'unauthorized_client: Client is unauthorized to retrieve access tokens '
+  'using this method, or client not authorized for any of the scopes '
+  'requested',
+  'unauthorized_client: Unauthorized client or scope in request',
+  ]
+
+
+def _create_http_error_dict(status_code, reason, message):
+  """Creates a basic error dict similar to most Google API Errors.
+
+  Args:
+    status_code: Int, the error's HTTP response status code.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    dict
+  """
+  return {
+    'error': {
+      'code': status_code,
+      'errors': [{
+        'reason': str(reason),
+        'message': message,
+        }]
+      }
+  }
+
+
+def get_gapi_error_detail(e,
+                          soft_errors=False,
+                          silent_errors=False,
+                          retry_on_http_error=False):
+  """Extracts error detail from a non-200 GAPI Response.
+
+  Args:
+    e: googleapiclient.HttpError, The HTTP Error received.
+    soft_errors: Boolean, If true, causes error messages to be surpressed,
+      rather than sending them to stderr.
+    silent_errors: Boolean, If true, suppresses and ignores any errors from
+      being displayed
+    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
+      code, indicating that the request can be retried. TODO: Remove this param,
+        as it seems to be outside the scope of this method.
+
+  Returns:
+    A tuple containing the HTTP Response code, GAPI error reason, and error
+        message.
+  """
+  try:
+    error = json.loads(e.content.decode(UTF8))
+  except ValueError:
+    error_content = e.content.decode(UTF8) if isinstance(e.content,
+                                                         bytes) else e.content
+    if (e.resp['status'] == '503') and (
+        error_content == 'Quota exceeded for the current request'):
+      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
+    if (e.resp['status'] == '403') and (
+        error_content.startswith('Request rate higher than configured')):
+      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
+    if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
+      return (e.resp['status'], ErrorReason.BAD_GATEWAY.value, error_content)
+    if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
+      return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value, error_content)
+    if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
+      error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
+                                      'Domain not found')
+    elif (e.resp['status'] == '400') and (
+        'InvalidSsoSigningKey' in error_content):
+      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                      'InvalidSsoSigningKey')
+    elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
+      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                      'UnknownError')
+    elif retry_on_http_error:
+      return (-1, None, None)
+    elif soft_errors:
+      if not silent_errors:
+        display.print_error(error_content)
+      return (0, None, None)
+    else:
+      controlflow.system_error_exit(5, error_content)
+    # END: ValueError catch
+
+  if 'error' in error:
+    http_status = error['error']['code']
+    try:
+      message = error['error']['errors'][0]['message']
+    except KeyError:
+      message = error['error']['message']
+  else:
+    if 'error_description' in error:
+      if error['error_description'] == 'Invalid Value':
+        message = error['error_description']
+        http_status = 400
+        error = _create_http_error_dict(400, ErrorReason.INVALID.value, message)
+      else:
+        controlflow.system_error_exit(4, str(error))
+    else:
+      controlflow.system_error_exit(4, str(error))
+
+  # Extract the error reason
+  try:
+    reason = error['error']['errors'][0]['reason']
+    if reason == 'notFound':
+      if 'userKey' in message:
+        reason = ErrorReason.USER_NOT_FOUND.value
+      elif 'groupKey' in message:
+        reason = ErrorReason.GROUP_NOT_FOUND.value
+      elif 'memberKey' in message:
+        reason = ErrorReason.MEMBER_NOT_FOUND.value
+      elif 'Domain not found' in message:
+        reason = ErrorReason.DOMAIN_NOT_FOUND.value
+      elif 'Resource Not Found' in message:
+        reason = ErrorReason.RESOURCE_NOT_FOUND.value
+    elif reason == 'invalid':
+      if 'userId' in message:
+        reason = ErrorReason.USER_NOT_FOUND.value
+      elif 'memberKey' in message:
+        reason = ErrorReason.INVALID_MEMBER.value
+    elif reason == 'failedPrecondition':
+      if 'Bad Request' in message:
+        reason = ErrorReason.BAD_REQUEST.value
+      elif 'Mail service not enabled' in message:
+        reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
+    elif reason == 'required':
+      if 'memberKey' in message:
+        reason = ErrorReason.MEMBER_NOT_FOUND.value
+    elif reason == 'conditionNotMet':
+      if 'Cyclic memberships not allowed' in message:
+        reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
+  except KeyError:
+    reason = f'{http_status}'
+  return (http_status, reason, message)

src/gapi/errors_test.py

@@ -0,0 +1,209 @@
+"""Python unit tests for gapi.errors"""
+
+import json
+
+import unittest
+from unittest.mock import patch
+
+import googleapiclient.errors
+from gapi import errors
+
+
+def create_simple_http_error(status, reason, message):
+  content = errors._create_http_error_dict(status, reason, message)
+  return create_http_error(status, content)
+
+
+def create_http_error(status, content):
+  response = {
+      'status': status,
+      'content-type': 'application/json',
+  }
+  content_as_bytes = json.dumps(content).encode('UTF-8')
+  return googleapiclient.errors.HttpError(response, content_as_bytes)
+
+
+class ErrorsTest(unittest.TestCase):
+
+  def test_get_gapi_error_detail_quota_exceeded(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_invalid_domain(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_invalid_signing_key(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_detail_unknown_error(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_retry_http_error(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_prints_soft_errors(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_exits_on_unrecoverable_errors(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_quota_exceeded_for_current_request(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_quota_exceeded_high_request_rate(self):
+    # TODO: Add test logic once the opening ValueError exception case has a
+    # repro case (i.e. an Exception type/format that will cause it to raise).
+    pass
+
+  def test_get_gapi_error_extracts_user_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: userKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: userKey.')
+
+  def test_get_gapi_error_extracts_group_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: groupKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: groupKey.')
+
+  def test_get_gapi_error_extracts_member_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: memberKey.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: memberKey.')
+
+  def test_get_gapi_error_extracts_domain_not_found(self):
+    err = create_simple_http_error(404, 'notFound', 'Domain not found.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
+    self.assertEqual(message, 'Domain not found.')
+
+  def test_get_gapi_error_extracts_generic_resource_not_found(self):
+    err = create_simple_http_error(404, 'notFound',
+                                   'Resource Not Found: unknownResource.')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 404)
+    self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
+    self.assertEqual(message, 'Resource Not Found: unknownResource.')
+
+  def test_get_gapi_error_extracts_invalid_userid(self):
+    err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+    self.assertEqual(message, 'Invalid Input: userId')
+
+  def test_get_gapi_error_extracts_invalid_member(self):
+    err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
+    self.assertEqual(message, 'Invalid Input: memberKey')
+
+  def test_get_gapi_error_extracts_bad_request(self):
+    err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
+    self.assertEqual(message, 'Bad Request')
+
+  def test_get_gapi_error_extracts_service_not_available(self):
+    err = create_simple_http_error(400, 'failedPrecondition',
+                                   'Mail service not enabled')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
+    self.assertEqual(message, 'Mail service not enabled')
+
+  def test_get_gapi_error_extracts_required_member_not_found(self):
+    err = create_simple_http_error(400, 'required',
+                                   'Missing required field: memberKey')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+    self.assertEqual(message, 'Missing required field: memberKey')
+
+  def test_get_gapi_error_extracts_cyclic_memberships_error(self):
+    err = create_simple_http_error(400, 'conditionNotMet',
+                                   'Cyclic memberships not allowed')
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, 400)
+    self.assertEqual(reason,
+                     errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
+    self.assertEqual(message, 'Cyclic memberships not allowed')
+
+  def test_get_gapi_error_extracts_single_error_with_message(self):
+    status_code = 999
+    response = {'status': status_code}
+    # This error does not have an "errors" key describing each error.
+    content = {'error': {'code': status_code, 'message': 'unknown error'}}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, status_code)
+    self.assertEqual(reason, str(status_code))
+    self.assertEqual(message, content['error']['message'])
+
+  def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
+      self):
+    status_code = 999
+    response = {'status': status_code}
+    # This error only has an error_description_field and an unknown description.
+    content = {'error_description': 'something errored'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    with self.assertRaises(SystemExit) as context:
+      errors.get_gapi_error_detail(err)
+    self.assertEqual(4, context.exception.code)
+
+  def test_get_gapi_error_exits_on_invalid_error_description(self):
+    status_code = 400
+    response = {'status': status_code}
+    content = {'error_description': 'Invalid Value'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    http_status, reason, message = errors.get_gapi_error_detail(err)
+    self.assertEqual(http_status, status_code)
+    self.assertEqual(reason, errors.ErrorReason.INVALID.value)
+    self.assertEqual(message, 'Invalid Value')
+
+  def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
+    status_code = 900
+    response = {'status': status_code}
+    content = {'notErrorContentThatIsExpected': 'foo'}
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+    with self.assertRaises(SystemExit) as context:
+      errors.get_gapi_error_detail(err)
+    self.assertEqual(4, context.exception.code)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/gapi/reports.py

@@ -0,0 +1,325 @@
+import datetime
+import sys
+
+import __main__
+from var import *
+import controlflow
+import display
+import gapi
+import utils
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('reports')
+
+
+REPORT_CHOICE_MAP = {
+    'access': 'access_transparency',
+    'accesstransparency': 'access_transparency',
+    'calendars': 'calendar',
+    'customers': 'customer',
+    'doc': 'drive',
+    'docs': 'drive',
+    'domain': 'customer',
+    'enterprisegroups': 'groups_enterprise',
+    'google+': 'gplus',
+    'group': 'groups',
+    'groupsenterprise': 'groups_enterprise',
+    'hangoutsmeet': 'meet',
+    'logins': 'login',
+    'oauthtoken': 'token',
+    'tokens': 'token',
+    'users': 'user',
+    'useraccounts': 'user_accounts',
+}
+
+
+def showReport():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    report = sys.argv[2].lower()
+    report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    valid_apps = gapi.get_enum_values_minus_unspecified(
+        rep._rootDesc['resources']['activities']['methods']['list'][
+            'parameters']['applicationName']['enum'])+['customer', 'user']
+    if report not in valid_apps:
+        controlflow.expected_argument_exit(
+            "report", ", ".join(sorted(valid_apps)), report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    filters = parameters = actorIpAddress = startTime = endTime = eventName = orgUnitId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    to_drive = False
+    userKey = 'all'
+    fullDataRequired = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'date':
+            tryDate = utils.get_yyyymmdd(sys.argv[i+1])
+            i += 2
+        elif myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            i += 2
+        elif myarg == 'fulldatarequired':
+            fullDataRequired = []
+            fdr = sys.argv[i+1].lower()
+            if fdr and fdr != 'all':
+                fullDataRequired = fdr.replace(',', ' ').split()
+            i += 2
+        elif myarg == 'start':
+            startTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'end':
+            endTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'event':
+            eventName = sys.argv[i+1]
+            i += 2
+        elif myarg == 'user':
+            userKey = __main__.normalizeEmailAddressOrUID(sys.argv[i+1])
+            i += 2
+        elif myarg in ['filter', 'filters']:
+            filters = sys.argv[i+1]
+            i += 2
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i+1]
+            i += 2
+        elif myarg == 'ip':
+            actorIpAddress = sys.argv[i+1]
+            i += 2
+        elif myarg == 'todrive':
+            to_drive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam report")
+    if report == 'user':
+        while True:
+            try:
+                if fullDataRequired is not None:
+                    warnings = gapi.get_items(rep.userUsageReport(), 'get',
+                                              'warnings',
+                                              throw_reasons=throw_reasons,
+                                              date=tryDate, userKey=userKey,
+                                              customerId=customerId,
+                                              orgUnitID=orgUnitId,
+                                              fields='warnings')
+                    fullData, tryDate = _check_full_data_available(
+                        warnings, tryDate, fullDataRequired)
+                    if fullData < 0:
+                        print('No user report available.')
+                        sys.exit(1)
+                    if fullData == 0:
+                        continue
+                page_message = gapi.got_total_items_msg('Users', '...\n')
+                usage = gapi.get_all_pages(rep.userUsageReport(), 'get',
+                                           'usageReports',
+                                           page_message=page_message,
+                                           throw_reasons=throw_reasons,
+                                           date=tryDate, userKey=userKey,
+                                           customerId=customerId,
+                                           orgUnitID=orgUnitId,
+                                           filters=filters,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No user report available.')
+            sys.exit(1)
+        titles = ['email', 'date']
+        csvRows = []
+        ptypes = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
+        for user_report in usage:
+            if 'entity' not in user_report:
+                continue
+            row = {'email': user_report['entity']
+                            ['userEmail'], 'date': tryDate}
+            for item in user_report.get('parameters', []):
+                if 'name' not in item:
+                    continue
+                name = item['name']
+                if not name in titles:
+                    titles.append(name)
+                for ptype in ptypes:
+                    if ptype in item:
+                        row[name] = item[ptype]
+                        break
+                else:
+                    row[name] = ''
+            csvRows.append(row)
+        display.write_csv_file(
+            csvRows, titles, f'User Reports - {tryDate}', to_drive)
+    elif report == 'customer':
+        while True:
+            try:
+                if fullDataRequired is not None:
+                    warnings = gapi.get_items(rep.customerUsageReports(),
+                                              'get', 'warnings',
+                                              throw_reasons=throw_reasons,
+                                              customerId=customerId,
+                                              date=tryDate,
+                                              fields='warnings')
+                    fullData, tryDate = _check_full_data_available(
+                        warnings, tryDate, fullDataRequired)
+                    if fullData < 0:
+                        print('No customer report available.')
+                        sys.exit(1)
+                    if fullData == 0:
+                        continue
+                usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                                           'usageReports',
+                                           throw_reasons=throw_reasons,
+                                           customerId=customerId,
+                                           date=tryDate,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No customer report available.')
+            sys.exit(1)
+        titles = ['name', 'value', 'client_id']
+        csvRows = []
+        auth_apps = list()
+        for item in usage[0]['parameters']:
+            if 'name' not in item:
+                continue
+            name = item['name']
+            if 'intValue' in item:
+                value = item['intValue']
+            elif 'msgValue' in item:
+                if name == 'accounts:authorized_apps':
+                    for subitem in item['msgValue']:
+                        app = {}
+                        for an_item in subitem:
+                            if an_item == 'client_name':
+                                app['name'] = 'App: ' + \
+                                    subitem[an_item].replace('\n', '\\n')
+                            elif an_item == 'num_users':
+                                app['value'] = f'{subitem[an_item]} users'
+                            elif an_item == 'client_id':
+                                app['client_id'] = subitem[an_item]
+                        auth_apps.append(app)
+                    continue
+                values = []
+                for subitem in item['msgValue']:
+                    if 'count' in subitem:
+                        mycount = myvalue = None
+                        for key, value in list(subitem.items()):
+                            if key == 'count':
+                                mycount = value
+                            else:
+                                myvalue = value
+                            if mycount and myvalue:
+                                values.append(f'{myvalue}:{mycount}')
+                        value = ' '.join(values)
+                    elif 'version_number' in subitem \
+                         and 'num_devices' in subitem:
+                        values.append(
+                            f'{subitem["version_number"]}:'
+                            f'{subitem["num_devices"]}')
+                    else:
+                        continue
+                    value = ' '.join(sorted(values, reverse=True))
+            csvRows.append({'name': name, 'value': value})
+        for app in auth_apps:  # put apps at bottom
+            csvRows.append(app)
+        display.write_csv_file(
+            csvRows, titles, f'Customer Report - {tryDate}', todrive=to_drive)
+    else:
+        page_message = gapi.got_total_items_msg('Activities', '...\n')
+        activities = gapi.get_all_pages(rep.activities(), 'list', 'items',
+                                        page_message=page_message,
+                                        applicationName=report,
+                                        userKey=userKey,
+                                        customerId=customerId,
+                                        actorIpAddress=actorIpAddress,
+                                        startTime=startTime, endTime=endTime,
+                                        eventName=eventName, filters=filters,
+                                        orgUnitID=orgUnitId)
+        if activities:
+            titles = ['name']
+            csvRows = []
+            for activity in activities:
+                events = activity['events']
+                del activity['events']
+                activity_row = utils.flatten_json(activity)
+                purge_parameters = True
+                for event in events:
+                    for item in event.get('parameters', []):
+                        if set(item) == set(['value', 'name']):
+                            event[item['name']] = item['value']
+                        elif set(item) == set(['intValue', 'name']):
+                            if item['name'] in ['start_time', 'end_time']:
+                                val = item.get('intValue')
+                                if val is not None:
+                                    val = int(val)
+                                    if val >= 62135683200:
+                                        event[item['name']] = \
+                                            datetime.datetime.fromtimestamp(
+                                                val-62135683200).isoformat()
+                            else:
+                                event[item['name']] = item['intValue']
+                        elif set(item) == set(['boolValue', 'name']):
+                            event[item['name']] = item['boolValue']
+                        elif set(item) == set(['multiValue', 'name']):
+                            event[item['name']] = ' '.join(item['multiValue'])
+                        elif item['name'] == 'scope_data':
+                            parts = {}
+                            for message in item['multiMessageValue']:
+                                for mess in message['parameter']:
+                                    value = mess.get('value', ' '.join(
+                                        mess.get('multiValue', [])))
+                                    parts[mess['name']] = parts.get(
+                                        mess['name'], [])+[value]
+                            for part, v in parts.items():
+                                if part == 'scope_name':
+                                    part = 'scope'
+                                event[part] = ' '.join(v)
+                        else:
+                            purge_parameters = False
+                    if purge_parameters:
+                        event.pop('parameters', None)
+                    row = utils.flatten_json(event)
+                    row.update(activity_row)
+                    for item in row:
+                        if item not in titles:
+                            titles.append(item)
+                    csvRows.append(row)
+            display.sort_csv_titles(['name', ], titles)
+            display.write_csv_file(
+                csvRows, titles, f'{report.capitalize()} Activity Report',
+                to_drive)
+
+
+def _adjust_date(errMsg):
+    match_date = re.match('Data for dates later than (.*) is not yet '
+                          'available. Please check back later', errMsg)
+    if not match_date:
+        match_date = re.match('Start date can not be later than (.*)', errMsg)
+    if not match_date:
+        controlflow.system_error_exit(4, errMsg)
+    return str(match_date.group(1))
+
+
+def _check_full_data_available(warnings, tryDate, fullDataRequired):
+    one_day = datetime.timedelta(days=1)
+    for warning in warnings:
+        if warning['code'] == 'PARTIAL_DATA_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    tryDateTime = datetime.datetime.strptime(
+                        tryDate, YYYYMMDD_FORMAT)
+                    tryDateTime -= one_day
+                    return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+        elif warning['code'] == 'DATA_NOT_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    return (-1, tryDate)
+    return (1, tryDate)

src/gapi/storage.py

@@ -0,0 +1,73 @@
+import base64
+import os
+import re
+import sys
+
+import googleapiclient
+
+import __main__
+from var import *
+import controlflow
+import fileutils
+import gapi
+import utils
+
+
+def build_gapi():
+    return __main__.buildGAPIObject('storage')
+
+
+def get_cloud_storage_object(s, bucket, object_, local_file=None,
+                           expectedMd5=None):
+    if not local_file:
+        local_file = object_
+    if os.path.exists(local_file):
+        sys.stdout.write(' File already exists. ')
+        sys.stdout.flush()
+        if expectedMd5:
+            sys.stdout.write(f'Verifying {expectedMd5} hash...')
+            sys.stdout.flush()
+            if utils.md5_matches_file(local_file, expectedMd5, False):
+                print('VERIFIED')
+                return
+            print('not verified. Downloading again and over-writing...')
+        else:
+            return  # nothing to verify, just assume we're good.
+    print(f'saving to {local_file}')
+    request = s.objects().get_media(bucket=bucket, object=object_)
+    file_path = os.path.dirname(local_file)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    f = fileutils.open_file(local_file, 'wb')
+    downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+    done = False
+    while not done:
+        status, done = downloader.next_chunk()
+        sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
+        sys.stdout.flush()
+    sys.stdout.write('\n Download complete. Flushing to disk...\n')
+    fileutils.close_file(f, True)
+    if expectedMd5:
+        f = fileutils.open_file(local_file, 'rb')
+        sys.stdout.write(f' Verifying file hash is {expectedMd5}...')
+        sys.stdout.flush()
+        utils.md5_matches_file(local_file, expectedMd5, True)
+        print('VERIFIED')
+        fileutils.close_file(f)
+
+
+def download_bucket():
+    bucket = sys.argv[3]
+    s = build_gapi()
+    page_message = gapi.got_total_items_msg('Files', '...')
+    fields = 'nextPageToken,items(name,id,md5Hash)'
+    objects = gapi.get_all_pages(s.objects(), 'list', 'items',
+                                 page_message=page_message, bucket=bucket,
+                                 projection='noAcl', fields=fields)
+    i = 1
+    for object_ in objects:
+        print(f'{i}/{len(objects)}')
+        expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
+        get_cloud_storage_object(
+            s, bucket, object_['name'], expectedMd5=expectedMd5)
+        i += 1

src/gapi/vault.py

@@ -0,0 +1,757 @@
+import datetime
+import json
+import sys
+
+import googleapiclient.http
+
+import __main__
+from var import *
+import controlflow
+import display
+import fileutils
+import gapi
+import gapi.storage
+import utils
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('vault')
+
+
+def validateCollaborators(collaboratorList, cd):
+    collaborators = []
+    for collaborator in collaboratorList.split(','):
+        collaborator_id = __main__.convertEmailAddressToUID(collaborator, cd)
+        if not collaborator_id:
+            controlflow.system_error_exit(4, f'failed to get a UID for '
+                                             f'{collaborator}. Please make '
+                                             f'sure this is a real user.')
+        collaborators.append({'email': collaborator, 'id': collaborator_id})
+    return collaborators
+
+
+def createMatter():
+    v = buildGAPIObject()
+    matter_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    body = {'name': f'New Matter - {matter_time}'}
+    collaborators = []
+    cd = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['collaborator', 'collaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create matter")
+    matterId = gapi.call(v.matters(), 'create', body=body,
+                         fields='matterId')['matterId']
+    print(f'Created matter {matterId}')
+    for collaborator in collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {'matterPermission': {
+            'role': 'COLLABORATOR',
+            'accountId': collaborator['id']}}
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+
+
+VAULT_SEARCH_METHODS_MAP = {
+    'account': 'ACCOUNT',
+    'accounts': 'ACCOUNT',
+    'entireorg': 'ENTIRE_ORG',
+    'everyone': 'ENTIRE_ORG',
+    'orgunit': 'ORG_UNIT',
+    'ou': 'ORG_UNIT',
+    'room': 'ROOM',
+    'rooms': 'ROOM',
+    'shareddrive': 'SHARED_DRIVE',
+    'shareddrives': 'SHARED_DRIVE',
+    'teamdrive': 'SHARED_DRIVE',
+    'teamdrives': 'SHARED_DRIVE',
+}
+VAULT_SEARCH_METHODS_LIST = ['accounts',
+                             'orgunit', 'shareddrives', 'rooms', 'everyone']
+
+
+def createExport():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['corpus']['enum'])
+    allowed_scopes = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['dataScope']['enum'])
+    allowed_formats = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['MailExportOptions']['properties']
+        ['exportFormat']['enum'])
+    export_format = 'MBOX'
+    showConfidentialModeContent = None  # default to not even set
+    matterId = None
+    body = {'query': {'dataScope': 'ALL_DATA'}, 'exportOptions': {}}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            body['matterId'] = matterId
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'corpus':
+            body['query']['corpus'] = sys.argv[i+1].upper()
+            if body['query']['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit(
+                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+            i += 2
+        elif myarg in VAULT_SEARCH_METHODS_MAP:
+            if body['query'].get('searchMethod'):
+                message = f'Multiple search methods ' \
+                          f'({", ".join(VAULT_SEARCH_METHODS_LIST)})' \
+                          f'specified, only one is allowed'
+                controlflow.system_error_exit(3, message)
+            searchMethod = VAULT_SEARCH_METHODS_MAP[myarg]
+            body['query']['searchMethod'] = searchMethod
+            if searchMethod == 'ACCOUNT':
+                body['query']['accountInfo'] = {
+                    'emails': sys.argv[i+1].split(',')}
+                i += 2
+            elif searchMethod == 'ORG_UNIT':
+                body['query']['orgUnitInfo'] = {
+                    'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+                i += 2
+            elif searchMethod == 'SHARED_DRIVE':
+                body['query']['sharedDriveInfo'] = {
+                    'sharedDriveIds': sys.argv[i+1].split(',')}
+                i += 2
+            elif searchMethod == 'ROOM':
+                body['query']['hangoutsChatInfo'] = {
+                    'roomId': sys.argv[i+1].split(',')}
+                i += 2
+            else:
+                i += 1
+        elif myarg == 'scope':
+            body['query']['dataScope'] = sys.argv[i+1].upper()
+            if body['query']['dataScope'] not in allowed_scopes:
+                controlflow.expected_argument_exit(
+                    "scope", ", ".join(allowed_scopes), sys.argv[i+1])
+            i += 2
+        elif myarg in ['terms']:
+            body['query']['terms'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            body['query']['startTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            body['query']['endTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i+1])
+            i += 2
+        elif myarg in ['timezone']:
+            body['query']['timeZone'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['excludedrafts']:
+            body['query']['mailOptions'] = {
+                'excludeDrafts': __main__.getBoolean(sys.argv[i+1], myarg)}
+            i += 2
+        elif myarg in ['driveversiondate']:
+            body['query'].setdefault('driveOptions', {})['versionDate'] = \
+                utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['includeshareddrives', 'includeteamdrives']:
+            body['query'].setdefault('driveOptions', {})[
+                'includeSharedDrives'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg in ['includerooms']:
+            body['query']['hangoutsChatOptions'] = {
+                'includeRooms': __main__.getBoolean(sys.argv[i+1], myarg)}
+            i += 2
+        elif myarg in ['format']:
+            export_format = sys.argv[i+1].upper()
+            if export_format not in allowed_formats:
+                controlflow.expected_argument_exit(
+                    "export format", ", ".join(allowed_formats), export_format)
+            i += 2
+        elif myarg in ['showconfidentialmodecontent']:
+            showConfidentialModeContent = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg in ['region']:
+            allowed_regions = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['ExportOptions']['properties'][
+                    'region']['enum'])
+            body['exportOptions']['region'] = sys.argv[i+1].upper()
+            if body['exportOptions']['region'] not in allowed_regions:
+                controlflow.expected_argument_exit("region", ", ".join(
+                    allowed_regions), body['exportOptions']['region'])
+            i += 2
+        elif myarg in ['includeaccessinfo']:
+            body['exportOptions'].setdefault('driveOptions', {})[
+                'includeAccessInfo'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create export")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new export.')
+    if 'corpus' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a corpus for the ' \
+          f'new export. Choose one of {", ".join(allowed_corpuses)}')
+    if 'searchMethod' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a search method ' \
+          'for the new export. Choose one of ' \
+          f'{", ".join(VAULT_SEARCH_METHODS_LIST)}')
+    if 'name' not in body:
+        corpus_name = body["query"]["corpus"]
+        corpus_date = datetime.datetime.now()
+        body['name'] = f'GAM {corpus_name} export - {corpus_date}'
+    options_field = None
+    if body['query']['corpus'] == 'MAIL':
+        options_field = 'mailOptions'
+    elif body['query']['corpus'] == 'GROUPS':
+        options_field = 'groupsOptions'
+    elif body['query']['corpus'] == 'HANGOUTS_CHAT':
+        options_field = 'hangoutsChatOptions'
+    if options_field:
+        body['exportOptions'].pop('driveOptions', None)
+        body['exportOptions'][options_field] = {'exportFormat': export_format}
+        if showConfidentialModeContent is not None:
+            body['exportOptions'][options_field][
+                'showConfidentialModeContent'] = showConfidentialModeContent
+    results = gapi.call(v.matters().exports(), 'create',
+                        matterId=matterId, body=body)
+    print(f'Created export {results["id"]}')
+    display.print_json(results)
+
+
+def deleteExport():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    print(f'Deleting export {sys.argv[4]} / {exportId}')
+    gapi.call(v.matters().exports(), 'delete',
+              matterId=matterId, exportId=exportId)
+
+
+def getExportInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    export = gapi.call(v.matters().exports(), 'get',
+                       matterId=matterId, exportId=exportId)
+    display.print_json(export)
+
+
+def createHold():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Hold']['properties']['corpus']['enum'])
+    body = {'query': {}}
+    i = 3
+    query = None
+    start_time = None
+    end_time = None
+    matterId = None
+    accounts = []
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg == 'corpus':
+            body['corpus'] = sys.argv[i+1].upper()
+            if body['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit(
+                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+            i += 2
+        elif myarg in ['accounts', 'users', 'groups']:
+            accounts = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new hold.')
+    if not body.get('name'):
+        controlflow.system_error_exit(
+            3, 'you must specify a name for the new hold.')
+    if not body.get('corpus'):
+        controlflow.system_error_exit(3, f'you must specify a corpus for ' \
+          f'the new hold. Choose one of {", ".join(allowed_corpuses)}')
+    if body['corpus'] == 'HANGOUTS_CHAT':
+        query_type = 'hangoutsChatQuery'
+    else:
+        query_type = f'{body["corpus"].lower()}Query'
+    body['query'][query_type] = {}
+    if body['corpus'] == 'DRIVE':
+        if query:
+            try:
+                body['query'][query_type] = json.loads(query)
+            except ValueError as e:
+                controlflow.system_error_exit(3, f'{str(e)}, query: {query}')
+    elif body['corpus'] in ['GROUPS', 'MAIL']:
+        if query:
+            body['query'][query_type] = {'terms': query}
+        if start_time:
+            body['query'][query_type]['startTime'] = start_time
+        if end_time:
+            body['query'][query_type]['endTime'] = end_time
+    if accounts:
+        body['accounts'] = []
+        cd = __main__.buildGAPIObject('directory')
+        account_type = 'group' if body['corpus'] == 'GROUPS' else 'user'
+        for account in accounts:
+            body['accounts'].append(
+                {'accountId': __main__.convertEmailAddressToUID(account,
+                                                                cd,
+                                                                account_type)}
+            )
+    holdId = gapi.call(v.matters().holds(), 'create',
+                       matterId=matterId, body=body, fields='holdId')
+    print(f'Created hold {holdId["holdId"]}')
+
+
+def deleteHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam delete hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    print(f'Deleting hold {hold} / {holdId}')
+    gapi.call(v.matters().holds(), 'delete', matterId=matterId, holdId=holdId)
+
+
+def getHoldInfo():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam info hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    results = gapi.call(v.matters().holds(), 'get',
+                        matterId=matterId, holdId=holdId)
+    cd = __main__.buildGAPIObject('directory')
+    if 'accounts' in results:
+        account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
+        for i in range(0, len(results['accounts'])):
+            uid = f'uid:{results["accounts"][i]["accountId"]}'
+            acct_email = __main__.convertUIDtoEmailAddress(
+                uid, cd, [account_type])
+            results['accounts'][i]['email'] = acct_email
+    if 'orgUnit' in results:
+        results['orgUnit']['orgUnitPath'] = __main__.doGetOrgInfo(
+            results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
+    display.print_json(results)
+
+
+def convertExportNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'exports(id,name),nextPageToken'
+    exports = gapi.get_all_pages(v.matters().exports(
+    ), 'list', 'exports', matterId=matterId, fields=fields)
+    for export in exports:
+        if export['name'].lower() == nameOrID:
+            return export['id']
+    controlflow.system_error_exit(4, f'could not find export name {nameOrID} '
+                                     f'in matter {matterId}')
+
+
+def convertHoldNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'holds(holdId,name),nextPageToken'
+    holds = gapi.get_all_pages(v.matters().holds(
+    ), 'list', 'holds', matterId=matterId, fields=fields)
+    for hold in holds:
+        if hold['name'].lower() == nameOrID:
+            return hold['holdId']
+    controlflow.system_error_exit(4, f'could not find hold name {nameOrID} '
+                                     f'in matter {matterId}')
+
+
+def convertMatterNameToID(v, nameOrID):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'matters(matterId,name),nextPageToken'
+    matters = gapi.get_all_pages(v.matters(
+    ), 'list', 'matters', view='BASIC', fields=fields)
+    for matter in matters:
+        if matter['name'].lower() == nameOrID:
+            return matter['matterId']
+    return None
+
+
+def getMatterItem(v, nameOrID):
+    matterId = convertMatterNameToID(v, nameOrID)
+    if not matterId:
+        controlflow.system_error_exit(4, f'could not find matter {nameOrID}')
+    return matterId
+
+
+def updateHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    body = {}
+    query = None
+    add_accounts = []
+    del_accounts = []
+    start_time = None
+    end_time = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['addusers', 'addaccounts', 'addgroups']:
+            add_accounts = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg in ['removeusers', 'removeaccounts', 'removegroups']:
+            del_accounts = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam update hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    if query or start_time or end_time or body.get('orgUnit'):
+        fields = 'corpus,query,orgUnit'
+        old_body = gapi.call(v.matters().holds(
+        ), 'get', matterId=matterId, holdId=holdId, fields=fields)
+        body['query'] = old_body['query']
+        body['corpus'] = old_body['corpus']
+        if 'orgUnit' in old_body and 'orgUnit' not in body:
+            # bah, API requires this to be sent
+            # on update even when it's not changing
+            body['orgUnit'] = old_body['orgUnit']
+        query_type = f'{body["corpus"].lower()}Query'
+        if body['corpus'] == 'DRIVE':
+            if query:
+                try:
+                    body['query'][query_type] = json.loads(query)
+                except ValueError as e:
+                    message = f'{str(e)}, query: {query}'
+                    controlflow.system_error_exit(3, message)
+        elif body['corpus'] in ['GROUPS', 'MAIL']:
+            if query:
+                body['query'][query_type]['terms'] = query
+            if start_time:
+                body['query'][query_type]['startTime'] = start_time
+            if end_time:
+                body['query'][query_type]['endTime'] = end_time
+    if body:
+        print(f'Updating hold {hold} / {holdId}')
+        gapi.call(v.matters().holds(), 'update',
+                  matterId=matterId, holdId=holdId, body=body)
+    if add_accounts or del_accounts:
+        cd = __main__.buildGAPIObject('directory')
+        for account in add_accounts:
+            print(f'adding {account} to hold.')
+            add_body = {'accountId': __main__.convertEmailAddressToUID(account, cd)}
+            gapi.call(v.matters().holds().accounts(), 'create',
+                      matterId=matterId, holdId=holdId, body=add_body)
+        for account in del_accounts:
+            print(f'removing {account} from hold.')
+            accountId = __main__.convertEmailAddressToUID(account, cd)
+            gapi.call(v.matters().holds().accounts(), 'delete',
+                      matterId=matterId, holdId=holdId, accountId=accountId)
+
+
+def updateMatter(action=None):
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    body = {}
+    action_kwargs = {'body': {}}
+    add_collaborators = []
+    remove_collaborators = []
+    cd = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            action = sys.argv[i+1].lower()
+            if action not in VAULT_MATTER_ACTIONS:
+                controlflow.system_error_exit(3, f'allowed actions are ' \
+                    f'{", ".join(VAULT_MATTER_ACTIONS)}, got {action}')
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['addcollaborator', 'addcollaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            add_collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        elif myarg in ['removecollaborator', 'removecollaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            remove_collaborators.extend(
+                validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam update matter")
+    if action == 'delete':
+        action_kwargs = {}
+    if body:
+        print(f'Updating matter {sys.argv[3]}...')
+        if 'name' not in body or 'description' not in body:
+            # bah, API requires name/description to be sent
+            # on update even when it's not changing
+            result = gapi.call(v.matters(), 'get',
+                               matterId=matterId, view='BASIC')
+            body.setdefault('name', result['name'])
+            body.setdefault('description', result.get('description'))
+        gapi.call(v.matters(), 'update', body=body, matterId=matterId)
+    if action:
+        print(f'Performing {action} on matter {sys.argv[3]}')
+        gapi.call(v.matters(), action, matterId=matterId, **action_kwargs)
+    for collaborator in add_collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {'matterPermission': {'role': 'COLLABORATOR',
+                                     'accountId': collaborator['id']}}
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+    for collaborator in remove_collaborators:
+        print(f' removing collaborator {collaborator["email"]}')
+        gapi.call(v.matters(), 'removePermissions', matterId=matterId,
+                  body={'accountId': collaborator['id']})
+
+
+def getMatterInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    result = gapi.call(v.matters(), 'get', matterId=matterId, view='FULL')
+    if 'matterPermissions' in result:
+        cd = __main__.buildGAPIObject('directory')
+        for i in range(0, len(result['matterPermissions'])):
+            uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
+            user_email = __main__.convertUIDtoEmailAddress(uid, cd)
+            result['matterPermissions'][i]['email'] = user_email
+    display.print_json(result)
+
+
+def downloadExport():
+    verifyFiles = True
+    extractFiles = True
+    v = buildGAPIObject()
+    s = gapi.storage.build_gapi()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i+1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        elif myarg == 'noverify':
+            verifyFiles = False
+            i += 1
+        elif myarg == 'noextract':
+            extractFiles = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam download export")
+    export = gapi.call(v.matters().exports(), 'get',
+                       matterId=matterId, exportId=exportId)
+    for s_file in export['cloudStorageSink']['files']:
+        bucket = s_file['bucketName']
+        s_object = s_file['objectName']
+        filename = os.path.join(targetFolder, s_object.replace('/', '-'))
+        print(f'saving to {filename}')
+        request = s.objects().get_media(bucket=bucket, object=s_object)
+        f = fileutils.open_file(filename, 'wb')
+        downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+        done = False
+        while not done:
+            status, done = downloader.next_chunk()
+            sys.stdout.write(
+                ' Downloaded: {0:>7.2%}\r'.format(status.progress()))
+            sys.stdout.flush()
+        sys.stdout.write('\n Download complete. Flushing to disk...\n')
+        fileutils.close_file(f, True)
+        if verifyFiles:
+            expected_hash = s_file['md5Hash']
+            sys.stdout.write(f' Verifying file hash is {expected_hash}...')
+            sys.stdout.flush()
+            utils.md5_matches_file(filename, expected_hash, True)
+            print('VERIFIED')
+        if extractFiles and re.search(r'\.zip$', filename):
+            __main__.extract_nested_zip(filename, targetFolder)
+
+
+def printMatters():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'name', 'description', 'state']
+    titles = initialTitles[:]
+    view = 'FULL'
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            view = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print matters")
+    __main__.printGettingAllItems('Vault Matters', None)
+    page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
+    matters = gapi.get_all_pages(
+        v.matters(), 'list', 'matters', page_message=page_message, view=view)
+    for matter in matters:
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(matter), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Matters', todrive)
+
+
+def printExports():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'id', 'name', 'createTime', 'status']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print exports")
+    if not matters:
+        fields = 'matters(matterId,state),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(
+        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        for matter in matters_results:
+            matterState = matter['state']
+            matterId = matter['matterId']
+            matterIds.append(matterId)
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving exports for matter {matterId}\n')
+        exports = gapi.get_all_pages(
+            v.matters().exports(), 'list', 'exports', matterId=matterId)
+        for export in exports:
+            display.add_row_titles_to_csv_file(utils.flatten_json(
+                export, flattened={'matterId': matterId}), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Exports', todrive)
+
+
+def printHolds():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'holdId', 'name', 'corpus', 'updateTime']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print holds")
+    if not matters:
+        fields = 'matters(matterId,state),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(
+        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        for matter in matters_results:
+            matterState = matter['state']
+            matterId = matter['matterId']
+            matterIds.append(matterId)
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving holds for matter {matterId}\n')
+        holds = gapi.get_all_pages(
+            v.matters().holds(), 'list', 'holds', matterId=matterId)
+        for hold in holds:
+            display.add_row_titles_to_csv_file(utils.flatten_json(
+                hold, flattened={'matterId': matterId}), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Holds', todrive)

src/googleapiclient/__init__.py

@@ -1,27 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-__version__ = "1.6.3"
-
-# Set default logging handler to avoid "No handler found" warnings.
-import logging
-
-try:  # Python 2.7+
-    from logging import NullHandler
-except ImportError:
-    class NullHandler(logging.Handler):
-        def emit(self, record):
-            pass
-
-logging.getLogger(__name__).addHandler(NullHandler())

src/googleapiclient/_auth.py

@@ -1,133 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helpers for authentication using oauth2client or google-auth."""
-
-import httplib2
-
-try:
-    import google.auth
-    import google.auth.credentials
-    import google_auth_httplib2
-    HAS_GOOGLE_AUTH = True
-except ImportError:  # pragma: NO COVER
-    HAS_GOOGLE_AUTH = False
-
-try:
-    import oauth2client
-    import oauth2client.client
-    HAS_OAUTH2CLIENT = True
-except ImportError:  # pragma: NO COVER
-    HAS_OAUTH2CLIENT = False
-
-
-def default_credentials():
-    """Returns Application Default Credentials."""
-    if HAS_GOOGLE_AUTH:
-        credentials, _ = google.auth.default()
-        return credentials
-    elif HAS_OAUTH2CLIENT:
-        return oauth2client.client.GoogleCredentials.get_application_default()
-    else:
-        raise EnvironmentError(
-            'No authentication library is available. Please install either '
-            'google-auth or oauth2client.')
-
-
-def with_scopes(credentials, scopes):
-    """Scopes the credentials if necessary.
-
-    Args:
-        credentials (Union[
-            google.auth.credentials.Credentials,
-            oauth2client.client.Credentials]): The credentials to scope.
-        scopes (Sequence[str]): The list of scopes.
-
-    Returns:
-        Union[google.auth.credentials.Credentials,
-            oauth2client.client.Credentials]: The scoped credentials.
-    """
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        return google.auth.credentials.with_scopes_if_required(
-            credentials, scopes)
-    else:
-        try:
-            if credentials.create_scoped_required():
-                return credentials.create_scoped(scopes)
-            else:
-                return credentials
-        except AttributeError:
-            return credentials
-
-
-def authorized_http(credentials):
-    """Returns an http client that is authorized with the given credentials.
-
-    Args:
-        credentials (Union[
-            google.auth.credentials.Credentials,
-            oauth2client.client.Credentials]): The credentials to use.
-
-    Returns:
-        Union[httplib2.Http, google_auth_httplib2.AuthorizedHttp]: An
-            authorized http client.
-    """
-    from googleapiclient.http import build_http
-
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        return google_auth_httplib2.AuthorizedHttp(credentials,
-                                                   http=build_http())
-    else:
-        return credentials.authorize(build_http())
-
-
-def refresh_credentials(credentials):
-    # Refresh must use a new http instance, as the one associated with the
-    # credentials could be a AuthorizedHttp or an oauth2client-decorated
-    # Http instance which would cause a weird recursive loop of refreshing
-    # and likely tear a hole in spacetime.
-    refresh_http = httplib2.Http()
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        request = google_auth_httplib2.Request(refresh_http)
-        return credentials.refresh(request)
-    else:
-        return credentials.refresh(refresh_http)
-
-
-def apply_credentials(credentials, headers):
-    # oauth2client and google-auth have the same interface for this.
-    return credentials.apply(headers)
-
-
-def is_valid(credentials):
-    if HAS_GOOGLE_AUTH and isinstance(
-            credentials, google.auth.credentials.Credentials):
-        return credentials.valid
-    else:
-        return not credentials.access_token_expired
-
-
-def get_credentials_from_http(http):
-    if http is None:
-        return None
-    elif hasattr(http.request, 'credentials'):
-        return http.request.credentials
-    elif (hasattr(http, 'credentials')
-          and not isinstance(http.credentials, httplib2.Credentials)):
-        return http.credentials
-    else:
-        return None

src/googleapiclient/channel.py

@@ -1,293 +0,0 @@
-"""Channel notifications support.
-
-Classes and functions to support channel subscriptions and notifications
-on those channels.
-
-Notes:
-  - This code is based on experimental APIs and is subject to change.
-  - Notification does not do deduplication of notification ids, that's up to
-    the receiver.
-  - Storing the Channel between calls is up to the caller.
-
-
-Example setting up a channel:
-
-  # Create a new channel that gets notifications via webhook.
-  channel = new_webhook_channel("https://example.com/my_web_hook")
-
-  # Store the channel, keyed by 'channel.id'. Store it before calling the
-  # watch method because notifications may start arriving before the watch
-  # method returns.
-  ...
-
-  resp = service.objects().watchAll(
-    bucket="some_bucket_id", body=channel.body()).execute()
-  channel.update(resp)
-
-  # Store the channel, keyed by 'channel.id'. Store it after being updated
-  # since the resource_id value will now be correct, and that's needed to
-  # stop a subscription.
-  ...
-
-
-An example Webhook implementation using webapp2. Note that webapp2 puts
-headers in a case insensitive dictionary, as headers aren't guaranteed to
-always be upper case.
-
-  id = self.request.headers[X_GOOG_CHANNEL_ID]
-
-  # Retrieve the channel by id.
-  channel = ...
-
-  # Parse notification from the headers, including validating the id.
-  n = notification_from_headers(channel, self.request.headers)
-
-  # Do app specific stuff with the notification here.
-  if n.resource_state == 'sync':
-    # Code to handle sync state.
-  elif n.resource_state == 'exists':
-    # Code to handle the exists state.
-  elif n.resource_state == 'not_exists':
-    # Code to handle the not exists state.
-
-
-Example of unsubscribing.
-
-  service.channels().stop(channel.body())
-"""
-from __future__ import absolute_import
-
-import datetime
-import uuid
-
-from googleapiclient import errors
-import six
-
-# Oauth2client < 3 has the positional helper in 'util', >= 3 has it
-# in '_helpers'.
-try:
-  from oauth2client import util
-except ImportError:
-  from oauth2client import _helpers as util
-
-
-# The unix time epoch starts at midnight 1970.
-EPOCH = datetime.datetime.utcfromtimestamp(0)
-
-# Map the names of the parameters in the JSON channel description to
-# the parameter names we use in the Channel class.
-CHANNEL_PARAMS = {
-    'address': 'address',
-    'id': 'id',
-    'expiration': 'expiration',
-    'params': 'params',
-    'resourceId': 'resource_id',
-    'resourceUri': 'resource_uri',
-    'type': 'type',
-    'token': 'token',
-    }
-
-X_GOOG_CHANNEL_ID     = 'X-GOOG-CHANNEL-ID'
-X_GOOG_MESSAGE_NUMBER = 'X-GOOG-MESSAGE-NUMBER'
-X_GOOG_RESOURCE_STATE = 'X-GOOG-RESOURCE-STATE'
-X_GOOG_RESOURCE_URI   = 'X-GOOG-RESOURCE-URI'
-X_GOOG_RESOURCE_ID    = 'X-GOOG-RESOURCE-ID'
-
-
-def _upper_header_keys(headers):
-  new_headers = {}
-  for k, v in six.iteritems(headers):
-    new_headers[k.upper()] = v
-  return new_headers
-
-
-class Notification(object):
-  """A Notification from a Channel.
-
-  Notifications are not usually constructed directly, but are returned
-  from functions like notification_from_headers().
-
-  Attributes:
-    message_number: int, The unique id number of this notification.
-    state: str, The state of the resource being monitored.
-    uri: str, The address of the resource being monitored.
-    resource_id: str, The unique identifier of the version of the resource at
-      this event.
-  """
-  @util.positional(5)
-  def __init__(self, message_number, state, resource_uri, resource_id):
-    """Notification constructor.
-
-    Args:
-      message_number: int, The unique id number of this notification.
-      state: str, The state of the resource being monitored. Can be one
-        of "exists", "not_exists", or "sync".
-      resource_uri: str, The address of the resource being monitored.
-      resource_id: str, The identifier of the watched resource.
-    """
-    self.message_number = message_number
-    self.state = state
-    self.resource_uri = resource_uri
-    self.resource_id = resource_id
-
-
-class Channel(object):
-  """A Channel for notifications.
-
-  Usually not constructed directly, instead it is returned from helper
-  functions like new_webhook_channel().
-
-  Attributes:
-    type: str, The type of delivery mechanism used by this channel. For
-      example, 'web_hook'.
-    id: str, A UUID for the channel.
-    token: str, An arbitrary string associated with the channel that
-      is delivered to the target address with each event delivered
-      over this channel.
-    address: str, The address of the receiving entity where events are
-      delivered. Specific to the channel type.
-    expiration: int, The time, in milliseconds from the epoch, when this
-      channel will expire.
-    params: dict, A dictionary of string to string, with additional parameters
-      controlling delivery channel behavior.
-    resource_id: str, An opaque id that identifies the resource that is
-      being watched. Stable across different API versions.
-    resource_uri: str, The canonicalized ID of the watched resource.
-  """
-
-  @util.positional(5)
-  def __init__(self, type, id, token, address, expiration=None,
-               params=None, resource_id="", resource_uri=""):
-    """Create a new Channel.
-
-    In user code, this Channel constructor will not typically be called
-    manually since there are functions for creating channels for each specific
-    type with a more customized set of arguments to pass.
-
-    Args:
-      type: str, The type of delivery mechanism used by this channel. For
-        example, 'web_hook'.
-      id: str, A UUID for the channel.
-      token: str, An arbitrary string associated with the channel that
-        is delivered to the target address with each event delivered
-        over this channel.
-      address: str,  The address of the receiving entity where events are
-        delivered. Specific to the channel type.
-      expiration: int, The time, in milliseconds from the epoch, when this
-        channel will expire.
-      params: dict, A dictionary of string to string, with additional parameters
-        controlling delivery channel behavior.
-      resource_id: str, An opaque id that identifies the resource that is
-        being watched. Stable across different API versions.
-      resource_uri: str, The canonicalized ID of the watched resource.
-    """
-    self.type = type
-    self.id = id
-    self.token = token
-    self.address = address
-    self.expiration = expiration
-    self.params = params
-    self.resource_id = resource_id
-    self.resource_uri = resource_uri
-
-  def body(self):
-    """Build a body from the Channel.
-
-    Constructs a dictionary that's appropriate for passing into watch()
-    methods as the value of body argument.
-
-    Returns:
-      A dictionary representation of the channel.
-    """
-    result = {
-        'id': self.id,
-        'token': self.token,
-        'type': self.type,
-        'address': self.address
-        }
-    if self.params:
-      result['params'] = self.params
-    if self.resource_id:
-      result['resourceId'] = self.resource_id
-    if self.resource_uri:
-      result['resourceUri'] = self.resource_uri
-    if self.expiration:
-      result['expiration'] = self.expiration
-
-    return result
-
-  def update(self, resp):
-    """Update a channel with information from the response of watch().
-
-    When a request is sent to watch() a resource, the response returned
-    from the watch() request is a dictionary with updated channel information,
-    such as the resource_id, which is needed when stopping a subscription.
-
-    Args:
-      resp: dict, The response from a watch() method.
-    """
-    for json_name, param_name in six.iteritems(CHANNEL_PARAMS):
-      value = resp.get(json_name)
-      if value is not None:
-        setattr(self, param_name, value)
-
-
-def notification_from_headers(channel, headers):
-  """Parse a notification from the webhook request headers, validate
-    the notification, and return a Notification object.
-
-  Args:
-    channel: Channel, The channel that the notification is associated with.
-    headers: dict, A dictionary like object that contains the request headers
-      from the webhook HTTP request.
-
-  Returns:
-    A Notification object.
-
-  Raises:
-    errors.InvalidNotificationError if the notification is invalid.
-    ValueError if the X-GOOG-MESSAGE-NUMBER can't be converted to an int.
-  """
-  headers = _upper_header_keys(headers)
-  channel_id = headers[X_GOOG_CHANNEL_ID]
-  if channel.id != channel_id:
-    raise errors.InvalidNotificationError(
-        'Channel id mismatch: %s != %s' % (channel.id, channel_id))
-  else:
-    message_number = int(headers[X_GOOG_MESSAGE_NUMBER])
-    state = headers[X_GOOG_RESOURCE_STATE]
-    resource_uri = headers[X_GOOG_RESOURCE_URI]
-    resource_id = headers[X_GOOG_RESOURCE_ID]
-    return Notification(message_number, state, resource_uri, resource_id)
-
-
-@util.positional(2)
-def new_webhook_channel(url, token=None, expiration=None, params=None):
-    """Create a new webhook Channel.
-
-    Args:
-      url: str, URL to post notifications to.
-      token: str, An arbitrary string associated with the channel that
-        is delivered to the target address with each notification delivered
-        over this channel.
-      expiration: datetime.datetime, A time in the future when the channel
-        should expire. Can also be None if the subscription should use the
-        default expiration. Note that different services may have different
-        limits on how long a subscription lasts. Check the response from the
-        watch() method to see the value the service has set for an expiration
-        time.
-      params: dict, Extra parameters to pass on channel creation. Currently
-        not used for webhook channels.
-    """
-    expiration_ms = 0
-    if expiration:
-      delta = expiration - EPOCH
-      expiration_ms = delta.microseconds/1000 + (
-          delta.seconds + delta.days*24*3600)*1000
-      if expiration_ms < 0:
-        expiration_ms = 0
-
-    return Channel('web_hook', str(uuid.uuid4()),
-                   token, url, expiration=expiration_ms,
-                   params=params)
-

src/googleapiclient/discovery_cache/__init__.py

@@ -1,45 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Caching utility for the discovery document."""
-
-from __future__ import absolute_import
-
-import logging
-import datetime
-
-
-LOGGER = logging.getLogger(__name__)
-
-DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24  # 1 day
-
-
-def autodetect():
-  """Detects an appropriate cache module and returns it.
-
-  Returns:
-    googleapiclient.discovery_cache.base.Cache, a cache object which
-    is auto detected, or None if no cache object is available.
-  """
-  try:
-    from google.appengine.api import memcache
-    from . import appengine_memcache
-    return appengine_memcache.cache
-  except Exception:
-    try:
-      from . import file_cache
-      return file_cache.cache
-    except Exception as e:
-      LOGGER.warning(e, exc_info=True)
-      return None

src/googleapiclient/discovery_cache/appengine_memcache.py

@@ -1,55 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""App Engine memcache based cache for the discovery document."""
-
-import logging
-
-# This is only an optional dependency because we only import this
-# module when google.appengine.api.memcache is available.
-from google.appengine.api import memcache
-
-from . import base
-from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
-
-
-LOGGER = logging.getLogger(__name__)
-
-NAMESPACE = 'google-api-client'
-
-
-class Cache(base.Cache):
-  """A cache with app engine memcache API."""
-
-  def __init__(self, max_age):
-      """Constructor.
-
-      Args:
-        max_age: Cache expiration in seconds.
-      """
-      self._max_age = max_age
-
-  def get(self, url):
-    try:
-      return memcache.get(url, namespace=NAMESPACE)
-    except Exception as e:
-      LOGGER.warning(e, exc_info=True)
-
-  def set(self, url, content):
-    try:
-      memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
-    except Exception as e:
-      LOGGER.warning(e, exc_info=True)
-
-cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE)

src/googleapiclient/discovery_cache/base.py

@@ -1,45 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""An abstract class for caching the discovery document."""
-
-import abc
-
-
-class Cache(object):
-  """A base abstract cache class."""
-  __metaclass__ = abc.ABCMeta
-
-  @abc.abstractmethod
-  def get(self, url):
-    """Gets the content from the memcache with a given key.
-
-    Args:
-      url: string, the key for the cache.
-
-    Returns:
-      object, the value in the cache for the given key, or None if the key is
-      not in the cache.
-    """
-    raise NotImplementedError()
-
-  @abc.abstractmethod
-  def set(self, url, content):
-    """Sets the given key and content in the cache.
-
-    Args:
-      url: string, the key for the cache.
-      content: string, the discovery document.
-    """
-    raise NotImplementedError()

src/googleapiclient/discovery_cache/file_cache.py

@@ -1,141 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""File based cache for the discovery document.
-
-The cache is stored in a single file so that multiple processes can
-share the same cache. It locks the file whenever accesing to the
-file. When the cache content is corrupted, it will be initialized with
-an empty cache.
-"""
-
-from __future__ import division
-
-import datetime
-import json
-import logging
-import os
-import tempfile
-import threading
-
-try:
-  from oauth2client.contrib.locked_file import LockedFile
-except ImportError:
-  # oauth2client < 2.0.0
-  try:
-    from oauth2client.locked_file import LockedFile
-  except ImportError:
-    # oauth2client > 4.0.0
-    raise ImportError(
-      'file_cache is unavailable when using oauth2client >= 4.0.0')
-
-from . import base
-from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
-
-LOGGER = logging.getLogger(__name__)
-
-FILENAME = 'google-api-python-client-discovery-doc.cache'
-EPOCH = datetime.datetime.utcfromtimestamp(0)
-
-
-def _to_timestamp(date):
-  try:
-    return (date - EPOCH).total_seconds()
-  except AttributeError:
-    # The following is the equivalent of total_seconds() in Python2.6.
-    # See also: https://docs.python.org/2/library/datetime.html
-    delta = date - EPOCH
-    return ((delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
-             * 10**6) / 10**6)
-
-
-def _read_or_initialize_cache(f):
-  f.file_handle().seek(0)
-  try:
-    cache = json.load(f.file_handle())
-  except Exception:
-    # This means it opens the file for the first time, or the cache is
-    # corrupted, so initializing the file with an empty dict.
-    cache = {}
-    f.file_handle().truncate(0)
-    f.file_handle().seek(0)
-    json.dump(cache, f.file_handle())
-  return cache
-
-
-class Cache(base.Cache):
-  """A file based cache for the discovery documents."""
-
-  def __init__(self, max_age):
-      """Constructor.
-
-      Args:
-        max_age: Cache expiration in seconds.
-      """
-      self._max_age = max_age
-      self._file = os.path.join(tempfile.gettempdir(), FILENAME)
-      f = LockedFile(self._file, 'a+', 'r')
-      try:
-        f.open_and_lock()
-        if f.is_locked():
-          _read_or_initialize_cache(f)
-        # If we can not obtain the lock, other process or thread must
-        # have initialized the file.
-      except Exception as e:
-        LOGGER.warning(e, exc_info=True)
-      finally:
-        f.unlock_and_close()
-
-  def get(self, url):
-    f = LockedFile(self._file, 'r+', 'r')
-    try:
-      f.open_and_lock()
-      if f.is_locked():
-        cache = _read_or_initialize_cache(f)
-        if url in cache:
-          content, t = cache.get(url, (None, 0))
-          if _to_timestamp(datetime.datetime.now()) < t + self._max_age:
-            return content
-        return None
-      else:
-        LOGGER.debug('Could not obtain a lock for the cache file.')
-        return None
-    except Exception as e:
-      LOGGER.warning(e, exc_info=True)
-    finally:
-      f.unlock_and_close()
-
-  def set(self, url, content):
-    f = LockedFile(self._file, 'r+', 'r')
-    try:
-      f.open_and_lock()
-      if f.is_locked():
-        cache = _read_or_initialize_cache(f)
-        cache[url] = (content, _to_timestamp(datetime.datetime.now()))
-        # Remove stale cache.
-        for k, (_, timestamp) in list(cache.items()):
-          if _to_timestamp(datetime.datetime.now()) >= timestamp + self._max_age:
-            del cache[k]
-        f.file_handle().truncate(0)
-        f.file_handle().seek(0)
-        json.dump(cache, f.file_handle())
-      else:
-        LOGGER.debug('Could not obtain a lock for the cache file.')
-    except Exception as e:
-      LOGGER.warning(e, exc_info=True)
-    finally:
-      f.unlock_and_close()
-
-
-cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE)

src/googleapiclient/errors.py

@@ -1,162 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Errors for the library.
-
-All exceptions defined by the library
-should be defined in this file.
-"""
-from __future__ import absolute_import
-
-__author__ = 'jcgregorio@google.com (Joe Gregorio)'
-
-import json
-
-# Oauth2client < 3 has the positional helper in 'util', >= 3 has it
-# in '_helpers'.
-try:
-  from oauth2client import util
-except ImportError:
-  from oauth2client import _helpers as util
-
-
-class Error(Exception):
-  """Base error for this module."""
-  pass
-
-
-class HttpError(Error):
-  """HTTP data was invalid or unexpected."""
-
-  @util.positional(3)
-  def __init__(self, resp, content, uri=None):
-    self.resp = resp
-    if not isinstance(content, bytes):
-        raise TypeError("HTTP content should be bytes")
-    self.content = content
-    self.uri = uri
-    self.error_details = ''
-
-  def _get_reason(self):
-    """Calculate the reason for the error from the response content."""
-    reason = self.resp.reason
-    try:
-      data = json.loads(self.content.decode('utf-8'))
-      if isinstance(data, dict):
-        reason = data['error']['message']
-        if 'details' in data['error']:
-            self.error_details = data['error']['details']
-      elif isinstance(data, list) and len(data) > 0:
-        first_error = data[0]
-        reason = first_error['error']['message']
-        if 'details' in first_error['error']:
-            self.error_details = first_error['error']['details']
-    except (ValueError, KeyError, TypeError):
-      pass
-    if reason is None:
-      reason = ''
-    return reason
-
-  def __repr__(self):
-    reason = self._get_reason()
-    if self.error_details:
-      return '<HttpError %s when requesting %s returned "%s". Details: "%s">' % \
-             (self.resp.status, self.uri, reason.strip(), self.error_details)
-    elif self.uri:
-      return '<HttpError %s when requesting %s returned "%s">' % (
-          self.resp.status, self.uri, self._get_reason().strip())
-    else:
-      return '<HttpError %s "%s">' % (self.resp.status, self._get_reason())
-
-  __str__ = __repr__
-
-
-class InvalidJsonError(Error):
-  """The JSON returned could not be parsed."""
-  pass
-
-
-class UnknownFileType(Error):
-  """File type unknown or unexpected."""
-  pass
-
-
-class UnknownLinkType(Error):
-  """Link type unknown or unexpected."""
-  pass
-
-
-class UnknownApiNameOrVersion(Error):
-  """No API with that name and version exists."""
-  pass
-
-
-class UnacceptableMimeTypeError(Error):
-  """That is an unacceptable mimetype for this operation."""
-  pass
-
-
-class MediaUploadSizeError(Error):
-  """Media is larger than the method can accept."""
-  pass
-
-
-class ResumableUploadError(HttpError):
-  """Error occured during resumable upload."""
-  pass
-
-
-class InvalidChunkSizeError(Error):
-  """The given chunksize is not valid."""
-  pass
-
-class InvalidNotificationError(Error):
-  """The channel Notification is invalid."""
-  pass
-
-class BatchError(HttpError):
-  """Error occured during batch operations."""
-
-  @util.positional(2)
-  def __init__(self, reason, resp=None, content=None):
-    self.resp = resp
-    self.content = content
-    self.reason = reason
-
-  def __repr__(self):
-    if getattr(self.resp, 'status', None) is None:
-      return '<BatchError "%s">' % (self.reason)
-    else:
-      return '<BatchError %s "%s">' % (self.resp.status, self.reason)
-
-  __str__ = __repr__
-
-
-class UnexpectedMethodError(Error):
-  """Exception raised by RequestMockBuilder on unexpected calls."""
-
-  @util.positional(1)
-  def __init__(self, methodId=None):
-    """Constructor for an UnexpectedMethodError."""
-    super(UnexpectedMethodError, self).__init__(
-        'Received unexpected call %s' % methodId)
-
-
-class UnexpectedBodyError(Error):
-  """Exception raised by RequestMockBuilder on unexpected bodies."""
-
-  def __init__(self, expected, provided):
-    """Constructor for an UnexpectedMethodError."""
-    super(UnexpectedBodyError, self).__init__(
-        'Expected: [%s] - Provided: [%s]' % (expected, provided))

src/googleapiclient/mimeparse.py

@@ -1,175 +0,0 @@
-# Copyright 2014 Joe Gregorio
-#
-# Licensed under the MIT License
-
-"""MIME-Type Parser
-
-This module provides basic functions for handling mime-types. It can handle
-matching mime-types against a list of media-ranges. See section 14.1 of the
-HTTP specification [RFC 2616] for a complete explanation.
-
-   http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
-
-Contents:
- - parse_mime_type():   Parses a mime-type into its component parts.
- - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q'
-                          quality parameter.
- - quality():           Determines the quality ('q') of a mime-type when
-                          compared against a list of media-ranges.
- - quality_parsed():    Just like quality() except the second parameter must be
-                          pre-parsed.
- - best_match():        Choose the mime-type with the highest quality ('q')
-                          from a list of candidates.
-"""
-from __future__ import absolute_import
-from functools import reduce
-import six
-
-__version__ = '0.1.3'
-__author__ = 'Joe Gregorio'
-__email__ = 'joe@bitworking.org'
-__license__ = 'MIT License'
-__credits__ = ''
-
-
-def parse_mime_type(mime_type):
-    """Parses a mime-type into its component parts.
-
-    Carves up a mime-type and returns a tuple of the (type, subtype, params)
-    where 'params' is a dictionary of all the parameters for the media range.
-    For example, the media range 'application/xhtml;q=0.5' would get parsed
-    into:
-
-       ('application', 'xhtml', {'q', '0.5'})
-       """
-    parts = mime_type.split(';')
-    params = dict([tuple([s.strip() for s in param.split('=', 1)])\
-            for param in parts[1:]
-                  ])
-    full_type = parts[0].strip()
-    # Java URLConnection class sends an Accept header that includes a
-    # single '*'. Turn it into a legal wildcard.
-    if full_type == '*':
-        full_type = '*/*'
-    (type, subtype) = full_type.split('/')
-
-    return (type.strip(), subtype.strip(), params)
-
-
-def parse_media_range(range):
-    """Parse a media-range into its component parts.
-
-    Carves up a media range and returns a tuple of the (type, subtype,
-    params) where 'params' is a dictionary of all the parameters for the media
-    range.  For example, the media range 'application/*;q=0.5' would get parsed
-    into:
-
-       ('application', '*', {'q', '0.5'})
-
-    In addition this function also guarantees that there is a value for 'q'
-    in the params dictionary, filling it in with a proper default if
-    necessary.
-    """
-    (type, subtype, params) = parse_mime_type(range)
-    if 'q' not in params or not params['q'] or \
-            not float(params['q']) or float(params['q']) > 1\
-            or float(params['q']) < 0:
-        params['q'] = '1'
-
-    return (type, subtype, params)
-
-
-def fitness_and_quality_parsed(mime_type, parsed_ranges):
-    """Find the best match for a mime-type amongst parsed media-ranges.
-
-    Find the best match for a given mime-type against a list of media_ranges
-    that have already been parsed by parse_media_range(). Returns a tuple of
-    the fitness value and the value of the 'q' quality parameter of the best
-    match, or (-1, 0) if no match was found. Just as for quality_parsed(),
-    'parsed_ranges' must be a list of parsed media ranges.
-    """
-    best_fitness = -1
-    best_fit_q = 0
-    (target_type, target_subtype, target_params) =\
-            parse_media_range(mime_type)
-    for (type, subtype, params) in parsed_ranges:
-        type_match = (type == target_type or\
-                      type == '*' or\
-                      target_type == '*')
-        subtype_match = (subtype == target_subtype or\
-                         subtype == '*' or\
-                         target_subtype == '*')
-        if type_match and subtype_match:
-            param_matches = reduce(lambda x, y: x + y, [1 for (key, value) in \
-                    six.iteritems(target_params) if key != 'q' and \
-                    key in params and value == params[key]], 0)
-            fitness = (type == target_type) and 100 or 0
-            fitness += (subtype == target_subtype) and 10 or 0
-            fitness += param_matches
-            if fitness > best_fitness:
-                best_fitness = fitness
-                best_fit_q = params['q']
-
-    return best_fitness, float(best_fit_q)
-
-
-def quality_parsed(mime_type, parsed_ranges):
-    """Find the best match for a mime-type amongst parsed media-ranges.
-
-    Find the best match for a given mime-type against a list of media_ranges
-    that have already been parsed by parse_media_range(). Returns the 'q'
-    quality parameter of the best match, 0 if no match was found. This function
-    bahaves the same as quality() except that 'parsed_ranges' must be a list of
-    parsed media ranges.
-    """
-
-    return fitness_and_quality_parsed(mime_type, parsed_ranges)[1]
-
-
-def quality(mime_type, ranges):
-    """Return the quality ('q') of a mime-type against a list of media-ranges.
-
-    Returns the quality 'q' of a mime-type when compared against the
-    media-ranges in ranges. For example:
-
-    >>> quality('text/html','text/*;q=0.3, text/html;q=0.7,
-                  text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5')
-    0.7
-
-    """
-    parsed_ranges = [parse_media_range(r) for r in ranges.split(',')]
-
-    return quality_parsed(mime_type, parsed_ranges)
-
-
-def best_match(supported, header):
-    """Return mime-type with the highest quality ('q') from list of candidates.
-
-    Takes a list of supported mime-types and finds the best match for all the
-    media-ranges listed in header. The value of header must be a string that
-    conforms to the format of the HTTP Accept: header. The value of 'supported'
-    is a list of mime-types. The list of supported mime-types should be sorted
-    in order of increasing desirability, in case of a situation where there is
-    a tie.
-
-    >>> best_match(['application/xbel+xml', 'text/xml'],
-                   'text/*;q=0.5,*/*; q=0.1')
-    'text/xml'
-    """
-    split_header = _filter_blank(header.split(','))
-    parsed_header = [parse_media_range(r) for r in split_header]
-    weighted_matches = []
-    pos = 0
-    for mime_type in supported:
-        weighted_matches.append((fitness_and_quality_parsed(mime_type,
-                                 parsed_header), pos, mime_type))
-        pos += 1
-    weighted_matches.sort()
-
-    return weighted_matches[-1][0][1] and weighted_matches[-1][2] or ''
-
-
-def _filter_blank(i):
-    for s in i:
-        if s.strip():
-            yield s

src/googleapiclient/model.py

@@ -1,389 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Model objects for requests and responses.
-
-Each API may support one or more serializations, such
-as JSON, Atom, etc. The model classes are responsible
-for converting between the wire format and the Python
-object representation.
-"""
-from __future__ import absolute_import
-import six
-
-__author__ = 'jcgregorio@google.com (Joe Gregorio)'
-
-import json
-import logging
-
-from six.moves.urllib.parse import urlencode
-
-from googleapiclient import __version__
-from googleapiclient.errors import HttpError
-
-
-LOGGER = logging.getLogger(__name__)
-
-dump_request_response = False
-
-
-def _abstract():
-  raise NotImplementedError('You need to override this function')
-
-
-class Model(object):
-  """Model base class.
-
-  All Model classes should implement this interface.
-  The Model serializes and de-serializes between a wire
-  format such as JSON and a Python object representation.
-  """
-
-  def request(self, headers, path_params, query_params, body_value):
-    """Updates outgoing requests with a serialized body.
-
-    Args:
-      headers: dict, request headers
-      path_params: dict, parameters that appear in the request path
-      query_params: dict, parameters that appear in the query
-      body_value: object, the request body as a Python object, which must be
-                  serializable.
-    Returns:
-      A tuple of (headers, path_params, query, body)
-
-      headers: dict, request headers
-      path_params: dict, parameters that appear in the request path
-      query: string, query part of the request URI
-      body: string, the body serialized in the desired wire format.
-    """
-    _abstract()
-
-  def response(self, resp, content):
-    """Convert the response wire format into a Python object.
-
-    Args:
-      resp: httplib2.Response, the HTTP response headers and status
-      content: string, the body of the HTTP response
-
-    Returns:
-      The body de-serialized as a Python object.
-
-    Raises:
-      googleapiclient.errors.HttpError if a non 2xx response is received.
-    """
-    _abstract()
-
-
-class BaseModel(Model):
-  """Base model class.
-
-  Subclasses should provide implementations for the "serialize" and
-  "deserialize" methods, as well as values for the following class attributes.
-
-  Attributes:
-    accept: The value to use for the HTTP Accept header.
-    content_type: The value to use for the HTTP Content-type header.
-    no_content_response: The value to return when deserializing a 204 "No
-        Content" response.
-    alt_param: The value to supply as the "alt" query parameter for requests.
-  """
-
-  accept = None
-  content_type = None
-  no_content_response = None
-  alt_param = None
-
-  def _log_request(self, headers, path_params, query, body):
-    """Logs debugging information about the request if requested."""
-    if dump_request_response:
-      LOGGER.info('--request-start--')
-      LOGGER.info('-headers-start-')
-      for h, v in six.iteritems(headers):
-        LOGGER.info('%s: %s', h, v)
-      LOGGER.info('-headers-end-')
-      LOGGER.info('-path-parameters-start-')
-      for h, v in six.iteritems(path_params):
-        LOGGER.info('%s: %s', h, v)
-      LOGGER.info('-path-parameters-end-')
-      LOGGER.info('body: %s', body)
-      LOGGER.info('query: %s', query)
-      LOGGER.info('--request-end--')
-
-  def request(self, headers, path_params, query_params, body_value):
-    """Updates outgoing requests with a serialized body.
-
-    Args:
-      headers: dict, request headers
-      path_params: dict, parameters that appear in the request path
-      query_params: dict, parameters that appear in the query
-      body_value: object, the request body as a Python object, which must be
-                  serializable by json.
-    Returns:
-      A tuple of (headers, path_params, query, body)
-
-      headers: dict, request headers
-      path_params: dict, parameters that appear in the request path
-      query: string, query part of the request URI
-      body: string, the body serialized as JSON
-    """
-    query = self._build_query(query_params)
-    headers['accept'] = self.accept
-    headers['accept-encoding'] = 'gzip, deflate'
-    if 'user-agent' in headers:
-      headers['user-agent'] += ' '
-    else:
-      headers['user-agent'] = ''
-    headers['user-agent'] += 'google-api-python-client/%s (gzip)' % __version__
-
-    if body_value is not None:
-      headers['content-type'] = self.content_type
-      body_value = self.serialize(body_value)
-    self._log_request(headers, path_params, query, body_value)
-    return (headers, path_params, query, body_value)
-
-  def _build_query(self, params):
-    """Builds a query string.
-
-    Args:
-      params: dict, the query parameters
-
-    Returns:
-      The query parameters properly encoded into an HTTP URI query string.
-    """
-    if self.alt_param is not None:
-      params.update({'alt': self.alt_param})
-    astuples = []
-    for key, value in six.iteritems(params):
-      if type(value) == type([]):
-        for x in value:
-          x = x.encode('utf-8')
-          astuples.append((key, x))
-      else:
-        if isinstance(value, six.text_type) and callable(value.encode):
-          value = value.encode('utf-8')
-        astuples.append((key, value))
-    return '?' + urlencode(astuples)
-
-  def _log_response(self, resp, content):
-    """Logs debugging information about the response if requested."""
-    if dump_request_response:
-      LOGGER.info('--response-start--')
-      for h, v in six.iteritems(resp):
-        LOGGER.info('%s: %s', h, v)
-      if content:
-        LOGGER.info(content)
-      LOGGER.info('--response-end--')
-
-  def response(self, resp, content):
-    """Convert the response wire format into a Python object.
-
-    Args:
-      resp: httplib2.Response, the HTTP response headers and status
-      content: string, the body of the HTTP response
-
-    Returns:
-      The body de-serialized as a Python object.
-
-    Raises:
-      googleapiclient.errors.HttpError if a non 2xx response is received.
-    """
-    self._log_response(resp, content)
-    # Error handling is TBD, for example, do we retry
-    # for some operation/error combinations?
-    if resp.status < 300:
-      if resp.status == 204:
-        # A 204: No Content response should be treated differently
-        # to all the other success states
-        return self.no_content_response
-      return self.deserialize(content)
-    else:
-      LOGGER.debug('Content from bad request was: %s' % content)
-      raise HttpError(resp, content)
-
-  def serialize(self, body_value):
-    """Perform the actual Python object serialization.
-
-    Args:
-      body_value: object, the request body as a Python object.
-
-    Returns:
-      string, the body in serialized form.
-    """
-    _abstract()
-
-  def deserialize(self, content):
-    """Perform the actual deserialization from response string to Python
-    object.
-
-    Args:
-      content: string, the body of the HTTP response
-
-    Returns:
-      The body de-serialized as a Python object.
-    """
-    _abstract()
-
-
-class JsonModel(BaseModel):
-  """Model class for JSON.
-
-  Serializes and de-serializes between JSON and the Python
-  object representation of HTTP request and response bodies.
-  """
-  accept = 'application/json'
-  content_type = 'application/json'
-  alt_param = 'json'
-
-  def __init__(self, data_wrapper=False):
-    """Construct a JsonModel.
-
-    Args:
-      data_wrapper: boolean, wrap requests and responses in a data wrapper
-    """
-    self._data_wrapper = data_wrapper
-
-  def serialize(self, body_value):
-    if (isinstance(body_value, dict) and 'data' not in body_value and
-        self._data_wrapper):
-      body_value = {'data': body_value}
-    return json.dumps(body_value)
-
-  def deserialize(self, content):
-    try:
-        content = content.decode('utf-8')
-    except AttributeError:
-        pass
-    body = json.loads(content)
-    if self._data_wrapper and isinstance(body, dict) and 'data' in body:
-      body = body['data']
-    return body
-
-  @property
-  def no_content_response(self):
-    return {}
-
-
-class RawModel(JsonModel):
-  """Model class for requests that don't return JSON.
-
-  Serializes and de-serializes between JSON and the Python
-  object representation of HTTP request, and returns the raw bytes
-  of the response body.
-  """
-  accept = '*/*'
-  content_type = 'application/json'
-  alt_param = None
-
-  def deserialize(self, content):
-    return content
-
-  @property
-  def no_content_response(self):
-    return ''
-
-
-class MediaModel(JsonModel):
-  """Model class for requests that return Media.
-
-  Serializes and de-serializes between JSON and the Python
-  object representation of HTTP request, and returns the raw bytes
-  of the response body.
-  """
-  accept = '*/*'
-  content_type = 'application/json'
-  alt_param = 'media'
-
-  def deserialize(self, content):
-    return content
-
-  @property
-  def no_content_response(self):
-    return ''
-
-
-class ProtocolBufferModel(BaseModel):
-  """Model class for protocol buffers.
-
-  Serializes and de-serializes the binary protocol buffer sent in the HTTP
-  request and response bodies.
-  """
-  accept = 'application/x-protobuf'
-  content_type = 'application/x-protobuf'
-  alt_param = 'proto'
-
-  def __init__(self, protocol_buffer):
-    """Constructs a ProtocolBufferModel.
-
-    The serialzed protocol buffer returned in an HTTP response will be
-    de-serialized using the given protocol buffer class.
-
-    Args:
-      protocol_buffer: The protocol buffer class used to de-serialize a
-      response from the API.
-    """
-    self._protocol_buffer = protocol_buffer
-
-  def serialize(self, body_value):
-    return body_value.SerializeToString()
-
-  def deserialize(self, content):
-    return self._protocol_buffer.FromString(content)
-
-  @property
-  def no_content_response(self):
-    return self._protocol_buffer()
-
-
-def makepatch(original, modified):
-  """Create a patch object.
-
-  Some methods support PATCH, an efficient way to send updates to a resource.
-  This method allows the easy construction of patch bodies by looking at the
-  differences between a resource before and after it was modified.
-
-  Args:
-    original: object, the original deserialized resource
-    modified: object, the modified deserialized resource
-  Returns:
-    An object that contains only the changes from original to modified, in a
-    form suitable to pass to a PATCH method.
-
-  Example usage:
-    item = service.activities().get(postid=postid, userid=userid).execute()
-    original = copy.deepcopy(item)
-    item['object']['content'] = 'This is updated.'
-    service.activities.patch(postid=postid, userid=userid,
-      body=makepatch(original, item)).execute()
-  """
-  patch = {}
-  for key, original_value in six.iteritems(original):
-    modified_value = modified.get(key, None)
-    if modified_value is None:
-      # Use None to signal that the element is deleted
-      patch[key] = None
-    elif original_value != modified_value:
-      if type(original_value) == type({}):
-        # Recursively descend objects
-        patch[key] = makepatch(original_value, modified_value)
-      else:
-        # In the case of simple types or arrays we just replace
-        patch[key] = modified_value
-    else:
-      # Don't add anything to patch if there's no change
-      pass
-  for key in modified:
-    if key not in original:
-      patch[key] = modified[key]
-
-  return patch

src/googleapiclient/sample_tools.py

@@ -1,103 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for making samples.
-
-Consolidates a lot of code commonly repeated in sample applications.
-"""
-from __future__ import absolute_import
-
-__author__ = 'jcgregorio@google.com (Joe Gregorio)'
-__all__ = ['init']
-
-
-import argparse
-import os
-
-from googleapiclient import discovery
-from googleapiclient.http import build_http
-from oauth2client import client
-from oauth2client import file
-from oauth2client import tools
-
-
-def init(argv, name, version, doc, filename, scope=None, parents=[], discovery_filename=None):
-  """A common initialization routine for samples.
-
-  Many of the sample applications do the same initialization, which has now
-  been consolidated into this function. This function uses common idioms found
-  in almost all the samples, i.e. for an API with name 'apiname', the
-  credentials are stored in a file named apiname.dat, and the
-  client_secrets.json file is stored in the same directory as the application
-  main file.
-
-  Args:
-    argv: list of string, the command-line parameters of the application.
-    name: string, name of the API.
-    version: string, version of the API.
-    doc: string, description of the application. Usually set to __doc__.
-    file: string, filename of the application. Usually set to __file__.
-    parents: list of argparse.ArgumentParser, additional command-line flags.
-    scope: string, The OAuth scope used.
-    discovery_filename: string, name of local discovery file (JSON). Use when discovery doc not available via URL.
-
-  Returns:
-    A tuple of (service, flags), where service is the service object and flags
-    is the parsed command-line flags.
-  """
-  if scope is None:
-    scope = 'https://www.googleapis.com/auth/' + name
-
-  # Parser command-line arguments.
-  parent_parsers = [tools.argparser]
-  parent_parsers.extend(parents)
-  parser = argparse.ArgumentParser(
-      description=doc,
-      formatter_class=argparse.RawDescriptionHelpFormatter,
-      parents=parent_parsers)
-  flags = parser.parse_args(argv[1:])
-
-  # Name of a file containing the OAuth 2.0 information for this
-  # application, including client_id and client_secret, which are found
-  # on the API Access tab on the Google APIs
-  # Console <http://code.google.com/apis/console>.
-  client_secrets = os.path.join(os.path.dirname(filename),
-                                'client_secrets.json')
-
-  # Set up a Flow object to be used if we need to authenticate.
-  flow = client.flow_from_clientsecrets(client_secrets,
-      scope=scope,
-      message=tools.message_if_missing(client_secrets))
-
-  # Prepare credentials, and authorize HTTP object with them.
-  # If the credentials don't exist or are invalid run through the native client
-  # flow. The Storage object will ensure that if successful the good
-  # credentials will get written back to a file.
-  storage = file.Storage(name + '.dat')
-  credentials = storage.get()
-  if credentials is None or credentials.invalid:
-    credentials = tools.run_flow(flow, storage, flags)
-  http = credentials.authorize(http=build_http())
-
-  if discovery_filename is None:
-    # Construct a service object via the discovery service.
-    service = discovery.build(name, version, http=http)
-  else:
-    # Construct a service object using a local discovery document file.
-    with open(discovery_filename) as discovery_file:
-      service = discovery.build_from_document(
-          discovery_file.read(),
-          base='https://www.googleapis.com/',
-          http=http)
-  return (service, flags)

src/googleapiclient/schema.py

@@ -1,319 +0,0 @@
-# Copyright 2014 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Schema processing for discovery based APIs
-
-Schemas holds an APIs discovery schemas. It can return those schema as
-deserialized JSON objects, or pretty print them as prototype objects that
-conform to the schema.
-
-For example, given the schema:
-
- schema = \"\"\"{
-   "Foo": {
-    "type": "object",
-    "properties": {
-     "etag": {
-      "type": "string",
-      "description": "ETag of the collection."
-     },
-     "kind": {
-      "type": "string",
-      "description": "Type of the collection ('calendar#acl').",
-      "default": "calendar#acl"
-     },
-     "nextPageToken": {
-      "type": "string",
-      "description": "Token used to access the next
-         page of this result. Omitted if no further results are available."
-     }
-    }
-   }
- }\"\"\"
-
- s = Schemas(schema)
- print s.prettyPrintByName('Foo')
-
- Produces the following output:
-
-  {
-   "nextPageToken": "A String", # Token used to access the
-       # next page of this result. Omitted if no further results are available.
-   "kind": "A String", # Type of the collection ('calendar#acl').
-   "etag": "A String", # ETag of the collection.
-  },
-
-The constructor takes a discovery document in which to look up named schema.
-"""
-from __future__ import absolute_import
-import six
-
-# TODO(jcgregorio) support format, enum, minimum, maximum
-
-__author__ = 'jcgregorio@google.com (Joe Gregorio)'
-
-import copy
-
-# Oauth2client < 3 has the positional helper in 'util', >= 3 has it
-# in '_helpers'.
-try:
-  from oauth2client import util
-except ImportError:
-  from oauth2client import _helpers as util
-
-
-class Schemas(object):
-  """Schemas for an API."""
-
-  def __init__(self, discovery):
-    """Constructor.
-
-    Args:
-      discovery: object, Deserialized discovery document from which we pull
-        out the named schema.
-    """
-    self.schemas = discovery.get('schemas', {})
-
-    # Cache of pretty printed schemas.
-    self.pretty = {}
-
-  @util.positional(2)
-  def _prettyPrintByName(self, name, seen=None, dent=0):
-    """Get pretty printed object prototype from the schema name.
-
-    Args:
-      name: string, Name of schema in the discovery document.
-      seen: list of string, Names of schema already seen. Used to handle
-        recursive definitions.
-
-    Returns:
-      string, A string that contains a prototype object with
-        comments that conforms to the given schema.
-    """
-    if seen is None:
-      seen = []
-
-    if name in seen:
-      # Do not fall into an infinite loop over recursive definitions.
-      return '# Object with schema name: %s' % name
-    seen.append(name)
-
-    if name not in self.pretty:
-      self.pretty[name] = _SchemaToStruct(self.schemas[name],
-          seen, dent=dent).to_str(self._prettyPrintByName)
-
-    seen.pop()
-
-    return self.pretty[name]
-
-  def prettyPrintByName(self, name):
-    """Get pretty printed object prototype from the schema name.
-
-    Args:
-      name: string, Name of schema in the discovery document.
-
-    Returns:
-      string, A string that contains a prototype object with
-        comments that conforms to the given schema.
-    """
-    # Return with trailing comma and newline removed.
-    return self._prettyPrintByName(name, seen=[], dent=1)[:-2]
-
-  @util.positional(2)
-  def _prettyPrintSchema(self, schema, seen=None, dent=0):
-    """Get pretty printed object prototype of schema.
-
-    Args:
-      schema: object, Parsed JSON schema.
-      seen: list of string, Names of schema already seen. Used to handle
-        recursive definitions.
-
-    Returns:
-      string, A string that contains a prototype object with
-        comments that conforms to the given schema.
-    """
-    if seen is None:
-      seen = []
-
-    return _SchemaToStruct(schema, seen, dent=dent).to_str(self._prettyPrintByName)
-
-  def prettyPrintSchema(self, schema):
-    """Get pretty printed object prototype of schema.
-
-    Args:
-      schema: object, Parsed JSON schema.
-
-    Returns:
-      string, A string that contains a prototype object with
-        comments that conforms to the given schema.
-    """
-    # Return with trailing comma and newline removed.
-    return self._prettyPrintSchema(schema, dent=1)[:-2]
-
-  def get(self, name, default=None):
-    """Get deserialized JSON schema from the schema name.
-
-    Args:
-      name: string, Schema name.
-      default: object, return value if name not found.
-    """
-    return self.schemas.get(name, default)
-
-
-class _SchemaToStruct(object):
-  """Convert schema to a prototype object."""
-
-  @util.positional(3)
-  def __init__(self, schema, seen, dent=0):
-    """Constructor.
-
-    Args:
-      schema: object, Parsed JSON schema.
-      seen: list, List of names of schema already seen while parsing. Used to
-        handle recursive definitions.
-      dent: int, Initial indentation depth.
-    """
-    # The result of this parsing kept as list of strings.
-    self.value = []
-
-    # The final value of the parsing.
-    self.string = None
-
-    # The parsed JSON schema.
-    self.schema = schema
-
-    # Indentation level.
-    self.dent = dent
-
-    # Method that when called returns a prototype object for the schema with
-    # the given name.
-    self.from_cache = None
-
-    # List of names of schema already seen while parsing.
-    self.seen = seen
-
-  def emit(self, text):
-    """Add text as a line to the output.
-
-    Args:
-      text: string, Text to output.
-    """
-    self.value.extend(["  " * self.dent, text, '\n'])
-
-  def emitBegin(self, text):
-    """Add text to the output, but with no line terminator.
-
-    Args:
-      text: string, Text to output.
-      """
-    self.value.extend(["  " * self.dent, text])
-
-  def emitEnd(self, text, comment):
-    """Add text and comment to the output with line terminator.
-
-    Args:
-      text: string, Text to output.
-      comment: string, Python comment.
-    """
-    if comment:
-      divider = '\n' + '  ' * (self.dent + 2) + '# '
-      lines = comment.splitlines()
-      lines = [x.rstrip() for x in lines]
-      comment = divider.join(lines)
-      self.value.extend([text, ' # ', comment, '\n'])
-    else:
-      self.value.extend([text, '\n'])
-
-  def indent(self):
-    """Increase indentation level."""
-    self.dent += 1
-
-  def undent(self):
-    """Decrease indentation level."""
-    self.dent -= 1
-
-  def _to_str_impl(self, schema):
-    """Prototype object based on the schema, in Python code with comments.
-
-    Args:
-      schema: object, Parsed JSON schema file.
-
-    Returns:
-      Prototype object based on the schema, in Python code with comments.
-    """
-    stype = schema.get('type')
-    if stype == 'object':
-      self.emitEnd('{', schema.get('description', ''))
-      self.indent()
-      if 'properties' in schema:
-        for pname, pschema in six.iteritems(schema.get('properties', {})):
-          self.emitBegin('"%s": ' % pname)
-          self._to_str_impl(pschema)
-      elif 'additionalProperties' in schema:
-        self.emitBegin('"a_key": ')
-        self._to_str_impl(schema['additionalProperties'])
-      self.undent()
-      self.emit('},')
-    elif '$ref' in schema:
-      schemaName = schema['$ref']
-      description = schema.get('description', '')
-      s = self.from_cache(schemaName, seen=self.seen)
-      parts = s.splitlines()
-      self.emitEnd(parts[0], description)
-      for line in parts[1:]:
-        self.emit(line.rstrip())
-    elif stype == 'boolean':
-      value = schema.get('default', 'True or False')
-      self.emitEnd('%s,' % str(value), schema.get('description', ''))
-    elif stype == 'string':
-      value = schema.get('default', 'A String')
-      self.emitEnd('"%s",' % str(value), schema.get('description', ''))
-    elif stype == 'integer':
-      value = schema.get('default', '42')
-      self.emitEnd('%s,' % str(value), schema.get('description', ''))
-    elif stype == 'number':
-      value = schema.get('default', '3.14')
-      self.emitEnd('%s,' % str(value), schema.get('description', ''))
-    elif stype == 'null':
-      self.emitEnd('None,', schema.get('description', ''))
-    elif stype == 'any':
-      self.emitEnd('"",', schema.get('description', ''))
-    elif stype == 'array':
-      self.emitEnd('[', schema.get('description'))
-      self.indent()
-      self.emitBegin('')
-      self._to_str_impl(schema['items'])
-      self.undent()
-      self.emit('],')
-    else:
-      self.emit('Unknown type! %s' % stype)
-      self.emitEnd('', '')
-
-    self.string = ''.join(self.value)
-    return self.string
-
-  def to_str(self, from_cache):
-    """Prototype object based on the schema, in Python code with comments.
-
-    Args:
-      from_cache: callable(name, seen), Callable that retrieves an object
-         prototype for a schema with the given name. Seen is a list of schema
-         names already seen as we recursively descend the schema definition.
-
-    Returns:
-      Prototype object based on the schema, in Python code with comments.
-      The lines of the code will all be properly indented.
-    """
-    self.from_cache = from_cache
-    return self._to_str_impl(self.schema)

src/httplib2/iri2uri.py

@@ -1,110 +0,0 @@
-"""
-iri2uri
-
-Converts an IRI to a URI.
-
-"""
-__author__ = "Joe Gregorio (joe@bitworking.org)"
-__copyright__ = "Copyright 2006, Joe Gregorio"
-__contributors__ = []
-__version__ = "1.0.0"
-__license__ = "MIT"
-__history__ = """
-"""
-
-import urlparse
-
-
-# Convert an IRI to a URI following the rules in RFC 3987
-#
-# The characters we need to enocde and escape are defined in the spec:
-#
-# iprivate =  %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD
-# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF
-#         / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD
-#         / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD
-#         / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD
-#         / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD
-#         / %xD0000-DFFFD / %xE1000-EFFFD
-
-escape_range = [
-    (0xA0, 0xD7FF),
-    (0xE000, 0xF8FF),
-    (0xF900, 0xFDCF),
-    (0xFDF0, 0xFFEF),
-    (0x10000, 0x1FFFD),
-    (0x20000, 0x2FFFD),
-    (0x30000, 0x3FFFD),
-    (0x40000, 0x4FFFD),
-    (0x50000, 0x5FFFD),
-    (0x60000, 0x6FFFD),
-    (0x70000, 0x7FFFD),
-    (0x80000, 0x8FFFD),
-    (0x90000, 0x9FFFD),
-    (0xA0000, 0xAFFFD),
-    (0xB0000, 0xBFFFD),
-    (0xC0000, 0xCFFFD),
-    (0xD0000, 0xDFFFD),
-    (0xE1000, 0xEFFFD),
-    (0xF0000, 0xFFFFD),
-    (0x100000, 0x10FFFD),
-]
-
-def encode(c):
-    retval = c
-    i = ord(c)
-    for low, high in escape_range:
-        if i < low:
-            break
-        if i >= low and i <= high:
-            retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')])
-            break
-    return retval
-
-
-def iri2uri(uri):
-    """Convert an IRI to a URI. Note that IRIs must be
-    passed in a unicode strings. That is, do not utf-8 encode
-    the IRI before passing it into the function."""
-    if isinstance(uri ,unicode):
-        (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
-        authority = authority.encode('idna')
-        # For each character in 'ucschar' or 'iprivate'
-        #  1. encode as utf-8
-        #  2. then %-encode each octet of that utf-8
-        uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
-        uri = "".join([encode(c) for c in uri])
-    return uri
-
-if __name__ == "__main__":
-    import unittest
-
-    class Test(unittest.TestCase):
-
-        def test_uris(self):
-            """Test that URIs are invariant under the transformation."""
-            invariant = [
-                u"ftp://ftp.is.co.za/rfc/rfc1808.txt",
-                u"http://www.ietf.org/rfc/rfc2396.txt",
-                u"ldap://[2001:db8::7]/c=GB?objectClass?one",
-                u"mailto:John.Doe@example.com",
-                u"news:comp.infosystems.www.servers.unix",
-                u"tel:+1-816-555-1212",
-                u"telnet://192.0.2.16:80/",
-                u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ]
-            for uri in invariant:
-                self.assertEqual(uri, iri2uri(uri))
-
-        def test_iri(self):
-            """ Test that the right type of escaping is done for each part of the URI."""
-            self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}"))
-            self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}"))
-            self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}"))
-            self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}"))
-            self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))
-            self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")))
-            self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8')))
-
-    unittest.main()
-
-

src/httplib2/socks.py

@@ -1,448 +0,0 @@
-"""SocksiPy - Python SOCKS module.
-Version 1.00
-
-Copyright 2006 Dan-Haim. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-3. Neither the name of Dan Haim nor the names of his contributors may be used
-   to endorse or promote products derived from this software without specific
-   prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED
-WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA
-OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
-
-
-This module provides a standard socket-like interface for Python
-for tunneling connections through SOCKS proxies.
-
-"""
-
-"""
-
-Minor modifications made by Christopher Gilbert (http://motomastyle.com/)
-for use in PyLoris (http://pyloris.sourceforge.net/)
-
-Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/)
-mainly to merge bug fixes found in Sourceforge
-
-"""
-
-import base64
-import socket
-import struct
-import sys
-
-if getattr(socket, 'socket', None) is None:
-    raise ImportError('socket.socket missing, proxy support unusable')
-
-PROXY_TYPE_SOCKS4 = 1
-PROXY_TYPE_SOCKS5 = 2
-PROXY_TYPE_HTTP = 3
-PROXY_TYPE_HTTP_NO_TUNNEL = 4
-
-_defaultproxy = None
-_orgsocket = socket.socket
-
-class ProxyError(Exception): pass
-class GeneralProxyError(ProxyError): pass
-class Socks5AuthError(ProxyError): pass
-class Socks5Error(ProxyError): pass
-class Socks4Error(ProxyError): pass
-class HTTPError(ProxyError): pass
-
-_generalerrors = ("success",
-    "invalid data",
-    "not connected",
-    "not available",
-    "bad proxy type",
-    "bad input")
-
-_socks5errors = ("succeeded",
-    "general SOCKS server failure",
-    "connection not allowed by ruleset",
-    "Network unreachable",
-    "Host unreachable",
-    "Connection refused",
-    "TTL expired",
-    "Command not supported",
-    "Address type not supported",
-    "Unknown error")
-
-_socks5autherrors = ("succeeded",
-    "authentication is required",
-    "all offered authentication methods were rejected",
-    "unknown username or invalid password",
-    "unknown error")
-
-_socks4errors = ("request granted",
-    "request rejected or failed",
-    "request rejected because SOCKS server cannot connect to identd on the client",
-    "request rejected because the client program and identd report different user-ids",
-    "unknown error")
-
-def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
-    """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
-    Sets a default proxy which all further socksocket objects will use,
-    unless explicitly changed.
-    """
-    global _defaultproxy
-    _defaultproxy = (proxytype, addr, port, rdns, username, password)
-
-def wrapmodule(module):
-    """wrapmodule(module)
-    Attempts to replace a module's socket library with a SOCKS socket. Must set
-    a default proxy using setdefaultproxy(...) first.
-    This will only work on modules that import socket directly into the namespace;
-    most of the Python Standard Library falls into this category.
-    """
-    if _defaultproxy != None:
-        module.socket.socket = socksocket
-    else:
-        raise GeneralProxyError((4, "no proxy specified"))
-
-class socksocket(socket.socket):
-    """socksocket([family[, type[, proto]]]) -> socket object
-    Open a SOCKS enabled socket. The parameters are the same as
-    those of the standard socket init. In order for SOCKS to work,
-    you must specify family=AF_INET, type=SOCK_STREAM and proto=0.
-    """
-
-    def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None):
-        _orgsocket.__init__(self, family, type, proto, _sock)
-        if _defaultproxy != None:
-            self.__proxy = _defaultproxy
-        else:
-            self.__proxy = (None, None, None, None, None, None)
-        self.__proxysockname = None
-        self.__proxypeername = None
-        self.__httptunnel = True
-
-    def __recvall(self, count):
-        """__recvall(count) -> data
-        Receive EXACTLY the number of bytes requested from the socket.
-        Blocks until the required number of bytes have been received.
-        """
-        data = self.recv(count)
-        while len(data) < count:
-            d = self.recv(count-len(data))
-            if not d: raise GeneralProxyError((0, "connection closed unexpectedly"))
-            data = data + d
-        return data
-
-    def sendall(self, content, *args):
-        """ override socket.socket.sendall method to rewrite the header
-        for non-tunneling proxies if needed
-        """
-        if not self.__httptunnel:
-            content = self.__rewriteproxy(content)
-        return super(socksocket, self).sendall(content, *args)
-
-    def __rewriteproxy(self, header):
-        """ rewrite HTTP request headers to support non-tunneling proxies
-        (i.e. those which do not support the CONNECT method).
-        This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
-        """
-        host, endpt = None, None
-        hdrs = header.split("\r\n")
-        for hdr in hdrs:
-            if hdr.lower().startswith("host:"):
-                host = hdr
-            elif hdr.lower().startswith("get") or hdr.lower().startswith("post"):
-                endpt = hdr
-        if host and endpt:
-            hdrs.remove(host)
-            hdrs.remove(endpt)
-            host = host.split(" ")[1]
-            endpt = endpt.split(" ")
-            if (self.__proxy[4] != None and self.__proxy[5] != None):
-                hdrs.insert(0, self.__getauthheader())
-            hdrs.insert(0, "Host: %s" % host)
-            hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
-        return "\r\n".join(hdrs)
-
-    def __getauthheader(self):
-        auth = self.__proxy[4] + ":" + self.__proxy[5]
-        return "Proxy-Authorization: Basic " + base64.b64encode(auth)
-
-    def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None, headers=None):
-        """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
-        Sets the proxy to be used.
-        proxytype -    The type of the proxy to be used. Three types
-                are supported: PROXY_TYPE_SOCKS4 (including socks4a),
-                PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
-        addr -        The address of the server (IP or DNS).
-        port -        The port of the server. Defaults to 1080 for SOCKS
-                servers and 8080 for HTTP proxy servers.
-        rdns -        Should DNS queries be preformed on the remote side
-                (rather than the local side). The default is True.
-                Note: This has no effect with SOCKS4 servers.
-        username -    Username to authenticate with to the server.
-                The default is no authentication.
-        password -    Password to authenticate with to the server.
-                Only relevant when username is also provided.
-        headers -     Additional or modified headers for the proxy connect request.
-        """
-        self.__proxy = (proxytype, addr, port, rdns, username, password, headers)
-
-    def __negotiatesocks5(self, destaddr, destport):
-        """__negotiatesocks5(self,destaddr,destport)
-        Negotiates a connection through a SOCKS5 server.
-        """
-        # First we'll send the authentication packages we support.
-        if (self.__proxy[4]!=None) and (self.__proxy[5]!=None):
-            # The username/password details were supplied to the
-            # setproxy method so we support the USERNAME/PASSWORD
-            # authentication (in addition to the standard none).
-            self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02))
-        else:
-            # No username/password were entered, therefore we
-            # only support connections with no authentication.
-            self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00))
-        # We'll receive the server's response to determine which
-        # method was selected
-        chosenauth = self.__recvall(2)
-        if chosenauth[0:1] != chr(0x05).encode():
-            self.close()
-            raise GeneralProxyError((1, _generalerrors[1]))
-        # Check the chosen authentication method
-        if chosenauth[1:2] == chr(0x00).encode():
-            # No authentication is required
-            pass
-        elif chosenauth[1:2] == chr(0x02).encode():
-            # Okay, we need to perform a basic username/password
-            # authentication.
-            self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5])
-            authstat = self.__recvall(2)
-            if authstat[0:1] != chr(0x01).encode():
-                # Bad response
-                self.close()
-                raise GeneralProxyError((1, _generalerrors[1]))
-            if authstat[1:2] != chr(0x00).encode():
-                # Authentication failed
-                self.close()
-                raise Socks5AuthError((3, _socks5autherrors[3]))
-            # Authentication succeeded
-        else:
-            # Reaching here is always bad
-            self.close()
-            if chosenauth[1] == chr(0xFF).encode():
-                raise Socks5AuthError((2, _socks5autherrors[2]))
-            else:
-                raise GeneralProxyError((1, _generalerrors[1]))
-        # Now we can request the actual connection
-        req = struct.pack('BBB', 0x05, 0x01, 0x00)
-        # If the given destination address is an IP address, we'll
-        # use the IPv4 address request even if remote resolving was specified.
-        try:
-            ipaddr = socket.inet_aton(destaddr)
-            req = req + chr(0x01).encode() + ipaddr
-        except socket.error:
-            # Well it's not an IP number,  so it's probably a DNS name.
-            if self.__proxy[3]:
-                # Resolve remotely
-                ipaddr = None
-                req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr
-            else:
-                # Resolve locally
-                ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
-                req = req + chr(0x01).encode() + ipaddr
-        req = req + struct.pack(">H", destport)
-        self.sendall(req)
-        # Get the response
-        resp = self.__recvall(4)
-        if resp[0:1] != chr(0x05).encode():
-            self.close()
-            raise GeneralProxyError((1, _generalerrors[1]))
-        elif resp[1:2] != chr(0x00).encode():
-            # Connection failed
-            self.close()
-            if ord(resp[1:2])<=8:
-                raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
-            else:
-                raise Socks5Error((9, _socks5errors[9]))
-        # Get the bound address/port
-        elif resp[3:4] == chr(0x01).encode():
-            boundaddr = self.__recvall(4)
-        elif resp[3:4] == chr(0x03).encode():
-            resp = resp + self.recv(1)
-            boundaddr = self.__recvall(ord(resp[4:5]))
-        else:
-            self.close()
-            raise GeneralProxyError((1,_generalerrors[1]))
-        boundport = struct.unpack(">H", self.__recvall(2))[0]
-        self.__proxysockname = (boundaddr, boundport)
-        if ipaddr != None:
-            self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
-        else:
-            self.__proxypeername = (destaddr, destport)
-
-    def getproxysockname(self):
-        """getsockname() -> address info
-        Returns the bound IP address and port number at the proxy.
-        """
-        return self.__proxysockname
-
-    def getproxypeername(self):
-        """getproxypeername() -> address info
-        Returns the IP and port number of the proxy.
-        """
-        return _orgsocket.getpeername(self)
-
-    def getpeername(self):
-        """getpeername() -> address info
-        Returns the IP address and port number of the destination
-        machine (note: getproxypeername returns the proxy)
-        """
-        return self.__proxypeername
-
-    def __negotiatesocks4(self,destaddr,destport):
-        """__negotiatesocks4(self,destaddr,destport)
-        Negotiates a connection through a SOCKS4 server.
-        """
-        # Check if the destination address provided is an IP address
-        rmtrslv = False
-        try:
-            ipaddr = socket.inet_aton(destaddr)
-        except socket.error:
-            # It's a DNS name. Check where it should be resolved.
-            if self.__proxy[3]:
-                ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01)
-                rmtrslv = True
-            else:
-                ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
-        # Construct the request packet
-        req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr
-        # The username parameter is considered userid for SOCKS4
-        if self.__proxy[4] != None:
-            req = req + self.__proxy[4]
-        req = req + chr(0x00).encode()
-        # DNS name if remote resolving is required
-        # NOTE: This is actually an extension to the SOCKS4 protocol
-        # called SOCKS4A and may not be supported in all cases.
-        if rmtrslv:
-            req = req + destaddr + chr(0x00).encode()
-        self.sendall(req)
-        # Get the response from the server
-        resp = self.__recvall(8)
-        if resp[0:1] != chr(0x00).encode():
-            # Bad data
-            self.close()
-            raise GeneralProxyError((1,_generalerrors[1]))
-        if resp[1:2] != chr(0x5A).encode():
-            # Server returned an error
-            self.close()
-            if ord(resp[1:2]) in (91, 92, 93):
-                self.close()
-                raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90]))
-            else:
-                raise Socks4Error((94, _socks4errors[4]))
-        # Get the bound address/port
-        self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0])
-        if rmtrslv != None:
-            self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
-        else:
-            self.__proxypeername = (destaddr, destport)
-
-    def __negotiatehttp(self, destaddr, destport):
-        """__negotiatehttp(self,destaddr,destport)
-        Negotiates a connection through an HTTP server.
-        """
-        # If we need to resolve locally, we do this now
-        if not self.__proxy[3]:
-            addr = socket.gethostbyname(destaddr)
-        else:
-            addr = destaddr
-        headers =  ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"]
-        wrote_host_header = False
-        wrote_auth_header = False
-        if self.__proxy[6] != None:
-            for key, val in self.__proxy[6].iteritems():
-                headers += [key, ": ", val, "\r\n"]
-                wrote_host_header = (key.lower() == "host")
-                wrote_auth_header = (key.lower() == "proxy-authorization")
-        if not wrote_host_header:
-            headers += ["Host: ", destaddr, "\r\n"]
-        if not wrote_auth_header:
-            if (self.__proxy[4] != None and self.__proxy[5] != None):
-                headers += [self.__getauthheader(), "\r\n"]
-        headers.append("\r\n")
-        self.sendall("".join(headers).encode())
-        # We read the response until we get the string "\r\n\r\n"
-        resp = self.recv(1)
-        while resp.find("\r\n\r\n".encode()) == -1:
-            resp = resp + self.recv(1)
-        # We just need the first line to check if the connection
-        # was successful
-        statusline = resp.splitlines()[0].split(" ".encode(), 2)
-        if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()):
-            self.close()
-            raise GeneralProxyError((1, _generalerrors[1]))
-        try:
-            statuscode = int(statusline[1])
-        except ValueError:
-            self.close()
-            raise GeneralProxyError((1, _generalerrors[1]))
-        if statuscode != 200:
-            self.close()
-            raise HTTPError((statuscode, statusline[2]))
-        self.__proxysockname = ("0.0.0.0", 0)
-        self.__proxypeername = (addr, destport)
-
-    def connect(self, destpair):
-        """connect(self, despair)
-        Connects to the specified destination through a proxy.
-        destpar - A tuple of the IP/DNS address and the port number.
-        (identical to socket's connect).
-        To select the proxy server use setproxy().
-        """
-        # Do a minimal input check first
-        if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int):
-            raise GeneralProxyError((5, _generalerrors[5]))
-        if self.__proxy[0] == PROXY_TYPE_SOCKS5:
-            if self.__proxy[2] != None:
-                portnum = self.__proxy[2]
-            else:
-                portnum = 1080
-            _orgsocket.connect(self, (self.__proxy[1], portnum))
-            self.__negotiatesocks5(destpair[0], destpair[1])
-        elif self.__proxy[0] == PROXY_TYPE_SOCKS4:
-            if self.__proxy[2] != None:
-                portnum = self.__proxy[2]
-            else:
-                portnum = 1080
-            _orgsocket.connect(self,(self.__proxy[1], portnum))
-            self.__negotiatesocks4(destpair[0], destpair[1])
-        elif self.__proxy[0] == PROXY_TYPE_HTTP:
-            if self.__proxy[2] != None:
-                portnum = self.__proxy[2]
-            else:
-                portnum = 8080
-            _orgsocket.connect(self,(self.__proxy[1], portnum))
-            self.__negotiatehttp(destpair[0], destpair[1])
-        elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL:
-            if self.__proxy[2] != None:
-                portnum = self.__proxy[2]
-            else:
-                portnum = 8080
-            _orgsocket.connect(self,(self.__proxy[1],portnum))
-            if destpair[1] == 443:
-                self.__negotiatehttp(destpair[0],destpair[1])
-            else:
-                self.__httptunnel = False
-        elif self.__proxy[0] == None:
-            _orgsocket.connect(self, (destpair[0], destpair[1]))
-        else:
-            raise GeneralProxyError((4, _generalerrors[4]))

src/httplib2/test/brokensocket/socket.py

@@ -1 +0,0 @@
-from realsocket import gaierror, error, getaddrinfo, SOCK_STREAM

src/httplib2/test/functional/test_proxies.py

@@ -1,88 +0,0 @@
-import unittest
-import errno
-import os
-import signal
-import subprocess
-import tempfile
-
-import nose
-
-import httplib2
-from httplib2 import socks
-from httplib2.test import miniserver
-
-tinyproxy_cfg = """
-User "%(user)s"
-Port %(port)s
-Listen 127.0.0.1
-PidFile "%(pidfile)s"
-LogFile "%(logfile)s"
-MaxClients 2
-StartServers 1
-LogLevel Info
-"""
-
-
-class FunctionalProxyHttpTest(unittest.TestCase):
-    def setUp(self):
-        if not socks:
-            raise nose.SkipTest('socks module unavailable')
-        if not subprocess:
-            raise nose.SkipTest('subprocess module unavailable')
-
-        # start a short-lived miniserver so we can get a likely port
-        # for the proxy
-        self.httpd, self.proxyport = miniserver.start_server(
-            miniserver.ThisDirHandler)
-        self.httpd.shutdown()
-        self.httpd, self.port = miniserver.start_server(
-            miniserver.ThisDirHandler)
-
-        self.pidfile = tempfile.mktemp()
-        self.logfile = tempfile.mktemp()
-        fd, self.conffile = tempfile.mkstemp()
-        f = os.fdopen(fd, 'w')
-        our_cfg = tinyproxy_cfg % {'user': os.getlogin(),
-                                   'pidfile': self.pidfile,
-                                   'port': self.proxyport,
-                                   'logfile': self.logfile}
-        f.write(our_cfg)
-        f.close()
-        try:
-            # TODO use subprocess.check_call when 2.4 is dropped
-            ret = subprocess.call(['tinyproxy', '-c', self.conffile])
-            self.assertEqual(0, ret)
-        except OSError, e:
-            if e.errno == errno.ENOENT:
-                raise nose.SkipTest('tinyproxy not available')
-            raise
-
-    def tearDown(self):
-        self.httpd.shutdown()
-        try:
-            pid = int(open(self.pidfile).read())
-            os.kill(pid, signal.SIGTERM)
-        except OSError, e:
-            if e.errno == errno.ESRCH:
-                print '\n\n\nTinyProxy Failed to start, log follows:'
-                print open(self.logfile).read()
-                print 'end tinyproxy log\n\n\n'
-            raise
-        map(os.unlink, (self.pidfile,
-                        self.logfile,
-                        self.conffile))
-
-    def testSimpleProxy(self):
-        proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP,
-                                        'localhost', self.proxyport)
-        client = httplib2.Http(proxy_info=proxy_info)
-        src = 'miniserver.py'
-        response, body = client.request('http://localhost:%d/%s' %
-                                        (self.port, src))
-        self.assertEqual(response.status, 200)
-        self.assertEqual(body, open(os.path.join(miniserver.HERE, src)).read())
-        lf = open(self.logfile).read()
-        expect = ('Established connection to host "127.0.0.1" '
-                  'using file descriptor')
-        self.assertTrue(expect in lf,
-                        'tinyproxy did not proxy a request for miniserver')

src/httplib2/test/miniserver.py

@@ -1,113 +0,0 @@
-import logging
-import os
-import select
-import SimpleHTTPServer
-import socket
-import SocketServer
-import threading
-
-HERE = os.path.dirname(__file__)
-logger = logging.getLogger(__name__)
-
-
-class ThisDirHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
-    def translate_path(self, path):
-        path = path.split('?', 1)[0].split('#', 1)[0]
-        return os.path.join(HERE, *filter(None, path.split('/')))
-
-    def log_message(self, s, *args):
-        # output via logging so nose can catch it
-        logger.info(s, *args)
-
-
-class ShutdownServer(SocketServer.TCPServer):
-    """Mixin that allows serve_forever to be shut down.
-
-    The methods in this mixin are backported from SocketServer.py in the Python
-    2.6.4 standard library. The mixin is unnecessary in 2.6 and later, when
-    BaseServer supports the shutdown method directly.
-    """
-
-    def __init__(self, use_tls, *args, **kwargs):
-        self.__use_tls = use_tls
-        SocketServer.TCPServer.__init__(self, *args, **kwargs)
-        self.__is_shut_down = threading.Event()
-        self.__serving = False
-
-    def server_bind(self):
-        SocketServer.TCPServer.server_bind(self)
-        if self.__use_tls:
-            import ssl
-            self.socket = ssl.wrap_socket(self.socket,
-                    os.path.join(os.path.dirname(__file__), 'server.key'),
-                    os.path.join(os.path.dirname(__file__), 'server.pem'),
-                    True
-            )
-
-
-    def serve_forever(self, poll_interval=0.1):
-        """Handle one request at a time until shutdown.
-
-        Polls for shutdown every poll_interval seconds. Ignores
-        self.timeout. If you need to do periodic tasks, do them in
-        another thread.
-        """
-        self.__serving = True
-        self.__is_shut_down.clear()
-        while self.__serving:
-            r, w, e = select.select([self.socket], [], [], poll_interval)
-            if r:
-                self._handle_request_noblock()
-        self.__is_shut_down.set()
-
-    def shutdown(self):
-        """Stops the serve_forever loop.
-
-        Blocks until the loop has finished. This must be called while
-        serve_forever() is running in another thread, or it will deadlock.
-        """
-        self.__serving = False
-        self.__is_shut_down.wait()
-
-    def handle_request(self):
-        """Handle one request, possibly blocking.
-
-        Respects self.timeout.
-        """
-        # Support people who used socket.settimeout() to escape
-        # handle_request before self.timeout was available.
-        timeout = self.socket.gettimeout()
-        if timeout is None:
-            timeout = self.timeout
-        elif self.timeout is not None:
-            timeout = min(timeout, self.timeout)
-        fd_sets = select.select([self], [], [], timeout)
-        if not fd_sets[0]:
-            self.handle_timeout()
-            return
-        self._handle_request_noblock()
-
-    def _handle_request_noblock(self):
-        """Handle one request, without blocking.
-
-        I assume that select.select has returned that the socket is
-        readable before this function was called, so there should be
-        no risk of blocking in get_request().
-        """
-        try:
-            request, client_address = self.get_request()
-        except socket.error:
-            return
-        if self.verify_request(request, client_address):
-            try:
-                self.process_request(request, client_address)
-            except:
-                self.handle_error(request, client_address)
-                self.close_request(request)
-
-
-def start_server(handler, use_tls=False):
-    httpd = ShutdownServer(use_tls, ("", 0), handler)
-    threading.Thread(target=httpd.serve_forever).start()
-    _, port = httpd.socket.getsockname()
-    return httpd, port

src/httplib2/test/other_cacerts.txt

@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDBzCCAe+gAwIBAgIJAIw94zvO7fk1MA0GCSqGSIb3DQEBBQUAMBoxGDAWBgNV
-BAMMD3d3dy5leGFtcGxlLmNvbTAeFw0xNjA2MDQwMjMxMTRaFw0yNjA2MDIwMjMx
-MTRaMBoxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBAK3YNcDIwK/wlTa0/iBARvDFOncQ6Jkk+Ymql1HXny7v
-mWPFWeLXEW+Zw1NrQEx/SIUGvxpRA+QyhTOhu2Gcwvtqilix/dHgaKgqWEcRYu8m
-L70uVDPVgB/kfNI8bpXM1Mz8Crjo0tHw5oUSD3wny8SyT6CYlXVmF923L8c2zdN9
-n9blFgYwxBq2+q+mqOiDErMFbwHES8FNBSWGBXdE1xjBdITtlfeHezmJhj/ylPW1
-7v8HInsv/WqU9DcJYlFxSnK0SZCLFBM/31Ez8O1gCfMlDUFvJoo59GyFqukUjuO1
-uB85wpu27gtcLm/J9X1Md71IxbDupV7a0dDoTvbhO4kCAwEAAaNQME4wHQYDVR0O
-BBYEFIHgAmwppZSKLz2peyFSO2kwVobNMB8GA1UdIwQYMBaAFIHgAmwppZSKLz2p
-eyFSO2kwVobNMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAJxz+AU/
-Iq8fMEStJ0BgPP1N86W9Jpb7aPMFCYTEZ+nd8hFPhPs4//55J0yIve+1I43MNFFz
-yflwwCzrIIhZdkvbsyea6CmlTo4jBc4+ihaDGobYnoNzFhavC47n5kYqJ8Ikyb2W
-OMrmNRiaTeSBl0wQmftnnQCbonenjmE1LDuJtE6bCwfFjfLbMxwdWtp/ymOlXsb5
-80XcWwcqc12UHWexYwHFzEJmDfncak/8tjHBsLWMJg5p2sVTY9kVt7TYgSIl+mFb
-4WVGrqZd2uTlJkRQQ4pCl+D+PKwadHuV6YI7oxkeajjcHCgbK/ANwW28MXYho6t6
-aWVIN4bWHrZ38kE=
------END CERTIFICATE-----

src/httplib2/test/server.key

@@ -1,146 +0,0 @@
-Private TLS server key file used for HTTPS-related unit tests
--------------------------------------------------------------
-
-Public Key Info:
-        Public Key Algorithm: RSA
-        Key Security Level: Medium (2048 bits)
-
-modulus:
-        00:cc:fb:f1:c5:de:29:29:40:3f:c4:9f:af:da:f6:be
-        27:f4:6a:00:ae:5a:f2:99:c3:5f:7a:e6:9b:cf:d9:08
-        34:01:9b:ea:fb:da:b5:d0:b5:b2:4e:60:b4:0d:8d:05
-        57:e4:2e:04:d4:57:1a:58:3c:0b:3a:ed:67:a3:13:31
-        58:0a:c2:eb:fd:d6:27:ee:07:95:30:35:b5:98:91:c7
-        a5:9b:be:a9:7e:ae:fd:73:c3:6b:21:bc:52:f8:ef:71
-        db:d3:b1:cd:51:df:b3:37:b3:fd:7d:ae:7e:02:38:be
-        8e:6f:45:55:e5:6d:8a:02:cb:36:c4:17:7a:ea:24:9a
-        72:8d:1e:75:03:3a:6f:c4:cb:a0:3a:50:56:32:bb:4c
-        e2:ea:74:f0:96:31:74:b2:c1:03:e8:c3:d4:a3:59:fc
-        7a:cc:68:35:c4:97:eb:aa:46:fa:64:c3:f9:55:59:22
-        b5:2b:3c:96:84:c6:d2:7d:b4:9f:b9:9c:af:d1:20:30
-        7c:e8:60:4e:ee:0a:60:a0:9d:4e:8a:d8:34:74:bd:f2
-        40:bc:d7:c2:b3:1a:b2:bb:d7:a5:4a:4c:65:94:43:82
-        16:9a:8f:76:2a:05:b0:9e:3d:a7:fb:e2:c7:78:25:f7
-        df:ca:08:ee:ec:4f:cd:1a:3c:03:41:ec:91:c5:50:70
-        4b:
-
-public exponent:
-        01:00:01:
-
-private exponent:
-        00:ad:01:83:b8:7d:dd:fd:ab:f5:66:2d:64:ce:08:ec
-        cb:6a:15:41:87:e6:c8:d5:10:39:78:d0:43:f7:73:f4
-        e1:77:ee:31:b0:e9:92:04:9a:25:e8:d2:e3:84:80:5e
-        5f:24:fd:d6:23:a5:74:5d:be:27:b8:4f:80:e5:f9:1f
-        ef:6f:fd:be:12:1a:7a:cf:02:65:5f:30:25:99:a4:88
-        7d:74:ea:c1:c1:63:4e:15:33:7d:2b:16:f8:6c:94:23
-        63:e6:d3:2d:38:89:f6:87:f0:08:e5:d7:ad:10:90:f5
-        fb:df:5c:04:b8:43:f0:74:95:31:1e:e5:b6:5f:02:0f
-        bb:55:cb:e1:b5:48:9f:1f:d3:1b:55:a7:bc:39:2b:8e
-        6d:14:64:3b:bf:e8:ca:6b:af:a9:f3:13:9a:c6:df:15
-        ef:6d:17:4e:8e:67:6c:41:20:dc:6b:08:0d:b9:14:cd
-        83:10:62:15:e6:b0:89:5d:37:fb:f6:fd:f0:bf:3b:9c
-        0b:e9:fd:b8:de:e4:64:90:bf:81:d5:59:2c:30:43:07
-        b9:60:8c:d0:ac:4f:95:87:aa:38:62:bd:c7:06:a7:c4
-        2d:08:c1:3c:86:10:c7:8e:1e:df:58:bf:95:ad:39:84
-        a0:2b:13:e2:18:e6:4a:80:f0:bc:04:50:bd:7d:cf:23
-        a1:
-
-prime1:
-        00:f0:8f:ad:2f:c9:64:f3:0d:2c:aa:06:17:05:8f:2f
-        d5:cb:92:22:90:05:66:3c:78:75:9d:7b:4c:6a:af:a9
-        1e:d6:28:4f:13:0e:3a:e7:31:49:3d:87:ef:2c:17:70
-        be:69:b3:42:82:6d:9c:b4:13:0a:e4:bc:8c:0f:1a:bd
-        04:b6:a0:be:ba:12:15:bf:04:db:91:1c:26:91:d6:d7
-        f2:ff:2f:0e:5f:96:a1:7c:4b:90:a8:2f:07:2a:cb:dc
-        40:a0:0b:1d:2a:1d:48:98:bd:4a:6b:9d:5c:69:b0:2b
-        6e:9b:2c:b2:a9:cb:28:fe:fa:7f:93:eb:20:c8:59:d0
-        11:
-
-prime2:
-        00:da:23:c0:3e:82:4c:88:7c:d4:fb:de:24:45:eb:9c
-        ae:2c:80:2d:52:a6:95:05:33:b9:d8:c1:7b:52:01:62
-        11:e6:b6:c6:0d:56:a3:68:39:26:9a:90:08:95:12:a9
-        1c:59:f6:0b:1d:af:6d:c0:c6:9b:2e:7a:62:98:21:36
-        e1:15:4c:e6:6d:a4:08:ac:90:af:57:86:71:78:2e:0e
-        cf:59:0f:35:79:cb:6a:a2:e2:30:2a:a8:f2:84:68:bc
-        8a:f2:48:3b:07:d5:a5:34:f3:d3:ec:25:61:38:f1:0a
-        07:f7:7e:29:61:e4:15:01:80:e3:7b:bd:63:9c:2e:16
-        9b:
-
-coefficient:
-        00:cb:b4:d2:9f:b4:04:db:8c:54:e6:ae:a9:28:a0:c9
-        70:ad:7a:94:72:5e:86:33:91:d9:43:61:2b:4d:55:e8
-        b7:25:d2:cd:db:1e:c4:56:95:68:85:e2:9b:4f:31:24
-        3a:40:06:41:1c:aa:7a:31:13:fa:07:e0:a6:59:c3:d1
-        d2:c5:2c:6a:82:98:bb:a1:59:c0:6f:ad:d7:2e:ed:5a
-        64:5f:e6:ea:4a:ee:45:29:d9:0f:96:b3:39:f7:ab:57
-        97:aa:c9:f7:b6:9c:c0:51:5d:9f:01:2c:ec:58:8d:06
-        6a:19:d0:33:74:11:6a:25:7c:8f:b7:31:d2:97:05:02
-        6f:
-
-exp1:
-        00:87:60:43:95:1d:e0:0a:8b:82:74:18:43:42:64:a7
-        05:c8:ae:ef:76:5f:23:7e:aa:47:7e:1d:52:0e:c3:d6
-        07:bd:7b:27:ac:d0:98:43:5c:d0:1b:a9:70:e6:3e:36
-        bb:61:5e:78:f2:4f:5f:1d:53:8e:10:d5:2e:78:9d:92
-        7b:a1:8e:ea:66:6a:21:04:c3:66:10:ce:67:c2:30:c6
-        8c:40:21:2a:14:8e:ff:47:a4:7a:be:ba:e0:6c:ac:16
-        c1:e3:8e:fd:95:a2:af:25:0d:79:61:00:48:6e:4d:ae
-        d3:6a:ce:07:a9:57:e4:35:41:a1:24:0b:f1:01:ee:d1
-        11:
-
-exp2:
-        00:ca:ca:bd:a7:de:fe:43:4c:b9:bb:c4:d2:37:e6:47
-        ec:6c:16:65:0c:17:2d:26:7e:e5:e1:2a:4d:f8:f8:ac
-        31:34:28:ea:89:ef:e7:4d:b7:03:ba:60:f8:79:8d:b5
-        85:53:e4:b6:84:cc:57:de:05:44:b2:ba:b7:f9:f1:b6
-        d1:1d:3a:36:65:eb:3e:dd:1e:4c:c3:b3:8a:bd:4d:24
-        1b:83:11:ee:86:e1:a2:aa:f6:58:0c:f0:af:34:85:21
-        f2:92:36:b0:1a:22:75:c9:7a:7b:a3:67:44:b0:e8:f4
-        88:5f:7e:fb:fd:b3:4a:0b:f1:c4:89:7e:91:a1:d9:fe
-        cd:
-
-
-Public Key ID: 92:D5:B4:2A:B6:A8:64:67:2C:2A:08:DB:51:B8:97:86:5E:44:CD:6C
-Public key's random art:
-+--[ RSA 2048]----+
-|     +    .      |
-|    . E  o .     |
-|   o .  . o      |
-|  . o  o .       |
-|   = .= S        |
-|. +.=o +         |
-|o++== .          |
-|++o=             |
-|o .              |
-+-----------------+
-
-
------BEGIN RSA PRIVATE KEY-----
-MIIEpgIBAAKCAQEAzPvxxd4pKUA/xJ+v2va+J/RqAK5a8pnDX3rmm8/ZCDQBm+r7
-2rXQtbJOYLQNjQVX5C4E1FcaWDwLOu1noxMxWArC6/3WJ+4HlTA1tZiRx6Wbvql+
-rv1zw2shvFL473Hb07HNUd+zN7P9fa5+Aji+jm9FVeVtigLLNsQXeuokmnKNHnUD
-Om/Ey6A6UFYyu0zi6nTwljF0ssED6MPUo1n8esxoNcSX66pG+mTD+VVZIrUrPJaE
-xtJ9tJ+5nK/RIDB86GBO7gpgoJ1Oitg0dL3yQLzXwrMasrvXpUpMZZRDghaaj3Yq
-BbCePaf74sd4Jfffygju7E/NGjwDQeyRxVBwSwIDAQABAoIBAQCtAYO4fd39q/Vm
-LWTOCOzLahVBh+bI1RA5eNBD93P04XfuMbDpkgSaJejS44SAXl8k/dYjpXRdvie4
-T4Dl+R/vb/2+Ehp6zwJlXzAlmaSIfXTqwcFjThUzfSsW+GyUI2Pm0y04ifaH8Ajl
-160QkPX731wEuEPwdJUxHuW2XwIPu1XL4bVInx/TG1WnvDkrjm0UZDu/6Mprr6nz
-E5rG3xXvbRdOjmdsQSDcawgNuRTNgxBiFeawiV03+/b98L87nAvp/bje5GSQv4HV
-WSwwQwe5YIzQrE+Vh6o4Yr3HBqfELQjBPIYQx44e31i/la05hKArE+IY5kqA8LwE
-UL19zyOhAoGBAPCPrS/JZPMNLKoGFwWPL9XLkiKQBWY8eHWde0xqr6ke1ihPEw46
-5zFJPYfvLBdwvmmzQoJtnLQTCuS8jA8avQS2oL66EhW/BNuRHCaR1tfy/y8OX5ah
-fEuQqC8HKsvcQKALHSodSJi9SmudXGmwK26bLLKpyyj++n+T6yDIWdARAoGBANoj
-wD6CTIh81PveJEXrnK4sgC1SppUFM7nYwXtSAWIR5rbGDVajaDkmmpAIlRKpHFn2
-Cx2vbcDGmy56YpghNuEVTOZtpAiskK9XhnF4Lg7PWQ81ectqouIwKqjyhGi8ivJI
-OwfVpTTz0+wlYTjxCgf3filh5BUBgON7vWOcLhabAoGBAIdgQ5Ud4AqLgnQYQ0Jk
-pwXIru92XyN+qkd+HVIOw9YHvXsnrNCYQ1zQG6lw5j42u2FeePJPXx1TjhDVLnid
-knuhjupmaiEEw2YQzmfCMMaMQCEqFI7/R6R6vrrgbKwWweOO/ZWiryUNeWEASG5N
-rtNqzgepV+Q1QaEkC/EB7tERAoGBAMrKvafe/kNMubvE0jfmR+xsFmUMFy0mfuXh
-Kk34+KwxNCjqie/nTbcDumD4eY21hVPktoTMV94FRLK6t/nxttEdOjZl6z7dHkzD
-s4q9TSQbgxHuhuGiqvZYDPCvNIUh8pI2sBoidcl6e6NnRLDo9Ihffvv9s0oL8cSJ
-fpGh2f7NAoGBAMu00p+0BNuMVOauqSigyXCtepRyXoYzkdlDYStNVei3JdLN2x7E
-VpVoheKbTzEkOkAGQRyqejET+gfgplnD0dLFLGqCmLuhWcBvrdcu7VpkX+bqSu5F
-KdkPlrM596tXl6rJ97acwFFdnwEs7FiNBmoZ0DN0EWolfI+3MdKXBQJv
------END RSA PRIVATE KEY-----

src/httplib2/test/server.pem

@@ -1,50 +0,0 @@
-Public, self-signed TLS server key file used for HTTPS-related unit tests
--------------------------------------------------------------------------
-
-Public Key Information:
-        Public Key Algorithm: RSA
-        Algorithm Security Level: Medium (2048 bits)
-                Modulus (bits 2048):
-                        00:cc:fb:f1:c5:de:29:29:40:3f:c4:9f:af:da:f6:be
-                        27:f4:6a:00:ae:5a:f2:99:c3:5f:7a:e6:9b:cf:d9:08
-                        34:01:9b:ea:fb:da:b5:d0:b5:b2:4e:60:b4:0d:8d:05
-                        57:e4:2e:04:d4:57:1a:58:3c:0b:3a:ed:67:a3:13:31
-                        58:0a:c2:eb:fd:d6:27:ee:07:95:30:35:b5:98:91:c7
-                        a5:9b:be:a9:7e:ae:fd:73:c3:6b:21:bc:52:f8:ef:71
-                        db:d3:b1:cd:51:df:b3:37:b3:fd:7d:ae:7e:02:38:be
-                        8e:6f:45:55:e5:6d:8a:02:cb:36:c4:17:7a:ea:24:9a
-                        72:8d:1e:75:03:3a:6f:c4:cb:a0:3a:50:56:32:bb:4c
-                        e2:ea:74:f0:96:31:74:b2:c1:03:e8:c3:d4:a3:59:fc
-                        7a:cc:68:35:c4:97:eb:aa:46:fa:64:c3:f9:55:59:22
-                        b5:2b:3c:96:84:c6:d2:7d:b4:9f:b9:9c:af:d1:20:30
-                        7c:e8:60:4e:ee:0a:60:a0:9d:4e:8a:d8:34:74:bd:f2
-                        40:bc:d7:c2:b3:1a:b2:bb:d7:a5:4a:4c:65:94:43:82
-                        16:9a:8f:76:2a:05:b0:9e:3d:a7:fb:e2:c7:78:25:f7
-                        df:ca:08:ee:ec:4f:cd:1a:3c:03:41:ec:91:c5:50:70
-                        4b
-                Exponent (bits 24):
-                        01:00:01
-
-Public Key Usage:
-
-Public Key ID: 92d5b42ab6a864672c2a08db51b897865e44cd6c
-
-
------BEGIN CERTIFICATE-----
-MIIC+zCCAeOgAwIBAgIJAISbkoXpX75CMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
-BAMMCWxvY2FsaG9zdDAeFw0xNjA2MTcxNDA1NTlaFw0yNjA2MTUxNDA1NTlaMBQx
-EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
-ggEBAMz78cXeKSlAP8Sfr9r2vif0agCuWvKZw1965pvP2Qg0AZvq+9q10LWyTmC0
-DY0FV+QuBNRXGlg8CzrtZ6MTMVgKwuv91ifuB5UwNbWYkcelm76pfq79c8NrIbxS
-+O9x29OxzVHfszez/X2ufgI4vo5vRVXlbYoCyzbEF3rqJJpyjR51AzpvxMugOlBW
-MrtM4up08JYxdLLBA+jD1KNZ/HrMaDXEl+uqRvpkw/lVWSK1KzyWhMbSfbSfuZyv
-0SAwfOhgTu4KYKCdTorYNHS98kC818KzGrK716VKTGWUQ4IWmo92KgWwnj2n++LH
-eCX338oI7uxPzRo8A0HskcVQcEsCAwEAAaNQME4wHQYDVR0OBBYEFFdXD8Z8k0et
-ZNyM4e4WypNnGlcCMB8GA1UdIwQYMBaAFFdXD8Z8k0etZNyM4e4WypNnGlcCMAwG
-A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAC4FsnK1Ph/JpdoqSRTCJiVM
-MPFaaavKEEyYdAKPk/Acmb9vf07sqsT+OZg/0obsZG9LxJb7x0iAnhfM3aS+CmO9
-Ym2lXeFDaJ2bHooB9MsG2C3+n8lJUMwxm7Cqpff/lpCK6Z+6MGPx3GRs6HUEl34k
-BB5pue2vqhtFQ03UdHMpAK0M7n3TloAWbFb1a/JmqzTbsQ0oaMHGoECQEAbaBl+a
-/up6vA3iZHq+ZPYS1KIx+xuT/SapLcyUtjfhmq1bROVZP4+6EHMsBMnhJBYKxxHy
-0qKvqJL9X3NQLMgMKKUKzX+BuG2u5aRRyVIqewT/ORjaUr9Y8lU7WlXPf7Ljm6s=
------END CERTIFICATE-----

src/httplib2/test/smoke_test.py

@@ -1,23 +0,0 @@
-import os
-import unittest
-
-import httplib2
-
-from httplib2.test import miniserver
-
-
-class HttpSmokeTest(unittest.TestCase):
-    def setUp(self):
-        self.httpd, self.port = miniserver.start_server(
-            miniserver.ThisDirHandler)
-
-    def tearDown(self):
-        self.httpd.shutdown()
-
-    def testGetFile(self):
-        client = httplib2.Http()
-        src = 'miniserver.py'
-        response, body = client.request('http://localhost:%d/%s' %
-                                        (self.port, src))
-        self.assertEqual(response.status, 200)
-        self.assertEqual(body, open(os.path.join(miniserver.HERE, src)).read())

src/httplib2/test/test_no_socket.py

@@ -1,24 +0,0 @@
-"""Tests for httplib2 when the socket module is missing.
-
-This helps ensure compatibility with environments such as AppEngine.
-"""
-import os
-import sys
-import unittest
-
-import httplib2
-
-class MissingSocketTest(unittest.TestCase):
-    def setUp(self):
-        self._oldsocks = httplib2.socks
-        httplib2.socks = None
-
-    def tearDown(self):
-        httplib2.socks = self._oldsocks
-
-    def testProxyDisabled(self):
-        proxy_info = httplib2.ProxyInfo('blah',
-                                        'localhost', 0)
-        client = httplib2.Http(proxy_info=proxy_info)
-        self.assertRaises(httplib2.ProxiesUnavailableError,
-                          client.request, 'http://localhost:-1/')

src/httplib2/test/test_ssl_context.py

@@ -1,66 +0,0 @@
-#!/usr/bin/python2
-import BaseHTTPServer
-import logging
-import os.path
-import unittest
-import sys
-
-import httplib2
-
-from httplib2.test import miniserver
-
-logger = logging.getLogger(__name__)
-
-class KeepAliveHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-    """
-    Request handler that keeps the HTTP connection open, so that the test can
-    inspect the resulting SSL connection object
-    """
-    def do_GET(self):
-        self.send_response(200)
-        self.send_header("Content-Length", "0")
-        self.send_header("Connection", "keep-alive")
-        self.end_headers()
-
-        self.close_connection = 0
-
-    def log_message(self, s, *args):
-        # output via logging so nose can catch it
-        logger.info(s, *args)
-
-class HttpsContextTest(unittest.TestCase):
-    def setUp(self):
-        if sys.version_info < (2, 7, 9):
-            return
-
-        self.httpd, self.port = miniserver.start_server(
-            KeepAliveHandler, True)
-
-    def tearDown(self):
-        self.httpd.shutdown()
-
-    def testHttpsContext(self):
-        if sys.version_info < (2, 7, 9):
-            if hasattr(unittest, "skipTest"):
-                self.skipTest("SSLContext requires Python 2.7.9")# Python 2.7.0
-            else:
-                return
-        import ssl
-
-        client = httplib2.Http(
-            ca_certs=os.path.join(os.path.dirname(__file__), 'server.pem'))
-
-        # Establish connection to local server
-        client.request('https://localhost:%d/' % (self.port))
-
-        # Verify that connection uses a TLS context with the correct hostname
-        conn = client.connections['https:localhost:%d' % self.port]
-
-        self.assertIsInstance(conn.sock, ssl.SSLSocket)
-        self.assertTrue(hasattr(conn.sock, 'context'))
-        self.assertIsInstance(conn.sock.context, ssl.SSLContext)
-        self.assertTrue(conn.sock.context.check_hostname)
-        self.assertEqual(conn.sock.server_hostname, 'localhost')
-        self.assertEqual(conn.sock.context.check_hostname, True)
-        self.assertEqual(conn.sock.context.verify_mode, ssl.CERT_REQUIRED)
-        self.assertEqual(conn.sock.context.protocol, ssl.PROTOCOL_SSLv23)

src/linux-gam.spec

@@ -1,24 +1,33 @@
-# -*- mode: python -*-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['_tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('httplib2/cacerts.txt', 'httplib2/cacerts.txt', 'DATA')]
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-a.datas += [('email-settings-v2.json', 'email-settings-v2.json', 'DATA')]
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )
+# -*- mode: python -*-
+
+import sys
+
+sys.modules['FixTk'] = None
+
+a = Analysis(['gam.py'],
+             hiddenimports=[],
+             hookspath=None,
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             runtime_hooks=None)
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
+
+# dynamically determine where httplib2/cacerts.txt lives
+import importlib
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
+
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam',
+          debug=False,
+          strip=None,
+          upx=False,
+          console=True )

src/macos-gam.spec

@@ -1,24 +1,32 @@
-# -*- mode: python -*-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['_tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('httplib2/cacerts.txt', 'httplib2/cacerts.txt', 'DATA')]
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-a.datas += [('email-settings-v2.json', 'email-settings-v2.json', 'DATA')]
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )
+# -*- mode: python -*-
+import sys
+
+sys.modules['FixTk'] = None
+
+a = Analysis(['gam.py'],
+             hiddenimports=[],
+             hookspath=None,
+             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             runtime_hooks=None)
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
+
+# dynamically determine where httplib2/cacerts.txt lives
+import importlib
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
+
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam',
+          debug=False,
+          strip=None,
+          upx=False,
+          console=True )

src/oauth2client/__init__.py

@@ -1,23 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Client library for using OAuth2, especially with Google APIs."""
-
-__version__ = '4.1.2'
-
-GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/v2/auth'
-GOOGLE_DEVICE_URI = 'https://accounts.google.com/o/oauth2/device/code'
-GOOGLE_REVOKE_URI = 'https://accounts.google.com/o/oauth2/revoke'
-GOOGLE_TOKEN_URI = 'https://www.googleapis.com/oauth2/v4/token'
-GOOGLE_TOKEN_INFO_URI = 'https://www.googleapis.com/oauth2/v3/tokeninfo'

src/oauth2client/_helpers.py

@@ -1,341 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helper functions for commonly used utilities."""
-
-import base64
-import functools
-import inspect
-import json
-import logging
-import os
-import warnings
-
-import six
-from six.moves import urllib
-
-
-logger = logging.getLogger(__name__)
-
-POSITIONAL_WARNING = 'WARNING'
-POSITIONAL_EXCEPTION = 'EXCEPTION'
-POSITIONAL_IGNORE = 'IGNORE'
-POSITIONAL_SET = frozenset([POSITIONAL_WARNING, POSITIONAL_EXCEPTION,
-                            POSITIONAL_IGNORE])
-
-positional_parameters_enforcement = POSITIONAL_WARNING
-
-_SYM_LINK_MESSAGE = 'File: {0}: Is a symbolic link.'
-_IS_DIR_MESSAGE = '{0}: Is a directory'
-_MISSING_FILE_MESSAGE = 'Cannot access {0}: No such file or directory'
-
-
-def positional(max_positional_args):
-    """A decorator to declare that only the first N arguments my be positional.
-
-    This decorator makes it easy to support Python 3 style keyword-only
-    parameters. For example, in Python 3 it is possible to write::
-
-        def fn(pos1, *, kwonly1=None, kwonly1=None):
-            ...
-
-    All named parameters after ``*`` must be a keyword::
-
-        fn(10, 'kw1', 'kw2')  # Raises exception.
-        fn(10, kwonly1='kw1')  # Ok.
-
-    Example
-    ^^^^^^^
-
-    To define a function like above, do::
-
-        @positional(1)
-        def fn(pos1, kwonly1=None, kwonly2=None):
-            ...
-
-    If no default value is provided to a keyword argument, it becomes a
-    required keyword argument::
-
-        @positional(0)
-        def fn(required_kw):
-            ...
-
-    This must be called with the keyword parameter::
-
-        fn()  # Raises exception.
-        fn(10)  # Raises exception.
-        fn(required_kw=10)  # Ok.
-
-    When defining instance or class methods always remember to account for
-    ``self`` and ``cls``::
-
-        class MyClass(object):
-
-            @positional(2)
-            def my_method(self, pos1, kwonly1=None):
-                ...
-
-            @classmethod
-            @positional(2)
-            def my_method(cls, pos1, kwonly1=None):
-                ...
-
-    The positional decorator behavior is controlled by
-    ``_helpers.positional_parameters_enforcement``, which may be set to
-    ``POSITIONAL_EXCEPTION``, ``POSITIONAL_WARNING`` or
-    ``POSITIONAL_IGNORE`` to raise an exception, log a warning, or do
-    nothing, respectively, if a declaration is violated.
-
-    Args:
-        max_positional_arguments: Maximum number of positional arguments. All
-                                  parameters after the this index must be
-                                  keyword only.
-
-    Returns:
-        A decorator that prevents using arguments after max_positional_args
-        from being used as positional parameters.
-
-    Raises:
-        TypeError: if a key-word only argument is provided as a positional
-                   parameter, but only if
-                   _helpers.positional_parameters_enforcement is set to
-                   POSITIONAL_EXCEPTION.
-    """
-
-    def positional_decorator(wrapped):
-        @functools.wraps(wrapped)
-        def positional_wrapper(*args, **kwargs):
-            if len(args) > max_positional_args:
-                plural_s = ''
-                if max_positional_args != 1:
-                    plural_s = 's'
-                message = ('{function}() takes at most {args_max} positional '
-                           'argument{plural} ({args_given} given)'.format(
-                               function=wrapped.__name__,
-                               args_max=max_positional_args,
-                               args_given=len(args),
-                               plural=plural_s))
-                if positional_parameters_enforcement == POSITIONAL_EXCEPTION:
-                    raise TypeError(message)
-                elif positional_parameters_enforcement == POSITIONAL_WARNING:
-                    logger.warning(message)
-            return wrapped(*args, **kwargs)
-        return positional_wrapper
-
-    if isinstance(max_positional_args, six.integer_types):
-        return positional_decorator
-    else:
-        args, _, _, defaults = inspect.getargspec(max_positional_args)
-        return positional(len(args) - len(defaults))(max_positional_args)
-
-
-def scopes_to_string(scopes):
-    """Converts scope value to a string.
-
-    If scopes is a string then it is simply passed through. If scopes is an
-    iterable then a string is returned that is all the individual scopes
-    concatenated with spaces.
-
-    Args:
-        scopes: string or iterable of strings, the scopes.
-
-    Returns:
-        The scopes formatted as a single string.
-    """
-    if isinstance(scopes, six.string_types):
-        return scopes
-    else:
-        return ' '.join(scopes)
-
-
-def string_to_scopes(scopes):
-    """Converts stringifed scope value to a list.
-
-    If scopes is a list then it is simply passed through. If scopes is an
-    string then a list of each individual scope is returned.
-
-    Args:
-        scopes: a string or iterable of strings, the scopes.
-
-    Returns:
-        The scopes in a list.
-    """
-    if not scopes:
-        return []
-    elif isinstance(scopes, six.string_types):
-        return scopes.split(' ')
-    else:
-        return scopes
-
-
-def parse_unique_urlencoded(content):
-    """Parses unique key-value parameters from urlencoded content.
-
-    Args:
-        content: string, URL-encoded key-value pairs.
-
-    Returns:
-        dict, The key-value pairs from ``content``.
-
-    Raises:
-        ValueError: if one of the keys is repeated.
-    """
-    urlencoded_params = urllib.parse.parse_qs(content)
-    params = {}
-    for key, value in six.iteritems(urlencoded_params):
-        if len(value) != 1:
-            msg = ('URL-encoded content contains a repeated value:'
-                   '%s -> %s' % (key, ', '.join(value)))
-            raise ValueError(msg)
-        params[key] = value[0]
-    return params
-
-
-def update_query_params(uri, params):
-    """Updates a URI with new query parameters.
-
-    If a given key from ``params`` is repeated in the ``uri``, then
-    the URI will be considered invalid and an error will occur.
-
-    If the URI is valid, then each value from ``params`` will
-    replace the corresponding value in the query parameters (if
-    it exists).
-
-    Args:
-        uri: string, A valid URI, with potential existing query parameters.
-        params: dict, A dictionary of query parameters.
-
-    Returns:
-        The same URI but with the new query parameters added.
-    """
-    parts = urllib.parse.urlparse(uri)
-    query_params = parse_unique_urlencoded(parts.query)
-    query_params.update(params)
-    new_query = urllib.parse.urlencode(query_params)
-    new_parts = parts._replace(query=new_query)
-    return urllib.parse.urlunparse(new_parts)
-
-
-def _add_query_parameter(url, name, value):
-    """Adds a query parameter to a url.
-
-    Replaces the current value if it already exists in the URL.
-
-    Args:
-        url: string, url to add the query parameter to.
-        name: string, query parameter name.
-        value: string, query parameter value.
-
-    Returns:
-        Updated query parameter. Does not update the url if value is None.
-    """
-    if value is None:
-        return url
-    else:
-        return update_query_params(url, {name: value})
-
-
-def validate_file(filename):
-    if os.path.islink(filename):
-        raise IOError(_SYM_LINK_MESSAGE.format(filename))
-    elif os.path.isdir(filename):
-        raise IOError(_IS_DIR_MESSAGE.format(filename))
-    elif not os.path.isfile(filename):
-        warnings.warn(_MISSING_FILE_MESSAGE.format(filename))
-
-
-def _parse_pem_key(raw_key_input):
-    """Identify and extract PEM keys.
-
-    Determines whether the given key is in the format of PEM key, and extracts
-    the relevant part of the key if it is.
-
-    Args:
-        raw_key_input: The contents of a private key file (either PEM or
-                       PKCS12).
-
-    Returns:
-        string, The actual key if the contents are from a PEM file, or
-        else None.
-    """
-    offset = raw_key_input.find(b'-----BEGIN ')
-    if offset != -1:
-        return raw_key_input[offset:]
-
-
-def _json_encode(data):
-    return json.dumps(data, separators=(',', ':'))
-
-
-def _to_bytes(value, encoding='ascii'):
-    """Converts a string value to bytes, if necessary.
-
-    Unfortunately, ``six.b`` is insufficient for this task since in
-    Python2 it does not modify ``unicode`` objects.
-
-    Args:
-        value: The string/bytes value to be converted.
-        encoding: The encoding to use to convert unicode to bytes. Defaults
-                  to "ascii", which will not allow any characters from ordinals
-                  larger than 127. Other useful values are "latin-1", which
-                  which will only allows byte ordinals (up to 255) and "utf-8",
-                  which will encode any unicode that needs to be.
-
-    Returns:
-        The original value converted to bytes (if unicode) or as passed in
-        if it started out as bytes.
-
-    Raises:
-        ValueError if the value could not be converted to bytes.
-    """
-    result = (value.encode(encoding)
-              if isinstance(value, six.text_type) else value)
-    if isinstance(result, six.binary_type):
-        return result
-    else:
-        raise ValueError('{0!r} could not be converted to bytes'.format(value))
-
-
-def _from_bytes(value):
-    """Converts bytes to a string value, if necessary.
-
-    Args:
-        value: The string/bytes value to be converted.
-
-    Returns:
-        The original value converted to unicode (if bytes) or as passed in
-        if it started out as unicode.
-
-    Raises:
-        ValueError if the value could not be converted to unicode.
-    """
-    result = (value.decode('utf-8')
-              if isinstance(value, six.binary_type) else value)
-    if isinstance(result, six.text_type):
-        return result
-    else:
-        raise ValueError(
-            '{0!r} could not be converted to unicode'.format(value))
-
-
-def _urlsafe_b64encode(raw_bytes):
-    raw_bytes = _to_bytes(raw_bytes, encoding='utf-8')
-    return base64.urlsafe_b64encode(raw_bytes).rstrip(b'=')
-
-
-def _urlsafe_b64decode(b64string):
-    # Guard against unicode strings, which base64 can't handle.
-    b64string = _to_bytes(b64string)
-    padded = b64string + b'=' * (4 - len(b64string) % 4)
-    return base64.urlsafe_b64decode(padded)

src/oauth2client/_openssl_crypt.py

@@ -1,136 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""OpenSSL Crypto-related routines for oauth2client."""
-
-from OpenSSL import crypto
-
-from oauth2client import _helpers
-
-
-class OpenSSLVerifier(object):
-    """Verifies the signature on a message."""
-
-    def __init__(self, pubkey):
-        """Constructor.
-
-        Args:
-            pubkey: OpenSSL.crypto.PKey, The public key to verify with.
-        """
-        self._pubkey = pubkey
-
-    def verify(self, message, signature):
-        """Verifies a message against a signature.
-
-        Args:
-        message: string or bytes, The message to verify. If string, will be
-                 encoded to bytes as utf-8.
-        signature: string or bytes, The signature on the message. If string,
-                   will be encoded to bytes as utf-8.
-
-        Returns:
-            True if message was signed by the private key associated with the
-            public key that this object was constructed with.
-        """
-        message = _helpers._to_bytes(message, encoding='utf-8')
-        signature = _helpers._to_bytes(signature, encoding='utf-8')
-        try:
-            crypto.verify(self._pubkey, signature, message, 'sha256')
-            return True
-        except crypto.Error:
-            return False
-
-    @staticmethod
-    def from_string(key_pem, is_x509_cert):
-        """Construct a Verified instance from a string.
-
-        Args:
-            key_pem: string, public key in PEM format.
-            is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it
-                          is expected to be an RSA key in PEM format.
-
-        Returns:
-            Verifier instance.
-
-        Raises:
-            OpenSSL.crypto.Error: if the key_pem can't be parsed.
-        """
-        key_pem = _helpers._to_bytes(key_pem)
-        if is_x509_cert:
-            pubkey = crypto.load_certificate(crypto.FILETYPE_PEM, key_pem)
-        else:
-            pubkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key_pem)
-        return OpenSSLVerifier(pubkey)
-
-
-class OpenSSLSigner(object):
-    """Signs messages with a private key."""
-
-    def __init__(self, pkey):
-        """Constructor.
-
-        Args:
-            pkey: OpenSSL.crypto.PKey (or equiv), The private key to sign with.
-        """
-        self._key = pkey
-
-    def sign(self, message):
-        """Signs a message.
-
-        Args:
-            message: bytes, Message to be signed.
-
-        Returns:
-            string, The signature of the message for the given key.
-        """
-        message = _helpers._to_bytes(message, encoding='utf-8')
-        return crypto.sign(self._key, message, 'sha256')
-
-    @staticmethod
-    def from_string(key, password=b'notasecret'):
-        """Construct a Signer instance from a string.
-
-        Args:
-            key: string, private key in PKCS12 or PEM format.
-            password: string, password for the private key file.
-
-        Returns:
-            Signer instance.
-
-        Raises:
-            OpenSSL.crypto.Error if the key can't be parsed.
-        """
-        key = _helpers._to_bytes(key)
-        parsed_pem_key = _helpers._parse_pem_key(key)
-        if parsed_pem_key:
-            pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, parsed_pem_key)
-        else:
-            password = _helpers._to_bytes(password, encoding='utf-8')
-            pkey = crypto.load_pkcs12(key, password).get_privatekey()
-        return OpenSSLSigner(pkey)
-
-
-def pkcs12_key_as_pem(private_key_bytes, private_key_password):
-    """Convert the contents of a PKCS#12 key to PEM using pyOpenSSL.
-
-    Args:
-        private_key_bytes: Bytes. PKCS#12 key in DER format.
-        private_key_password: String. Password for PKCS#12 key.
-
-    Returns:
-        String. PEM contents of ``private_key_bytes``.
-    """
-    private_key_password = _helpers._to_bytes(private_key_password)
-    pkcs12 = crypto.load_pkcs12(private_key_bytes, private_key_password)
-    return crypto.dump_privatekey(crypto.FILETYPE_PEM,
-                                  pkcs12.get_privatekey())

src/oauth2client/_pkce.py

@@ -1,67 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Utility functions for implementing Proof Key for Code Exchange (PKCE) by OAuth
-Public Clients
-
-See RFC7636.
-"""
-
-import base64
-import hashlib
-import os
-
-
-def code_verifier(n_bytes=64):
-    """
-    Generates a 'code_verifier' as described in section 4.1 of RFC 7636.
-
-    This is a 'high-entropy cryptographic random string' that will be
-    impractical for an attacker to guess.
-
-    Args:
-        n_bytes: integer between 31 and 96, inclusive. default: 64
-            number of bytes of entropy to include in verifier.
-
-    Returns:
-        Bytestring, representing urlsafe base64-encoded random data.
-    """
-    verifier = base64.urlsafe_b64encode(os.urandom(n_bytes)).rstrip(b'=')
-    # https://tools.ietf.org/html/rfc7636#section-4.1
-    # minimum length of 43 characters and a maximum length of 128 characters.
-    if len(verifier) < 43:
-        raise ValueError("Verifier too short. n_bytes must be > 30.")
-    elif len(verifier) > 128:
-        raise ValueError("Verifier too long. n_bytes must be < 97.")
-    else:
-        return verifier
-
-
-def code_challenge(verifier):
-    """
-    Creates a 'code_challenge' as described in section 4.2 of RFC 7636
-    by taking the sha256 hash of the verifier and then urlsafe
-    base64-encoding it.
-
-    Args:
-        verifier: bytestring, representing a code_verifier as generated by
-            code_verifier().
-
-    Returns:
-        Bytestring, representing a urlsafe base64-encoded sha256 hash digest,
-            without '=' padding.
-    """
-    digest = hashlib.sha256(verifier).digest()
-    return base64.urlsafe_b64encode(digest).rstrip(b'=')

src/oauth2client/_pure_python_crypt.py

@@ -1,184 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Pure Python crypto-related routines for oauth2client.
-
-Uses the ``rsa``, ``pyasn1`` and ``pyasn1_modules`` packages
-to parse PEM files storing PKCS#1 or PKCS#8 keys as well as
-certificates.
-"""
-
-from pyasn1.codec.der import decoder
-from pyasn1_modules import pem
-from pyasn1_modules.rfc2459 import Certificate
-from pyasn1_modules.rfc5208 import PrivateKeyInfo
-import rsa
-import six
-
-from oauth2client import _helpers
-
-
-_PKCS12_ERROR = r"""\
-PKCS12 format is not supported by the RSA library.
-Either install PyOpenSSL, or please convert .p12 format
-to .pem format:
-    $ cat key.p12 | \
-    >   openssl pkcs12 -nodes -nocerts -passin pass:notasecret | \
-    >   openssl rsa > key.pem
-"""
-
-_POW2 = (128, 64, 32, 16, 8, 4, 2, 1)
-_PKCS1_MARKER = ('-----BEGIN RSA PRIVATE KEY-----',
-                 '-----END RSA PRIVATE KEY-----')
-_PKCS8_MARKER = ('-----BEGIN PRIVATE KEY-----',
-                 '-----END PRIVATE KEY-----')
-_PKCS8_SPEC = PrivateKeyInfo()
-
-
-def _bit_list_to_bytes(bit_list):
-    """Converts an iterable of 1's and 0's to bytes.
-
-    Combines the list 8 at a time, treating each group of 8 bits
-    as a single byte.
-    """
-    num_bits = len(bit_list)
-    byte_vals = bytearray()
-    for start in six.moves.xrange(0, num_bits, 8):
-        curr_bits = bit_list[start:start + 8]
-        char_val = sum(val * digit
-                       for val, digit in zip(_POW2, curr_bits))
-        byte_vals.append(char_val)
-    return bytes(byte_vals)
-
-
-class RsaVerifier(object):
-    """Verifies the signature on a message.
-
-    Args:
-        pubkey: rsa.key.PublicKey (or equiv), The public key to verify with.
-    """
-
-    def __init__(self, pubkey):
-        self._pubkey = pubkey
-
-    def verify(self, message, signature):
-        """Verifies a message against a signature.
-
-        Args:
-            message: string or bytes, The message to verify. If string, will be
-                     encoded to bytes as utf-8.
-            signature: string or bytes, The signature on the message. If
-                       string, will be encoded to bytes as utf-8.
-
-        Returns:
-            True if message was signed by the private key associated with the
-            public key that this object was constructed with.
-        """
-        message = _helpers._to_bytes(message, encoding='utf-8')
-        try:
-            return rsa.pkcs1.verify(message, signature, self._pubkey)
-        except (ValueError, rsa.pkcs1.VerificationError):
-            return False
-
-    @classmethod
-    def from_string(cls, key_pem, is_x509_cert):
-        """Construct an RsaVerifier instance from a string.
-
-        Args:
-            key_pem: string, public key in PEM format.
-            is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it
-                          is expected to be an RSA key in PEM format.
-
-        Returns:
-            RsaVerifier instance.
-
-        Raises:
-            ValueError: if the key_pem can't be parsed. In either case, error
-                        will begin with 'No PEM start marker'. If
-                        ``is_x509_cert`` is True, will fail to find the
-                        "-----BEGIN CERTIFICATE-----" error, otherwise fails
-                        to find "-----BEGIN RSA PUBLIC KEY-----".
-        """
-        key_pem = _helpers._to_bytes(key_pem)
-        if is_x509_cert:
-            der = rsa.pem.load_pem(key_pem, 'CERTIFICATE')
-            asn1_cert, remaining = decoder.decode(der, asn1Spec=Certificate())
-            if remaining != b'':
-                raise ValueError('Unused bytes', remaining)
-
-            cert_info = asn1_cert['tbsCertificate']['subjectPublicKeyInfo']
-            key_bytes = _bit_list_to_bytes(cert_info['subjectPublicKey'])
-            pubkey = rsa.PublicKey.load_pkcs1(key_bytes, 'DER')
-        else:
-            pubkey = rsa.PublicKey.load_pkcs1(key_pem, 'PEM')
-        return cls(pubkey)
-
-
-class RsaSigner(object):
-    """Signs messages with a private key.
-
-    Args:
-        pkey: rsa.key.PrivateKey (or equiv), The private key to sign with.
-    """
-
-    def __init__(self, pkey):
-        self._key = pkey
-
-    def sign(self, message):
-        """Signs a message.
-
-        Args:
-            message: bytes, Message to be signed.
-
-        Returns:
-            string, The signature of the message for the given key.
-        """
-        message = _helpers._to_bytes(message, encoding='utf-8')
-        return rsa.pkcs1.sign(message, self._key, 'SHA-256')
-
-    @classmethod
-    def from_string(cls, key, password='notasecret'):
-        """Construct an RsaSigner instance from a string.
-
-        Args:
-            key: string, private key in PEM format.
-            password: string, password for private key file. Unused for PEM
-                      files.
-
-        Returns:
-            RsaSigner instance.
-
-        Raises:
-            ValueError if the key cannot be parsed as PKCS#1 or PKCS#8 in
-            PEM format.
-        """
-        key = _helpers._from_bytes(key)  # pem expects str in Py3
-        marker_id, key_bytes = pem.readPemBlocksFromFile(
-            six.StringIO(key), _PKCS1_MARKER, _PKCS8_MARKER)
-
-        if marker_id == 0:
-            pkey = rsa.key.PrivateKey.load_pkcs1(key_bytes,
-                                                 format='DER')
-        elif marker_id == 1:
-            key_info, remaining = decoder.decode(
-                key_bytes, asn1Spec=_PKCS8_SPEC)
-            if remaining != b'':
-                raise ValueError('Unused bytes', remaining)
-            pkey_info = key_info.getComponentByName('privateKey')
-            pkey = rsa.key.PrivateKey.load_pkcs1(pkey_info.asOctets(),
-                                                 format='DER')
-        else:
-            raise ValueError('No key could be detected.')
-
-        return cls(pkey)

src/oauth2client/_pycrypto_crypt.py

@@ -1,124 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""pyCrypto Crypto-related routines for oauth2client."""
-
-from Crypto.Hash import SHA256
-from Crypto.PublicKey import RSA
-from Crypto.Signature import PKCS1_v1_5
-from Crypto.Util.asn1 import DerSequence
-
-from oauth2client import _helpers
-
-
-class PyCryptoVerifier(object):
-    """Verifies the signature on a message."""
-
-    def __init__(self, pubkey):
-        """Constructor.
-
-        Args:
-            pubkey: OpenSSL.crypto.PKey (or equiv), The public key to verify
-            with.
-        """
-        self._pubkey = pubkey
-
-    def verify(self, message, signature):
-        """Verifies a message against a signature.
-
-        Args:
-            message: string or bytes, The message to verify. If string, will be
-                     encoded to bytes as utf-8.
-            signature: string or bytes, The signature on the message.
-
-        Returns:
-            True if message was signed by the private key associated with the
-            public key that this object was constructed with.
-        """
-        message = _helpers._to_bytes(message, encoding='utf-8')
-        return PKCS1_v1_5.new(self._pubkey).verify(
-            SHA256.new(message), signature)
-
-    @staticmethod
-    def from_string(key_pem, is_x509_cert):
-        """Construct a Verified instance from a string.
-
-        Args:
-            key_pem: string, public key in PEM format.
-            is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it
-                          is expected to be an RSA key in PEM format.
-
-        Returns:
-            Verifier instance.
-        """
-        if is_x509_cert:
-            key_pem = _helpers._to_bytes(key_pem)
-            pemLines = key_pem.replace(b' ', b'').split()
-            certDer = _helpers._urlsafe_b64decode(b''.join(pemLines[1:-1]))
-            certSeq = DerSequence()
-            certSeq.decode(certDer)
-            tbsSeq = DerSequence()
-            tbsSeq.decode(certSeq[0])
-            pubkey = RSA.importKey(tbsSeq[6])
-        else:
-            pubkey = RSA.importKey(key_pem)
-        return PyCryptoVerifier(pubkey)
-
-
-class PyCryptoSigner(object):
-    """Signs messages with a private key."""
-
-    def __init__(self, pkey):
-        """Constructor.
-
-        Args:
-            pkey, OpenSSL.crypto.PKey (or equiv), The private key to sign with.
-        """
-        self._key = pkey
-
-    def sign(self, message):
-        """Signs a message.
-
-        Args:
-            message: string, Message to be signed.
-
-        Returns:
-            string, The signature of the message for the given key.
-        """
-        message = _helpers._to_bytes(message, encoding='utf-8')
-        return PKCS1_v1_5.new(self._key).sign(SHA256.new(message))
-
-    @staticmethod
-    def from_string(key, password='notasecret'):
-        """Construct a Signer instance from a string.
-
-        Args:
-            key: string, private key in PEM format.
-            password: string, password for private key file. Unused for PEM
-                      files.
-
-        Returns:
-            Signer instance.
-
-        Raises:
-            NotImplementedError if the key isn't in PEM format.
-        """
-        parsed_pem_key = _helpers._parse_pem_key(_helpers._to_bytes(key))
-        if parsed_pem_key:
-            pkey = RSA.importKey(parsed_pem_key)
-        else:
-            raise NotImplementedError(
-                'No key in PEM format was detected. This implementation '
-                'can only use the PyCrypto library for keys in PEM '
-                'format.')
-        return PyCryptoSigner(pkey)

src/oauth2client/clientsecrets.py

@@ -1,173 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for reading OAuth 2.0 client secret files.
-
-A client_secrets.json file contains all the information needed to interact with
-an OAuth 2.0 protected service.
-"""
-
-import json
-
-import six
-
-
-# Properties that make a client_secrets.json file valid.
-TYPE_WEB = 'web'
-TYPE_INSTALLED = 'installed'
-
-VALID_CLIENT = {
-    TYPE_WEB: {
-        'required': [
-            'client_id',
-            'client_secret',
-            'redirect_uris',
-            'auth_uri',
-            'token_uri',
-        ],
-        'string': [
-            'client_id',
-            'client_secret',
-        ],
-    },
-    TYPE_INSTALLED: {
-        'required': [
-            'client_id',
-            'client_secret',
-            'redirect_uris',
-            'auth_uri',
-            'token_uri',
-        ],
-        'string': [
-            'client_id',
-            'client_secret',
-        ],
-    },
-}
-
-
-class Error(Exception):
-    """Base error for this module."""
-
-
-class InvalidClientSecretsError(Error):
-    """Format of ClientSecrets file is invalid."""
-
-
-def _validate_clientsecrets(clientsecrets_dict):
-    """Validate parsed client secrets from a file.
-
-    Args:
-        clientsecrets_dict: dict, a dictionary holding the client secrets.
-
-    Returns:
-        tuple, a string of the client type and the information parsed
-        from the file.
-    """
-    _INVALID_FILE_FORMAT_MSG = (
-        'Invalid file format. See '
-        'https://developers.google.com/api-client-library/'
-        'python/guide/aaa_client_secrets')
-
-    if clientsecrets_dict is None:
-        raise InvalidClientSecretsError(_INVALID_FILE_FORMAT_MSG)
-    try:
-        (client_type, client_info), = clientsecrets_dict.items()
-    except (ValueError, AttributeError):
-        raise InvalidClientSecretsError(
-            _INVALID_FILE_FORMAT_MSG + ' '
-            'Expected a JSON object with a single property for a "web" or '
-            '"installed" application')
-
-    if client_type not in VALID_CLIENT:
-        raise InvalidClientSecretsError(
-            'Unknown client type: {0}.'.format(client_type))
-
-    for prop_name in VALID_CLIENT[client_type]['required']:
-        if prop_name not in client_info:
-            raise InvalidClientSecretsError(
-                'Missing property "{0}" in a client type of "{1}".'.format(
-                    prop_name, client_type))
-    for prop_name in VALID_CLIENT[client_type]['string']:
-        if client_info[prop_name].startswith('[['):
-            raise InvalidClientSecretsError(
-                'Property "{0}" is not configured.'.format(prop_name))
-    return client_type, client_info
-
-
-def load(fp):
-    obj = json.load(fp)
-    return _validate_clientsecrets(obj)
-
-
-def loads(s):
-    obj = json.loads(s)
-    return _validate_clientsecrets(obj)
-
-
-def _loadfile(filename):
-    try:
-        with open(filename, 'r') as fp:
-            obj = json.load(fp)
-    except IOError as exc:
-        raise InvalidClientSecretsError('Error opening file', exc.filename,
-                                        exc.strerror, exc.errno)
-    return _validate_clientsecrets(obj)
-
-
-def loadfile(filename, cache=None):
-    """Loading of client_secrets JSON file, optionally backed by a cache.
-
-    Typical cache storage would be App Engine memcache service,
-    but you can pass in any other cache client that implements
-    these methods:
-
-    * ``get(key, namespace=ns)``
-    * ``set(key, value, namespace=ns)``
-
-    Usage::
-
-        # without caching
-        client_type, client_info = loadfile('secrets.json')
-        # using App Engine memcache service
-        from google.appengine.api import memcache
-        client_type, client_info = loadfile('secrets.json', cache=memcache)
-
-    Args:
-        filename: string, Path to a client_secrets.json file on a filesystem.
-        cache: An optional cache service client that implements get() and set()
-        methods. If not specified, the file is always being loaded from
-                 a filesystem.
-
-    Raises:
-        InvalidClientSecretsError: In case of a validation error or some
-                                   I/O failure. Can happen only on cache miss.
-
-    Returns:
-        (client_type, client_info) tuple, as _loadfile() normally would.
-        JSON contents is validated only during first load. Cache hits are not
-        validated.
-    """
-    _SECRET_NAMESPACE = 'oauth2client:secrets#ns'
-
-    if not cache:
-        return _loadfile(filename)
-
-    obj = cache.get(filename, namespace=_SECRET_NAMESPACE)
-    if obj is None:
-        client_type, client_info = _loadfile(filename)
-        obj = {client_type: client_info}
-        cache.set(filename, obj, namespace=_SECRET_NAMESPACE)
-
-    return next(six.iteritems(obj))

src/oauth2client/contrib/__init__.py

@@ -1,6 +0,0 @@
-"""Contributed modules.
-
-Contrib contains modules that are not considered part of the core oauth2client
-library but provide additional functionality. These modules are intended to
-make it easier to use oauth2client.
-"""

src/oauth2client/contrib/_appengine_ndb.py

@@ -1,163 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Google App Engine utilities helper.
-
-Classes that directly require App Engine's ndb library. Provided
-as a separate module in case of failure to import ndb while
-other App Engine libraries are present.
-"""
-
-import logging
-
-from google.appengine.ext import ndb
-
-from oauth2client import client
-
-
-NDB_KEY = ndb.Key
-"""Key constant used by :mod:`oauth2client.contrib.appengine`."""
-
-NDB_MODEL = ndb.Model
-"""Model constant used by :mod:`oauth2client.contrib.appengine`."""
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class SiteXsrfSecretKeyNDB(ndb.Model):
-    """NDB Model for storage for the sites XSRF secret key.
-
-    Since this model uses the same kind as SiteXsrfSecretKey, it can be
-    used interchangeably. This simply provides an NDB model for interacting
-    with the same data the DB model interacts with.
-
-    There should only be one instance stored of this model, the one used
-    for the site.
-    """
-    secret = ndb.StringProperty()
-
-    @classmethod
-    def _get_kind(cls):
-        """Return the kind name for this class."""
-        return 'SiteXsrfSecretKey'
-
-
-class FlowNDBProperty(ndb.PickleProperty):
-    """App Engine NDB datastore Property for Flow.
-
-    Serves the same purpose as the DB FlowProperty, but for NDB models.
-    Since PickleProperty inherits from BlobProperty, the underlying
-    representation of the data in the datastore will be the same as in the
-    DB case.
-
-    Utility property that allows easy storage and retrieval of an
-    oauth2client.Flow
-    """
-
-    def _validate(self, value):
-        """Validates a value as a proper Flow object.
-
-        Args:
-            value: A value to be set on the property.
-
-        Raises:
-            TypeError if the value is not an instance of Flow.
-        """
-        _LOGGER.info('validate: Got type %s', type(value))
-        if value is not None and not isinstance(value, client.Flow):
-            raise TypeError(
-                'Property {0} must be convertible to a flow '
-                'instance; received: {1}.'.format(self._name, value))
-
-
-class CredentialsNDBProperty(ndb.BlobProperty):
-    """App Engine NDB datastore Property for Credentials.
-
-    Serves the same purpose as the DB CredentialsProperty, but for NDB
-    models. Since CredentialsProperty stores data as a blob and this
-    inherits from BlobProperty, the data in the datastore will be the same
-    as in the DB case.
-
-    Utility property that allows easy storage and retrieval of Credentials
-    and subclasses.
-    """
-
-    def _validate(self, value):
-        """Validates a value as a proper credentials object.
-
-        Args:
-            value: A value to be set on the property.
-
-        Raises:
-            TypeError if the value is not an instance of Credentials.
-        """
-        _LOGGER.info('validate: Got type %s', type(value))
-        if value is not None and not isinstance(value, client.Credentials):
-            raise TypeError(
-                'Property {0} must be convertible to a credentials '
-                'instance; received: {1}.'.format(self._name, value))
-
-    def _to_base_type(self, value):
-        """Converts our validated value to a JSON serialized string.
-
-        Args:
-            value: A value to be set in the datastore.
-
-        Returns:
-            A JSON serialized version of the credential, else '' if value
-            is None.
-        """
-        if value is None:
-            return ''
-        else:
-            return value.to_json()
-
-    def _from_base_type(self, value):
-        """Converts our stored JSON string back to the desired type.
-
-        Args:
-            value: A value from the datastore to be converted to the
-                   desired type.
-
-        Returns:
-            A deserialized Credentials (or subclass) object, else None if
-            the value can't be parsed.
-        """
-        if not value:
-            return None
-        try:
-            # Uses the from_json method of the implied class of value
-            credentials = client.Credentials.new_from_json(value)
-        except ValueError:
-            credentials = None
-        return credentials
-
-
-class CredentialsNDBModel(ndb.Model):
-    """NDB Model for storage of OAuth 2.0 Credentials
-
-    Since this model uses the same kind as CredentialsModel and has a
-    property which can serialize and deserialize Credentials correctly, it
-    can be used interchangeably with a CredentialsModel to access, insert
-    and delete the same entities. This simply provides an NDB model for
-    interacting with the same data the DB model interacts with.
-
-    Storage of the model is keyed by the user.user_id().
-    """
-    credentials = CredentialsNDBProperty()
-
-    @classmethod
-    def _get_kind(cls):
-        """Return the kind name for this class."""
-        return 'CredentialsModel'

src/oauth2client/contrib/_metadata.py

@@ -1,118 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Provides helper methods for talking to the Compute Engine metadata server.
-
-See https://cloud.google.com/compute/docs/metadata
-"""
-
-import datetime
-import json
-import os
-
-from six.moves import http_client
-from six.moves.urllib import parse as urlparse
-
-from oauth2client import _helpers
-from oauth2client import client
-from oauth2client import transport
-
-
-METADATA_ROOT = 'http://{}/computeMetadata/v1/'.format(
-    os.getenv('GCE_METADATA_ROOT', 'metadata.google.internal'))
-METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
-
-
-def get(http, path, root=METADATA_ROOT, recursive=None):
-    """Fetch a resource from the metadata server.
-
-    Args:
-        http: an object to be used to make HTTP requests.
-        path: A string indicating the resource to retrieve. For example,
-            'instance/service-accounts/default'
-        root: A string indicating the full path to the metadata server root.
-        recursive: A boolean indicating whether to do a recursive query of
-            metadata. See
-            https://cloud.google.com/compute/docs/metadata#aggcontents
-
-    Returns:
-        A dictionary if the metadata server returns JSON, otherwise a string.
-
-    Raises:
-        http_client.HTTPException if an error corrured while
-        retrieving metadata.
-    """
-    url = urlparse.urljoin(root, path)
-    url = _helpers._add_query_parameter(url, 'recursive', recursive)
-
-    response, content = transport.request(
-        http, url, headers=METADATA_HEADERS)
-
-    if response.status == http_client.OK:
-        decoded = _helpers._from_bytes(content)
-        if response['content-type'] == 'application/json':
-            return json.loads(decoded)
-        else:
-            return decoded
-    else:
-        raise http_client.HTTPException(
-            'Failed to retrieve {0} from the Google Compute Engine'
-            'metadata service. Response:\n{1}'.format(url, response))
-
-
-def get_service_account_info(http, service_account='default'):
-    """Get information about a service account from the metadata server.
-
-    Args:
-        http: an object to be used to make HTTP requests.
-        service_account: An email specifying the service account for which to
-            look up information. Default will be information for the "default"
-            service account of the current compute engine instance.
-
-    Returns:
-         A dictionary with information about the specified service account,
-         for example:
-
-            {
-                'email': '...',
-                'scopes': ['scope', ...],
-                'aliases': ['default', '...']
-            }
-    """
-    return get(
-        http,
-        'instance/service-accounts/{0}/'.format(service_account),
-        recursive=True)
-
-
-def get_token(http, service_account='default'):
-    """Fetch an oauth token for the
-
-    Args:
-        http: an object to be used to make HTTP requests.
-        service_account: An email specifying the service account this token
-            should represent. Default will be a token for the "default" service
-            account of the current compute engine instance.
-
-    Returns:
-         A tuple of (access token, token expiration), where access token is the
-         access token as a string and token expiration is a datetime object
-         that indicates when the access token will expire.
-    """
-    token_json = get(
-        http,
-        'instance/service-accounts/{0}/token'.format(service_account))
-    token_expiry = client._UTCNOW() + datetime.timedelta(
-        seconds=token_json['expires_in'])
-    return token_json['access_token'], token_expiry

src/oauth2client/contrib/appengine.py

@@ -1,910 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for Google App Engine
-
-Utilities for making it easier to use OAuth 2.0 on Google App Engine.
-"""
-
-import cgi
-import json
-import logging
-import os
-import pickle
-import threading
-
-from google.appengine.api import app_identity
-from google.appengine.api import memcache
-from google.appengine.api import users
-from google.appengine.ext import db
-from google.appengine.ext.webapp.util import login_required
-import webapp2 as webapp
-
-import oauth2client
-from oauth2client import _helpers
-from oauth2client import client
-from oauth2client import clientsecrets
-from oauth2client import transport
-from oauth2client.contrib import xsrfutil
-
-# This is a temporary fix for a Google internal issue.
-try:
-    from oauth2client.contrib import _appengine_ndb
-except ImportError:  # pragma: NO COVER
-    _appengine_ndb = None
-
-
-logger = logging.getLogger(__name__)
-
-OAUTH2CLIENT_NAMESPACE = 'oauth2client#ns'
-
-XSRF_MEMCACHE_ID = 'xsrf_secret_key'
-
-if _appengine_ndb is None:  # pragma: NO COVER
-    CredentialsNDBModel = None
-    CredentialsNDBProperty = None
-    FlowNDBProperty = None
-    _NDB_KEY = None
-    _NDB_MODEL = None
-    SiteXsrfSecretKeyNDB = None
-else:
-    CredentialsNDBModel = _appengine_ndb.CredentialsNDBModel
-    CredentialsNDBProperty = _appengine_ndb.CredentialsNDBProperty
-    FlowNDBProperty = _appengine_ndb.FlowNDBProperty
-    _NDB_KEY = _appengine_ndb.NDB_KEY
-    _NDB_MODEL = _appengine_ndb.NDB_MODEL
-    SiteXsrfSecretKeyNDB = _appengine_ndb.SiteXsrfSecretKeyNDB
-
-
-def _safe_html(s):
-    """Escape text to make it safe to display.
-
-    Args:
-        s: string, The text to escape.
-
-    Returns:
-        The escaped text as a string.
-    """
-    return cgi.escape(s, quote=1).replace("'", ''')
-
-
-class SiteXsrfSecretKey(db.Model):
-    """Storage for the sites XSRF secret key.
-
-    There will only be one instance stored of this model, the one used for the
-    site.
-    """
-    secret = db.StringProperty()
-
-
-def _generate_new_xsrf_secret_key():
-    """Returns a random XSRF secret key."""
-    return os.urandom(16).encode("hex")
-
-
-def xsrf_secret_key():
-    """Return the secret key for use for XSRF protection.
-
-    If the Site entity does not have a secret key, this method will also create
-    one and persist it.
-
-    Returns:
-        The secret key.
-    """
-    secret = memcache.get(XSRF_MEMCACHE_ID, namespace=OAUTH2CLIENT_NAMESPACE)
-    if not secret:
-        # Load the one and only instance of SiteXsrfSecretKey.
-        model = SiteXsrfSecretKey.get_or_insert(key_name='site')
-        if not model.secret:
-            model.secret = _generate_new_xsrf_secret_key()
-            model.put()
-        secret = model.secret
-        memcache.add(XSRF_MEMCACHE_ID, secret,
-                     namespace=OAUTH2CLIENT_NAMESPACE)
-
-    return str(secret)
-
-
-class AppAssertionCredentials(client.AssertionCredentials):
-    """Credentials object for App Engine Assertion Grants
-
-    This object will allow an App Engine application to identify itself to
-    Google and other OAuth 2.0 servers that can verify assertions. It can be
-    used for the purpose of accessing data stored under an account assigned to
-    the App Engine application itself.
-
-    This credential does not require a flow to instantiate because it
-    represents a two legged flow, and therefore has all of the required
-    information to generate and refresh its own access tokens.
-    """
-
-    @_helpers.positional(2)
-    def __init__(self, scope, **kwargs):
-        """Constructor for AppAssertionCredentials
-
-        Args:
-            scope: string or iterable of strings, scope(s) of the credentials
-                   being requested.
-            **kwargs: optional keyword args, including:
-            service_account_id: service account id of the application. If None
-                                or unspecified, the default service account for
-                                the app is used.
-        """
-        self.scope = _helpers.scopes_to_string(scope)
-        self._kwargs = kwargs
-        self.service_account_id = kwargs.get('service_account_id', None)
-        self._service_account_email = None
-
-        # Assertion type is no longer used, but still in the
-        # parent class signature.
-        super(AppAssertionCredentials, self).__init__(None)
-
-    @classmethod
-    def from_json(cls, json_data):
-        data = json.loads(json_data)
-        return AppAssertionCredentials(data['scope'])
-
-    def _refresh(self, http):
-        """Refreshes the access token.
-
-        Since the underlying App Engine app_identity implementation does its
-        own caching we can skip all the storage hoops and just to a refresh
-        using the API.
-
-        Args:
-            http: unused HTTP object
-
-        Raises:
-            AccessTokenRefreshError: When the refresh fails.
-        """
-        try:
-            scopes = self.scope.split()
-            (token, _) = app_identity.get_access_token(
-                scopes, service_account_id=self.service_account_id)
-        except app_identity.Error as e:
-            raise client.AccessTokenRefreshError(str(e))
-        self.access_token = token
-
-    @property
-    def serialization_data(self):
-        raise NotImplementedError('Cannot serialize credentials '
-                                  'for Google App Engine.')
-
-    def create_scoped_required(self):
-        return not self.scope
-
-    def create_scoped(self, scopes):
-        return AppAssertionCredentials(scopes, **self._kwargs)
-
-    def sign_blob(self, blob):
-        """Cryptographically sign a blob (of bytes).
-
-        Implements abstract method
-        :meth:`oauth2client.client.AssertionCredentials.sign_blob`.
-
-        Args:
-            blob: bytes, Message to be signed.
-
-        Returns:
-            tuple, A pair of the private key ID used to sign the blob and
-            the signed contents.
-        """
-        return app_identity.sign_blob(blob)
-
-    @property
-    def service_account_email(self):
-        """Get the email for the current service account.
-
-        Returns:
-            string, The email associated with the Google App Engine
-            service account.
-        """
-        if self._service_account_email is None:
-            self._service_account_email = (
-                app_identity.get_service_account_name())
-        return self._service_account_email
-
-
-class FlowProperty(db.Property):
-    """App Engine datastore Property for Flow.
-
-    Utility property that allows easy storage and retrieval of an
-    oauth2client.Flow
-    """
-
-    # Tell what the user type is.
-    data_type = client.Flow
-
-    # For writing to datastore.
-    def get_value_for_datastore(self, model_instance):
-        flow = super(FlowProperty, self).get_value_for_datastore(
-            model_instance)
-        return db.Blob(pickle.dumps(flow))
-
-    # For reading from datastore.
-    def make_value_from_datastore(self, value):
-        if value is None:
-            return None
-        return pickle.loads(value)
-
-    def validate(self, value):
-        if value is not None and not isinstance(value, client.Flow):
-            raise db.BadValueError(
-                'Property {0} must be convertible '
-                'to a FlowThreeLegged instance ({1})'.format(self.name, value))
-        return super(FlowProperty, self).validate(value)
-
-    def empty(self, value):
-        return not value
-
-
-class CredentialsProperty(db.Property):
-    """App Engine datastore Property for Credentials.
-
-    Utility property that allows easy storage and retrieval of
-    oauth2client.Credentials
-    """
-
-    # Tell what the user type is.
-    data_type = client.Credentials
-
-    # For writing to datastore.
-    def get_value_for_datastore(self, model_instance):
-        logger.info("get: Got type " + str(type(model_instance)))
-        cred = super(CredentialsProperty, self).get_value_for_datastore(
-            model_instance)
-        if cred is None:
-            cred = ''
-        else:
-            cred = cred.to_json()
-        return db.Blob(cred)
-
-    # For reading from datastore.
-    def make_value_from_datastore(self, value):
-        logger.info("make: Got type " + str(type(value)))
-        if value is None:
-            return None
-        if len(value) == 0:
-            return None
-        try:
-            credentials = client.Credentials.new_from_json(value)
-        except ValueError:
-            credentials = None
-        return credentials
-
-    def validate(self, value):
-        value = super(CredentialsProperty, self).validate(value)
-        logger.info("validate: Got type " + str(type(value)))
-        if value is not None and not isinstance(value, client.Credentials):
-            raise db.BadValueError(
-                'Property {0} must be convertible '
-                'to a Credentials instance ({1})'.format(self.name, value))
-        return value
-
-
-class StorageByKeyName(client.Storage):
-    """Store and retrieve a credential to and from the App Engine datastore.
-
-    This Storage helper presumes the Credentials have been stored as a
-    CredentialsProperty or CredentialsNDBProperty on a datastore model class,
-    and that entities are stored by key_name.
-    """
-
-    @_helpers.positional(4)
-    def __init__(self, model, key_name, property_name, cache=None, user=None):
-        """Constructor for Storage.
-
-        Args:
-            model: db.Model or ndb.Model, model class
-            key_name: string, key name for the entity that has the credentials
-            property_name: string, name of the property that is a
-                           CredentialsProperty or CredentialsNDBProperty.
-            cache: memcache, a write-through cache to put in front of the
-                   datastore. If the model you are using is an NDB model, using
-                   a cache will be redundant since the model uses an instance
-                   cache and memcache for you.
-            user: users.User object, optional. Can be used to grab user ID as a
-                  key_name if no key name is specified.
-        """
-        super(StorageByKeyName, self).__init__()
-
-        if key_name is None:
-            if user is None:
-                raise ValueError('StorageByKeyName called with no '
-                                 'key name or user.')
-            key_name = user.user_id()
-
-        self._model = model
-        self._key_name = key_name
-        self._property_name = property_name
-        self._cache = cache
-
-    def _is_ndb(self):
-        """Determine whether the model of the instance is an NDB model.
-
-        Returns:
-            Boolean indicating whether or not the model is an NDB or DB model.
-        """
-        # issubclass will fail if one of the arguments is not a class, only
-        # need worry about new-style classes since ndb and db models are
-        # new-style
-        if isinstance(self._model, type):
-            if _NDB_MODEL is not None and issubclass(self._model, _NDB_MODEL):
-                return True
-            elif issubclass(self._model, db.Model):
-                return False
-
-        raise TypeError(
-            'Model class not an NDB or DB model: {0}.'.format(self._model))
-
-    def _get_entity(self):
-        """Retrieve entity from datastore.
-
-        Uses a different model method for db or ndb models.
-
-        Returns:
-            Instance of the model corresponding to the current storage object
-            and stored using the key name of the storage object.
-        """
-        if self._is_ndb():
-            return self._model.get_by_id(self._key_name)
-        else:
-            return self._model.get_by_key_name(self._key_name)
-
-    def _delete_entity(self):
-        """Delete entity from datastore.
-
-        Attempts to delete using the key_name stored on the object, whether or
-        not the given key is in the datastore.
-        """
-        if self._is_ndb():
-            _NDB_KEY(self._model, self._key_name).delete()
-        else:
-            entity_key = db.Key.from_path(self._model.kind(), self._key_name)
-            db.delete(entity_key)
-
-    @db.non_transactional(allow_existing=True)
-    def locked_get(self):
-        """Retrieve Credential from datastore.
-
-        Returns:
-            oauth2client.Credentials
-        """
-        credentials = None
-        if self._cache:
-            json = self._cache.get(self._key_name)
-            if json:
-                credentials = client.Credentials.new_from_json(json)
-        if credentials is None:
-            entity = self._get_entity()
-            if entity is not None:
-                credentials = getattr(entity, self._property_name)
-                if self._cache:
-                    self._cache.set(self._key_name, credentials.to_json())
-
-        if credentials and hasattr(credentials, 'set_store'):
-            credentials.set_store(self)
-        return credentials
-
-    @db.non_transactional(allow_existing=True)
-    def locked_put(self, credentials):
-        """Write a Credentials to the datastore.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-        """
-        entity = self._model.get_or_insert(self._key_name)
-        setattr(entity, self._property_name, credentials)
-        entity.put()
-        if self._cache:
-            self._cache.set(self._key_name, credentials.to_json())
-
-    @db.non_transactional(allow_existing=True)
-    def locked_delete(self):
-        """Delete Credential from datastore."""
-
-        if self._cache:
-            self._cache.delete(self._key_name)
-
-        self._delete_entity()
-
-
-class CredentialsModel(db.Model):
-    """Storage for OAuth 2.0 Credentials
-
-    Storage of the model is keyed by the user.user_id().
-    """
-    credentials = CredentialsProperty()
-
-
-def _build_state_value(request_handler, user):
-    """Composes the value for the 'state' parameter.
-
-    Packs the current request URI and an XSRF token into an opaque string that
-    can be passed to the authentication server via the 'state' parameter.
-
-    Args:
-        request_handler: webapp.RequestHandler, The request.
-        user: google.appengine.api.users.User, The current user.
-
-    Returns:
-        The state value as a string.
-    """
-    uri = request_handler.request.url
-    token = xsrfutil.generate_token(xsrf_secret_key(), user.user_id(),
-                                    action_id=str(uri))
-    return uri + ':' + token
-
-
-def _parse_state_value(state, user):
-    """Parse the value of the 'state' parameter.
-
-    Parses the value and validates the XSRF token in the state parameter.
-
-    Args:
-        state: string, The value of the state parameter.
-        user: google.appengine.api.users.User, The current user.
-
-    Returns:
-        The redirect URI, or None if XSRF token is not valid.
-    """
-    uri, token = state.rsplit(':', 1)
-    if xsrfutil.validate_token(xsrf_secret_key(), token, user.user_id(),
-                               action_id=uri):
-        return uri
-    else:
-        return None
-
-
-class OAuth2Decorator(object):
-    """Utility for making OAuth 2.0 easier.
-
-    Instantiate and then use with oauth_required or oauth_aware
-    as decorators on webapp.RequestHandler methods.
-
-    ::
-
-        decorator = OAuth2Decorator(
-            client_id='837...ent.com',
-            client_secret='Qh...wwI',
-            scope='https://www.googleapis.com/auth/plus')
-
-        class MainHandler(webapp.RequestHandler):
-            @decorator.oauth_required
-            def get(self):
-                http = decorator.http()
-                # http is authorized with the user's Credentials and can be
-                # used in API calls
-
-    """
-
-    def set_credentials(self, credentials):
-        self._tls.credentials = credentials
-
-    def get_credentials(self):
-        """A thread local Credentials object.
-
-        Returns:
-            A client.Credentials object, or None if credentials hasn't been set
-            in this thread yet, which may happen when calling has_credentials
-            inside oauth_aware.
-        """
-        return getattr(self._tls, 'credentials', None)
-
-    credentials = property(get_credentials, set_credentials)
-
-    def set_flow(self, flow):
-        self._tls.flow = flow
-
-    def get_flow(self):
-        """A thread local Flow object.
-
-        Returns:
-            A credentials.Flow object, or None if the flow hasn't been set in
-            this thread yet, which happens in _create_flow() since Flows are
-            created lazily.
-        """
-        return getattr(self._tls, 'flow', None)
-
-    flow = property(get_flow, set_flow)
-
-    @_helpers.positional(4)
-    def __init__(self, client_id, client_secret, scope,
-                 auth_uri=oauth2client.GOOGLE_AUTH_URI,
-                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
-                 user_agent=None,
-                 message=None,
-                 callback_path='/oauth2callback',
-                 token_response_param=None,
-                 _storage_class=StorageByKeyName,
-                 _credentials_class=CredentialsModel,
-                 _credentials_property_name='credentials',
-                 **kwargs):
-        """Constructor for OAuth2Decorator
-
-        Args:
-            client_id: string, client identifier.
-            client_secret: string client secret.
-            scope: string or iterable of strings, scope(s) of the credentials
-                   being requested.
-            auth_uri: string, URI for authorization endpoint. For convenience
-                      defaults to Google's endpoints but any OAuth 2.0 provider
-                      can be used.
-            token_uri: string, URI for token endpoint. For convenience defaults
-                       to Google's endpoints but any OAuth 2.0 provider can be
-                       used.
-            revoke_uri: string, URI for revoke endpoint. For convenience
-                        defaults to Google's endpoints but any OAuth 2.0
-                        provider can be used.
-            user_agent: string, User agent of your application, default to
-                        None.
-            message: Message to display if there are problems with the
-                     OAuth 2.0 configuration. The message may contain HTML and
-                     will be presented on the web interface for any method that
-                     uses the decorator.
-            callback_path: string, The absolute path to use as the callback
-                           URI. Note that this must match up with the URI given
-                           when registering the application in the APIs
-                           Console.
-            token_response_param: string. If provided, the full JSON response
-                                  to the access token request will be encoded
-                                  and included in this query parameter in the
-                                  callback URI. This is useful with providers
-                                  (e.g. wordpress.com) that include extra
-                                  fields that the client may want.
-            _storage_class: "Protected" keyword argument not typically provided
-                            to this constructor. A storage class to aid in
-                            storing a Credentials object for a user in the
-                            datastore. Defaults to StorageByKeyName.
-            _credentials_class: "Protected" keyword argument not typically
-                                provided to this constructor. A db or ndb Model
-                                class to hold credentials. Defaults to
-                                CredentialsModel.
-            _credentials_property_name: "Protected" keyword argument not
-                                        typically provided to this constructor.
-                                        A string indicating the name of the
-                                        field on the _credentials_class where a
-                                        Credentials object will be stored.
-                                        Defaults to 'credentials'.
-            **kwargs: dict, Keyword arguments are passed along as kwargs to
-                      the OAuth2WebServerFlow constructor.
-        """
-        self._tls = threading.local()
-        self.flow = None
-        self.credentials = None
-        self._client_id = client_id
-        self._client_secret = client_secret
-        self._scope = _helpers.scopes_to_string(scope)
-        self._auth_uri = auth_uri
-        self._token_uri = token_uri
-        self._revoke_uri = revoke_uri
-        self._user_agent = user_agent
-        self._kwargs = kwargs
-        self._message = message
-        self._in_error = False
-        self._callback_path = callback_path
-        self._token_response_param = token_response_param
-        self._storage_class = _storage_class
-        self._credentials_class = _credentials_class
-        self._credentials_property_name = _credentials_property_name
-
-    def _display_error_message(self, request_handler):
-        request_handler.response.out.write('<html><body>')
-        request_handler.response.out.write(_safe_html(self._message))
-        request_handler.response.out.write('</body></html>')
-
-    def oauth_required(self, method):
-        """Decorator that starts the OAuth 2.0 dance.
-
-        Starts the OAuth dance for the logged in user if they haven't already
-        granted access for this application.
-
-        Args:
-            method: callable, to be decorated method of a webapp.RequestHandler
-                    instance.
-        """
-
-        def check_oauth(request_handler, *args, **kwargs):
-            if self._in_error:
-                self._display_error_message(request_handler)
-                return
-
-            user = users.get_current_user()
-            # Don't use @login_decorator as this could be used in a
-            # POST request.
-            if not user:
-                request_handler.redirect(users.create_login_url(
-                    request_handler.request.uri))
-                return
-
-            self._create_flow(request_handler)
-
-            # Store the request URI in 'state' so we can use it later
-            self.flow.params['state'] = _build_state_value(
-                request_handler, user)
-            self.credentials = self._storage_class(
-                self._credentials_class, None,
-                self._credentials_property_name, user=user).get()
-
-            if not self.has_credentials():
-                return request_handler.redirect(self.authorize_url())
-            try:
-                resp = method(request_handler, *args, **kwargs)
-            except client.AccessTokenRefreshError:
-                return request_handler.redirect(self.authorize_url())
-            finally:
-                self.credentials = None
-            return resp
-
-        return check_oauth
-
-    def _create_flow(self, request_handler):
-        """Create the Flow object.
-
-        The Flow is calculated lazily since we don't know where this app is
-        running until it receives a request, at which point redirect_uri can be
-        calculated and then the Flow object can be constructed.
-
-        Args:
-            request_handler: webapp.RequestHandler, the request handler.
-        """
-        if self.flow is None:
-            redirect_uri = request_handler.request.relative_url(
-                self._callback_path)  # Usually /oauth2callback
-            self.flow = client.OAuth2WebServerFlow(
-                self._client_id, self._client_secret, self._scope,
-                redirect_uri=redirect_uri, user_agent=self._user_agent,
-                auth_uri=self._auth_uri, token_uri=self._token_uri,
-                revoke_uri=self._revoke_uri, **self._kwargs)
-
-    def oauth_aware(self, method):
-        """Decorator that sets up for OAuth 2.0 dance, but doesn't do it.
-
-        Does all the setup for the OAuth dance, but doesn't initiate it.
-        This decorator is useful if you want to create a page that knows
-        whether or not the user has granted access to this application.
-        From within a method decorated with @oauth_aware the has_credentials()
-        and authorize_url() methods can be called.
-
-        Args:
-            method: callable, to be decorated method of a webapp.RequestHandler
-                    instance.
-        """
-
-        def setup_oauth(request_handler, *args, **kwargs):
-            if self._in_error:
-                self._display_error_message(request_handler)
-                return
-
-            user = users.get_current_user()
-            # Don't use @login_decorator as this could be used in a
-            # POST request.
-            if not user:
-                request_handler.redirect(users.create_login_url(
-                    request_handler.request.uri))
-                return
-
-            self._create_flow(request_handler)
-
-            self.flow.params['state'] = _build_state_value(request_handler,
-                                                           user)
-            self.credentials = self._storage_class(
-                self._credentials_class, None,
-                self._credentials_property_name, user=user).get()
-            try:
-                resp = method(request_handler, *args, **kwargs)
-            finally:
-                self.credentials = None
-            return resp
-        return setup_oauth
-
-    def has_credentials(self):
-        """True if for the logged in user there are valid access Credentials.
-
-        Must only be called from with a webapp.RequestHandler subclassed method
-        that had been decorated with either @oauth_required or @oauth_aware.
-        """
-        return self.credentials is not None and not self.credentials.invalid
-
-    def authorize_url(self):
-        """Returns the URL to start the OAuth dance.
-
-        Must only be called from with a webapp.RequestHandler subclassed method
-        that had been decorated with either @oauth_required or @oauth_aware.
-        """
-        url = self.flow.step1_get_authorize_url()
-        return str(url)
-
-    def http(self, *args, **kwargs):
-        """Returns an authorized http instance.
-
-        Must only be called from within an @oauth_required decorated method, or
-        from within an @oauth_aware decorated method where has_credentials()
-        returns True.
-
-        Args:
-            *args: Positional arguments passed to httplib2.Http constructor.
-            **kwargs: Positional arguments passed to httplib2.Http constructor.
-        """
-        return self.credentials.authorize(
-            transport.get_http_object(*args, **kwargs))
-
-    @property
-    def callback_path(self):
-        """The absolute path where the callback will occur.
-
-        Note this is the absolute path, not the absolute URI, that will be
-        calculated by the decorator at runtime. See callback_handler() for how
-        this should be used.
-
-        Returns:
-            The callback path as a string.
-        """
-        return self._callback_path
-
-    def callback_handler(self):
-        """RequestHandler for the OAuth 2.0 redirect callback.
-
-        Usage::
-
-            app = webapp.WSGIApplication([
-                ('/index', MyIndexHandler),
-                ...,
-                (decorator.callback_path, decorator.callback_handler())
-            ])
-
-        Returns:
-            A webapp.RequestHandler that handles the redirect back from the
-            server during the OAuth 2.0 dance.
-        """
-        decorator = self
-
-        class OAuth2Handler(webapp.RequestHandler):
-            """Handler for the redirect_uri of the OAuth 2.0 dance."""
-
-            @login_required
-            def get(self):
-                error = self.request.get('error')
-                if error:
-                    errormsg = self.request.get('error_description', error)
-                    self.response.out.write(
-                        'The authorization request failed: {0}'.format(
-                            _safe_html(errormsg)))
-                else:
-                    user = users.get_current_user()
-                    decorator._create_flow(self)
-                    credentials = decorator.flow.step2_exchange(
-                        self.request.params)
-                    decorator._storage_class(
-                        decorator._credentials_class, None,
-                        decorator._credentials_property_name,
-                        user=user).put(credentials)
-                    redirect_uri = _parse_state_value(
-                        str(self.request.get('state')), user)
-                    if redirect_uri is None:
-                        self.response.out.write(
-                            'The authorization request failed')
-                        return
-
-                    if (decorator._token_response_param and
-                            credentials.token_response):
-                        resp_json = json.dumps(credentials.token_response)
-                        redirect_uri = _helpers._add_query_parameter(
-                            redirect_uri, decorator._token_response_param,
-                            resp_json)
-
-                    self.redirect(redirect_uri)
-
-        return OAuth2Handler
-
-    def callback_application(self):
-        """WSGI application for handling the OAuth 2.0 redirect callback.
-
-        If you need finer grained control use `callback_handler` which returns
-        just the webapp.RequestHandler.
-
-        Returns:
-            A webapp.WSGIApplication that handles the redirect back from the
-            server during the OAuth 2.0 dance.
-        """
-        return webapp.WSGIApplication([
-            (self.callback_path, self.callback_handler())
-        ])
-
-
-class OAuth2DecoratorFromClientSecrets(OAuth2Decorator):
-    """An OAuth2Decorator that builds from a clientsecrets file.
-
-    Uses a clientsecrets file as the source for all the information when
-    constructing an OAuth2Decorator.
-
-    ::
-
-        decorator = OAuth2DecoratorFromClientSecrets(
-            os.path.join(os.path.dirname(__file__), 'client_secrets.json')
-            scope='https://www.googleapis.com/auth/plus')
-
-        class MainHandler(webapp.RequestHandler):
-            @decorator.oauth_required
-            def get(self):
-                http = decorator.http()
-                # http is authorized with the user's Credentials and can be
-                # used in API calls
-
-    """
-
-    @_helpers.positional(3)
-    def __init__(self, filename, scope, message=None, cache=None, **kwargs):
-        """Constructor
-
-        Args:
-            filename: string, File name of client secrets.
-            scope: string or iterable of strings, scope(s) of the credentials
-                   being requested.
-            message: string, A friendly string to display to the user if the
-                     clientsecrets file is missing or invalid. The message may
-                     contain HTML and will be presented on the web interface
-                     for any method that uses the decorator.
-            cache: An optional cache service client that implements get() and
-                   set()
-            methods. See clientsecrets.loadfile() for details.
-            **kwargs: dict, Keyword arguments are passed along as kwargs to
-                      the OAuth2WebServerFlow constructor.
-        """
-        client_type, client_info = clientsecrets.loadfile(filename,
-                                                          cache=cache)
-        if client_type not in (clientsecrets.TYPE_WEB,
-                               clientsecrets.TYPE_INSTALLED):
-            raise clientsecrets.InvalidClientSecretsError(
-                "OAuth2Decorator doesn't support this OAuth 2.0 flow.")
-
-        constructor_kwargs = dict(kwargs)
-        constructor_kwargs.update({
-            'auth_uri': client_info['auth_uri'],
-            'token_uri': client_info['token_uri'],
-            'message': message,
-        })
-        revoke_uri = client_info.get('revoke_uri')
-        if revoke_uri is not None:
-            constructor_kwargs['revoke_uri'] = revoke_uri
-        super(OAuth2DecoratorFromClientSecrets, self).__init__(
-            client_info['client_id'], client_info['client_secret'],
-            scope, **constructor_kwargs)
-        if message is not None:
-            self._message = message
-        else:
-            self._message = 'Please configure your application for OAuth 2.0.'
-
-
-@_helpers.positional(2)
-def oauth2decorator_from_clientsecrets(filename, scope,
-                                       message=None, cache=None):
-    """Creates an OAuth2Decorator populated from a clientsecrets file.
-
-    Args:
-        filename: string, File name of client secrets.
-        scope: string or list of strings, scope(s) of the credentials being
-               requested.
-        message: string, A friendly string to display to the user if the
-                 clientsecrets file is missing or invalid. The message may
-                 contain HTML and will be presented on the web interface for
-                 any method that uses the decorator.
-        cache: An optional cache service client that implements get() and set()
-               methods. See clientsecrets.loadfile() for details.
-
-    Returns: An OAuth2Decorator
-    """
-    return OAuth2DecoratorFromClientSecrets(filename, scope,
-                                            message=message, cache=cache)

src/oauth2client/contrib/devshell.py

@@ -1,152 +0,0 @@
-# Copyright 2015 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""OAuth 2.0 utitilies for Google Developer Shell environment."""
-
-import datetime
-import json
-import os
-import socket
-
-from oauth2client import _helpers
-from oauth2client import client
-
-DEVSHELL_ENV = 'DEVSHELL_CLIENT_PORT'
-
-
-class Error(Exception):
-    """Errors for this module."""
-    pass
-
-
-class CommunicationError(Error):
-    """Errors for communication with the Developer Shell server."""
-
-
-class NoDevshellServer(Error):
-    """Error when no Developer Shell server can be contacted."""
-
-
-# The request for credential information to the Developer Shell client socket
-# is always an empty PBLite-formatted JSON object, so just define it as a
-# constant.
-CREDENTIAL_INFO_REQUEST_JSON = '[]'
-
-
-class CredentialInfoResponse(object):
-    """Credential information response from Developer Shell server.
-
-    The credential information response from Developer Shell socket is a
-    PBLite-formatted JSON array with fields encoded by their index in the
-    array:
-
-    * Index 0 - user email
-    * Index 1 - default project ID. None if the project context is not known.
-    * Index 2 - OAuth2 access token. None if there is no valid auth context.
-    * Index 3 - Seconds until the access token expires. None if not present.
-    """
-
-    def __init__(self, json_string):
-        """Initialize the response data from JSON PBLite array."""
-        pbl = json.loads(json_string)
-        if not isinstance(pbl, list):
-            raise ValueError('Not a list: ' + str(pbl))
-        pbl_len = len(pbl)
-        self.user_email = pbl[0] if pbl_len > 0 else None
-        self.project_id = pbl[1] if pbl_len > 1 else None
-        self.access_token = pbl[2] if pbl_len > 2 else None
-        self.expires_in = pbl[3] if pbl_len > 3 else None
-
-
-def _SendRecv():
-    """Communicate with the Developer Shell server socket."""
-
-    port = int(os.getenv(DEVSHELL_ENV, 0))
-    if port == 0:
-        raise NoDevshellServer()
-
-    sock = socket.socket()
-    sock.connect(('localhost', port))
-
-    data = CREDENTIAL_INFO_REQUEST_JSON
-    msg = '{0}\n{1}'.format(len(data), data)
-    sock.sendall(_helpers._to_bytes(msg, encoding='utf-8'))
-
-    header = sock.recv(6).decode()
-    if '\n' not in header:
-        raise CommunicationError('saw no newline in the first 6 bytes')
-    len_str, json_str = header.split('\n', 1)
-    to_read = int(len_str) - len(json_str)
-    if to_read > 0:
-        json_str += sock.recv(to_read, socket.MSG_WAITALL).decode()
-
-    return CredentialInfoResponse(json_str)
-
-
-class DevshellCredentials(client.GoogleCredentials):
-    """Credentials object for Google Developer Shell environment.
-
-    This object will allow a Google Developer Shell session to identify its
-    user to Google and other OAuth 2.0 servers that can verify assertions. It
-    can be used for the purpose of accessing data stored under the user
-    account.
-
-    This credential does not require a flow to instantiate because it
-    represents a two legged flow, and therefore has all of the required
-    information to generate and refresh its own access tokens.
-    """
-
-    def __init__(self, user_agent=None):
-        super(DevshellCredentials, self).__init__(
-            None,  # access_token, initialized below
-            None,  # client_id
-            None,  # client_secret
-            None,  # refresh_token
-            None,  # token_expiry
-            None,  # token_uri
-            user_agent)
-        self._refresh(None)
-
-    def _refresh(self, http):
-        """Refreshes the access token.
-
-        Args:
-            http: unused HTTP object
-        """
-        self.devshell_response = _SendRecv()
-        self.access_token = self.devshell_response.access_token
-        expires_in = self.devshell_response.expires_in
-        if expires_in is not None:
-            delta = datetime.timedelta(seconds=expires_in)
-            self.token_expiry = client._UTCNOW() + delta
-        else:
-            self.token_expiry = None
-
-    @property
-    def user_email(self):
-        return self.devshell_response.user_email
-
-    @property
-    def project_id(self):
-        return self.devshell_response.project_id
-
-    @classmethod
-    def from_json(cls, json_data):
-        raise NotImplementedError(
-            'Cannot load Developer Shell credentials from JSON.')
-
-    @property
-    def serialization_data(self):
-        raise NotImplementedError(
-            'Cannot serialize Developer Shell credentials.')

src/oauth2client/contrib/dictionary_storage.py

@@ -1,65 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Dictionary storage for OAuth2 Credentials."""
-
-from oauth2client import client
-
-
-class DictionaryStorage(client.Storage):
-    """Store and retrieve credentials to and from a dictionary-like object.
-
-    Args:
-        dictionary: A dictionary or dictionary-like object.
-        key: A string or other hashable. The credentials will be stored in
-             ``dictionary[key]``.
-        lock: An optional threading.Lock-like object. The lock will be
-              acquired before anything is written or read from the
-              dictionary.
-    """
-
-    def __init__(self, dictionary, key, lock=None):
-        """Construct a DictionaryStorage instance."""
-        super(DictionaryStorage, self).__init__(lock=lock)
-        self._dictionary = dictionary
-        self._key = key
-
-    def locked_get(self):
-        """Retrieve the credentials from the dictionary, if they exist.
-
-        Returns: A :class:`oauth2client.client.OAuth2Credentials` instance.
-        """
-        serialized = self._dictionary.get(self._key)
-
-        if serialized is None:
-            return None
-
-        credentials = client.OAuth2Credentials.from_json(serialized)
-        credentials.set_store(self)
-
-        return credentials
-
-    def locked_put(self, credentials):
-        """Save the credentials to the dictionary.
-
-        Args:
-            credentials: A :class:`oauth2client.client.OAuth2Credentials`
-                         instance.
-        """
-        serialized = credentials.to_json()
-        self._dictionary[self._key] = serialized
-
-    def locked_delete(self):
-        """Remove the credentials from the dictionary, if they exist."""
-        self._dictionary.pop(self._key, None)

src/oauth2client/contrib/django_util/__init__.py

@@ -1,489 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for the Django web framework.
-
-Provides Django views and helpers the make using the OAuth2 web server
-flow easier. It includes an ``oauth_required`` decorator to automatically
-ensure that user credentials are available, and an ``oauth_enabled`` decorator
-to check if the user has authorized, and helper shortcuts to create the
-authorization URL otherwise.
-
-There are two basic use cases supported. The first is using Google OAuth as the
-primary form of authentication, which is the simpler approach recommended
-for applications without their own user system.
-
-The second use case is adding Google OAuth credentials to an
-existing Django model containing a Django user field. Most of the
-configuration is the same, except for `GOOGLE_OAUTH_MODEL_STORAGE` in
-settings.py. See "Adding Credentials To An Existing Django User System" for
-usage differences.
-
-Only Django versions 1.8+ are supported.
-
-Configuration
-===============
-
-To configure, you'll need a set of OAuth2 web application credentials from
-`Google Developer's Console <https://console.developers.google.com/project/_/apiui/credential>`.
-
-Add the helper to your INSTALLED_APPS:
-
-.. code-block:: python
-   :caption: settings.py
-   :name: installed_apps
-
-    INSTALLED_APPS = (
-        # other apps
-        "django.contrib.sessions.middleware"
-        "oauth2client.contrib.django_util"
-    )
-
-This helper also requires the Django Session Middleware, so
-``django.contrib.sessions.middleware`` should be in INSTALLED_APPS as well.
-MIDDLEWARE or MIDDLEWARE_CLASSES (in Django  versions <1.10) should also
-contain the string 'django.contrib.sessions.middleware.SessionMiddleware'.
-
-
-Add the client secrets created earlier to the settings. You can either
-specify the path to the credentials file in JSON format
-
-.. code-block:: python
-   :caption:  settings.py
-   :name: secrets_file
-
-   GOOGLE_OAUTH2_CLIENT_SECRETS_JSON=/path/to/client-secret.json
-
-Or, directly configure the client Id and client secret.
-
-
-.. code-block:: python
-   :caption: settings.py
-   :name: secrets_config
-
-   GOOGLE_OAUTH2_CLIENT_ID=client-id-field
-   GOOGLE_OAUTH2_CLIENT_SECRET=client-secret-field
-
-By default, the default scopes for the required decorator only contains the
-``email`` scopes. You can change that default in the settings.
-
-.. code-block:: python
-   :caption: settings.py
-   :name: scopes
-
-   GOOGLE_OAUTH2_SCOPES = ('email', 'https://www.googleapis.com/auth/calendar',)
-
-By default, the decorators will add an `oauth` object to the Django request
-object, and include all of its state and helpers inside that object. If the
-`oauth` name conflicts with another usage, it can be changed
-
-.. code-block:: python
-   :caption: settings.py
-   :name: request_prefix
-
-   # changes request.oauth to request.google_oauth
-   GOOGLE_OAUTH2_REQUEST_ATTRIBUTE = 'google_oauth'
-
-Add the oauth2 routes to your application's urls.py urlpatterns.
-
-.. code-block:: python
-   :caption: urls.py
-   :name: urls
-
-   from oauth2client.contrib.django_util.site import urls as oauth2_urls
-
-   urlpatterns += [url(r'^oauth2/', include(oauth2_urls))]
-
-To require OAuth2 credentials for a view, use the `oauth2_required` decorator.
-This creates a credentials object with an id_token, and allows you to create
-an `http` object to build service clients with. These are all attached to the
-request.oauth
-
-.. code-block:: python
-   :caption: views.py
-   :name: views_required
-
-   from oauth2client.contrib.django_util.decorators import oauth_required
-
-   @oauth_required
-   def requires_default_scopes(request):
-      email = request.oauth.credentials.id_token['email']
-      service = build(serviceName='calendar', version='v3',
-                    http=request.oauth.http,
-                   developerKey=API_KEY)
-      events = service.events().list(calendarId='primary').execute()['items']
-      return HttpResponse("email: {0} , calendar: {1}".format(
-                           email,str(events)))
-      return HttpResponse(
-          "email: {0} , calendar: {1}".format(email, str(events)))
-
-To make OAuth2 optional and provide an authorization link in your own views.
-
-.. code-block:: python
-   :caption: views.py
-   :name: views_enabled2
-
-   from oauth2client.contrib.django_util.decorators import oauth_enabled
-
-   @oauth_enabled
-   def optional_oauth2(request):
-       if request.oauth.has_credentials():
-           # this could be passed into a view
-           # request.oauth.http is also initialized
-           return HttpResponse("User email: {0}".format(
-               request.oauth.credentials.id_token['email']))
-       else:
-           return HttpResponse(
-               'Here is an OAuth Authorize link: <a href="{0}">Authorize'
-               '</a>'.format(request.oauth.get_authorize_redirect()))
-
-If a view needs a scope not included in the default scopes specified in
-the settings, you can use [incremental auth](https://developers.google.com/identity/sign-in/web/incremental-auth)
-and specify additional scopes in the decorator arguments.
-
-.. code-block:: python
-   :caption: views.py
-   :name: views_required_additional_scopes
-
-   @oauth_enabled(scopes=['https://www.googleapis.com/auth/drive'])
-   def drive_required(request):
-       if request.oauth.has_credentials():
-           service = build(serviceName='drive', version='v2',
-                http=request.oauth.http,
-                developerKey=API_KEY)
-           events = service.files().list().execute()['items']
-           return HttpResponse(str(events))
-       else:
-           return HttpResponse(
-               'Here is an OAuth Authorize link: <a href="{0}">Authorize'
-               '</a>'.format(request.oauth.get_authorize_redirect()))
-
-
-To provide a callback on authorization being completed, use the
-oauth2_authorized signal:
-
-.. code-block:: python
-   :caption: views.py
-   :name: signals
-
-   from oauth2client.contrib.django_util.signals import oauth2_authorized
-
-   def test_callback(sender, request, credentials, **kwargs):
-       print("Authorization Signal Received {0}".format(
-               credentials.id_token['email']))
-
-   oauth2_authorized.connect(test_callback)
-
-Adding Credentials To An Existing Django User System
-=====================================================
-
-As an alternative to storing the credentials in the session, the helper
-can be configured to store the fields on a Django model. This might be useful
-if you need to use the credentials outside the context of a user request. It
-also prevents the need for a logged in user to repeat the OAuth flow when
-starting a new session.
-
-To use, change ``settings.py``
-
-.. code-block:: python
-   :caption:  settings.py
-   :name: storage_model_config
-
-   GOOGLE_OAUTH2_STORAGE_MODEL = {
-       'model': 'path.to.model.MyModel',
-       'user_property': 'user_id',
-       'credentials_property': 'credential'
-    }
-
-Where ``path.to.model`` class is the fully qualified name of a
-``django.db.model`` class containing a ``django.contrib.auth.models.User``
-field with the name specified by `user_property` and a
-:class:`oauth2client.contrib.django_util.models.CredentialsField` with the name
-specified by `credentials_property`. For the sample configuration given,
-our model would look like
-
-.. code-block:: python
-   :caption: models.py
-   :name: storage_model_model
-
-   from django.contrib.auth.models import User
-   from oauth2client.contrib.django_util.models import CredentialsField
-
-   class MyModel(models.Model):
-       #  ... other fields here ...
-       user = models.OneToOneField(User)
-       credential = CredentialsField()
-"""
-
-import importlib
-
-import django.conf
-from django.core import exceptions
-from django.core import urlresolvers
-from six.moves.urllib import parse
-
-from oauth2client import clientsecrets
-from oauth2client import transport
-from oauth2client.contrib import dictionary_storage
-from oauth2client.contrib.django_util import storage
-
-GOOGLE_OAUTH2_DEFAULT_SCOPES = ('email',)
-GOOGLE_OAUTH2_REQUEST_ATTRIBUTE = 'oauth'
-
-
-def _load_client_secrets(filename):
-    """Loads client secrets from the given filename.
-
-    Args:
-        filename: The name of the file containing the JSON secret key.
-
-    Returns:
-        A 2-tuple, the first item containing the client id, and the second
-        item containing a client secret.
-    """
-    client_type, client_info = clientsecrets.loadfile(filename)
-
-    if client_type != clientsecrets.TYPE_WEB:
-        raise ValueError(
-            'The flow specified in {} is not supported, only the WEB flow '
-            'type  is supported.'.format(client_type))
-    return client_info['client_id'], client_info['client_secret']
-
-
-def _get_oauth2_client_id_and_secret(settings_instance):
-    """Initializes client id and client secret based on the settings.
-
-    Args:
-        settings_instance: An instance of ``django.conf.settings``.
-
-    Returns:
-        A 2-tuple, the first item is the client id and the second
-         item is the client secret.
-    """
-    secret_json = getattr(settings_instance,
-                          'GOOGLE_OAUTH2_CLIENT_SECRETS_JSON', None)
-    if secret_json is not None:
-        return _load_client_secrets(secret_json)
-    else:
-        client_id = getattr(settings_instance, "GOOGLE_OAUTH2_CLIENT_ID",
-                            None)
-        client_secret = getattr(settings_instance,
-                                "GOOGLE_OAUTH2_CLIENT_SECRET", None)
-        if client_id is not None and client_secret is not None:
-            return client_id, client_secret
-        else:
-            raise exceptions.ImproperlyConfigured(
-                "Must specify either GOOGLE_OAUTH2_CLIENT_SECRETS_JSON, or "
-                "both GOOGLE_OAUTH2_CLIENT_ID and "
-                "GOOGLE_OAUTH2_CLIENT_SECRET in settings.py")
-
-
-def _get_storage_model():
-    """This configures whether the credentials will be stored in the session
-    or the Django ORM based on the settings. By default, the credentials
-    will be stored in the session, unless `GOOGLE_OAUTH2_STORAGE_MODEL`
-    is found in the settings. Usually, the ORM storage is used to integrate
-    credentials into an existing Django user system.
-
-    Returns:
-        A tuple containing three strings, or None. If
-        ``GOOGLE_OAUTH2_STORAGE_MODEL`` is configured, the tuple
-        will contain the fully qualifed path of the `django.db.model`,
-        the name of the ``django.contrib.auth.models.User`` field on the
-        model, and the name of the
-        :class:`oauth2client.contrib.django_util.models.CredentialsField`
-        field on the model. If Django ORM storage is not configured,
-        this function returns None.
-    """
-    storage_model_settings = getattr(django.conf.settings,
-                                     'GOOGLE_OAUTH2_STORAGE_MODEL', None)
-    if storage_model_settings is not None:
-        return (storage_model_settings['model'],
-                storage_model_settings['user_property'],
-                storage_model_settings['credentials_property'])
-    else:
-        return None, None, None
-
-
-class OAuth2Settings(object):
-    """Initializes Django OAuth2 Helper Settings
-
-    This class loads the OAuth2 Settings from the Django settings, and then
-    provides those settings as attributes to the rest of the views and
-    decorators in the module.
-
-    Attributes:
-      scopes: A list of OAuth2 scopes that the decorators and views will use
-              as defaults.
-      request_prefix: The name of the attribute that the decorators use to
-                    attach the UserOAuth2 object to the Django request object.
-      client_id: The OAuth2 Client ID.
-      client_secret: The OAuth2 Client Secret.
-    """
-
-    def __init__(self, settings_instance):
-        self.scopes = getattr(settings_instance, 'GOOGLE_OAUTH2_SCOPES',
-                              GOOGLE_OAUTH2_DEFAULT_SCOPES)
-        self.request_prefix = getattr(settings_instance,
-                                      'GOOGLE_OAUTH2_REQUEST_ATTRIBUTE',
-                                      GOOGLE_OAUTH2_REQUEST_ATTRIBUTE)
-        info = _get_oauth2_client_id_and_secret(settings_instance)
-        self.client_id, self.client_secret = info
-
-        # Django 1.10 deprecated MIDDLEWARE_CLASSES in favor of MIDDLEWARE
-        middleware_settings = getattr(settings_instance, 'MIDDLEWARE', None)
-        if middleware_settings is None:
-            middleware_settings = getattr(
-                settings_instance, 'MIDDLEWARE_CLASSES', None)
-        if middleware_settings is None:
-            raise exceptions.ImproperlyConfigured(
-                'Django settings has neither MIDDLEWARE nor MIDDLEWARE_CLASSES'
-                'configured')
-
-        if ('django.contrib.sessions.middleware.SessionMiddleware' not in
-                middleware_settings):
-            raise exceptions.ImproperlyConfigured(
-                'The Google OAuth2 Helper requires session middleware to '
-                'be installed. Edit your MIDDLEWARE_CLASSES or MIDDLEWARE '
-                'setting to include \'django.contrib.sessions.middleware.'
-                'SessionMiddleware\'.')
-        (self.storage_model, self.storage_model_user_property,
-         self.storage_model_credentials_property) = _get_storage_model()
-
-
-oauth2_settings = OAuth2Settings(django.conf.settings)
-
-_CREDENTIALS_KEY = 'google_oauth2_credentials'
-
-
-def get_storage(request):
-    """ Gets a Credentials storage object provided by the Django OAuth2 Helper
-    object.
-
-    Args:
-        request: Reference to the current request object.
-
-    Returns:
-       An :class:`oauth2.client.Storage` object.
-    """
-    storage_model = oauth2_settings.storage_model
-    user_property = oauth2_settings.storage_model_user_property
-    credentials_property = oauth2_settings.storage_model_credentials_property
-
-    if storage_model:
-        module_name, class_name = storage_model.rsplit('.', 1)
-        module = importlib.import_module(module_name)
-        storage_model_class = getattr(module, class_name)
-        return storage.DjangoORMStorage(storage_model_class,
-                                        user_property,
-                                        request.user,
-                                        credentials_property)
-    else:
-        # use session
-        return dictionary_storage.DictionaryStorage(
-            request.session, key=_CREDENTIALS_KEY)
-
-
-def _redirect_with_params(url_name, *args, **kwargs):
-    """Helper method to create a redirect response with URL params.
-
-    This builds a redirect string that converts kwargs into a
-    query string.
-
-    Args:
-        url_name: The name of the url to redirect to.
-        kwargs: the query string param and their values to build.
-
-    Returns:
-        A properly formatted redirect string.
-    """
-    url = urlresolvers.reverse(url_name, args=args)
-    params = parse.urlencode(kwargs, True)
-    return "{0}?{1}".format(url, params)
-
-
-def _credentials_from_request(request):
-    """Gets the authorized credentials for this flow, if they exist."""
-    # ORM storage requires a logged in user
-    if (oauth2_settings.storage_model is None or
-            request.user.is_authenticated()):
-        return get_storage(request).get()
-    else:
-        return None
-
-
-class UserOAuth2(object):
-    """Class to create oauth2 objects on Django request objects containing
-    credentials and helper methods.
-    """
-
-    def __init__(self, request, scopes=None, return_url=None):
-        """Initialize the Oauth2 Object.
-
-        Args:
-            request: Django request object.
-            scopes: Scopes desired for this OAuth2 flow.
-            return_url: The url to return to after the OAuth flow is complete,
-                 defaults to the request's current URL path.
-        """
-        self.request = request
-        self.return_url = return_url or request.get_full_path()
-        if scopes:
-            self._scopes = set(oauth2_settings.scopes) | set(scopes)
-        else:
-            self._scopes = set(oauth2_settings.scopes)
-
-    def get_authorize_redirect(self):
-        """Creates a URl to start the OAuth2 authorization flow."""
-        get_params = {
-            'return_url': self.return_url,
-            'scopes': self._get_scopes()
-        }
-
-        return _redirect_with_params('google_oauth:authorize', **get_params)
-
-    def has_credentials(self):
-        """Returns True if there are valid credentials for the current user
-        and required scopes."""
-        credentials = _credentials_from_request(self.request)
-        return (credentials and not credentials.invalid and
-                credentials.has_scopes(self._get_scopes()))
-
-    def _get_scopes(self):
-        """Returns the scopes associated with this object, kept up to
-         date for incremental auth."""
-        if _credentials_from_request(self.request):
-            return (self._scopes |
-                    _credentials_from_request(self.request).scopes)
-        else:
-            return self._scopes
-
-    @property
-    def scopes(self):
-        """Returns the scopes associated with this OAuth2 object."""
-        # make sure previously requested custom scopes are maintained
-        # in future authorizations
-        return self._get_scopes()
-
-    @property
-    def credentials(self):
-        """Gets the authorized credentials for this flow, if they exist."""
-        return _credentials_from_request(self.request)
-
-    @property
-    def http(self):
-        """Helper: create HTTP client authorized with OAuth2 credentials."""
-        if self.has_credentials():
-            return self.credentials.authorize(transport.get_http_object())
-        return None

src/oauth2client/contrib/django_util/apps.py

@@ -1,32 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Application Config For Django OAuth2 Helper.
-
-Django 1.7+ provides an
-[applications](https://docs.djangoproject.com/en/1.8/ref/applications/)
-API so that Django projects can introspect on installed applications using a
-stable API. This module exists to follow that convention.
-"""
-
-import sys
-
-# Django 1.7+ only supports Python 2.7+
-if sys.hexversion >= 0x02070000:  # pragma: NO COVER
-    from django.apps import AppConfig
-
-    class GoogleOAuth2HelperConfig(AppConfig):
-        """ App Config for Django Helper"""
-        name = 'oauth2client.django_util'
-        verbose_name = "Google OAuth2 Django Helper"

src/oauth2client/contrib/django_util/decorators.py

@@ -1,145 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Decorators for Django OAuth2 Flow.
-
-Contains two decorators, ``oauth_required`` and ``oauth_enabled``.
-
-``oauth_required`` will ensure that a user has an oauth object containing
-credentials associated with the request, and if not, redirect to the
-authorization flow.
-
-``oauth_enabled`` will attach the oauth2 object containing credentials if it
-exists. If it doesn't, the view will still render, but helper methods will be
-attached to start the oauth2 flow.
-"""
-
-from django import shortcuts
-import django.conf
-from six import wraps
-from six.moves.urllib import parse
-
-from oauth2client.contrib import django_util
-
-
-def oauth_required(decorated_function=None, scopes=None, **decorator_kwargs):
-    """ Decorator to require OAuth2 credentials for a view.
-
-
-    .. code-block:: python
-       :caption: views.py
-       :name: views_required_2
-
-
-       from oauth2client.django_util.decorators import oauth_required
-
-       @oauth_required
-       def requires_default_scopes(request):
-          email = request.credentials.id_token['email']
-          service = build(serviceName='calendar', version='v3',
-                       http=request.oauth.http,
-                       developerKey=API_KEY)
-          events = service.events().list(
-                                    calendarId='primary').execute()['items']
-          return HttpResponse(
-              "email: {0}, calendar: {1}".format(email, str(events)))
-
-    Args:
-        decorated_function: View function to decorate, must have the Django
-           request object as the first argument.
-        scopes: Scopes to require, will default.
-        decorator_kwargs: Can include ``return_url`` to specify the URL to
-           return to after OAuth2 authorization is complete.
-
-    Returns:
-        An OAuth2 Authorize view if credentials are not found or if the
-        credentials are missing the required scopes. Otherwise,
-        the decorated view.
-    """
-    def curry_wrapper(wrapped_function):
-        @wraps(wrapped_function)
-        def required_wrapper(request, *args, **kwargs):
-            if not (django_util.oauth2_settings.storage_model is None or
-                    request.user.is_authenticated()):
-                redirect_str = '{0}?next={1}'.format(
-                    django.conf.settings.LOGIN_URL,
-                    parse.quote(request.path))
-                return shortcuts.redirect(redirect_str)
-
-            return_url = decorator_kwargs.pop('return_url',
-                                              request.get_full_path())
-            user_oauth = django_util.UserOAuth2(request, scopes, return_url)
-            if not user_oauth.has_credentials():
-                return shortcuts.redirect(user_oauth.get_authorize_redirect())
-            setattr(request, django_util.oauth2_settings.request_prefix,
-                    user_oauth)
-            return wrapped_function(request, *args, **kwargs)
-
-        return required_wrapper
-
-    if decorated_function:
-        return curry_wrapper(decorated_function)
-    else:
-        return curry_wrapper
-
-
-def oauth_enabled(decorated_function=None, scopes=None, **decorator_kwargs):
-    """ Decorator to enable OAuth Credentials if authorized, and setup
-    the oauth object on the request object to provide helper functions
-    to start the flow otherwise.
-
-    .. code-block:: python
-       :caption: views.py
-       :name: views_enabled3
-
-       from oauth2client.django_util.decorators import oauth_enabled
-
-       @oauth_enabled
-       def optional_oauth2(request):
-           if request.oauth.has_credentials():
-               # this could be passed into a view
-               # request.oauth.http is also initialized
-               return HttpResponse("User email: {0}".format(
-                                   request.oauth.credentials.id_token['email'])
-           else:
-               return HttpResponse('Here is an OAuth Authorize link:
-               <a href="{0}">Authorize</a>'.format(
-                   request.oauth.get_authorize_redirect()))
-
-
-    Args:
-        decorated_function: View function to decorate.
-        scopes: Scopes to require, will default.
-        decorator_kwargs: Can include ``return_url`` to specify the URL to
-           return to after OAuth2 authorization is complete.
-
-    Returns:
-         The decorated view function.
-    """
-    def curry_wrapper(wrapped_function):
-        @wraps(wrapped_function)
-        def enabled_wrapper(request, *args, **kwargs):
-            return_url = decorator_kwargs.pop('return_url',
-                                              request.get_full_path())
-            user_oauth = django_util.UserOAuth2(request, scopes, return_url)
-            setattr(request, django_util.oauth2_settings.request_prefix,
-                    user_oauth)
-            return wrapped_function(request, *args, **kwargs)
-
-        return enabled_wrapper
-
-    if decorated_function:
-        return curry_wrapper(decorated_function)
-    else:
-        return curry_wrapper

src/oauth2client/contrib/django_util/models.py

@@ -1,82 +0,0 @@
-# Copyright 2016 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains classes used for the Django ORM storage."""
-
-import base64
-import pickle
-
-from django.db import models
-from django.utils import encoding
-import jsonpickle
-
-import oauth2client
-
-
-class CredentialsField(models.Field):
-    """Django ORM field for storing OAuth2 Credentials."""
-
-    def __init__(self, *args, **kwargs):
-        if 'null' not in kwargs:
-            kwargs['null'] = True
-        super(CredentialsField, self).__init__(*args, **kwargs)
-
-    def get_internal_type(self):
-        return 'BinaryField'
-
-    def from_db_value(self, value, expression, connection, context):
-        """Overrides ``models.Field`` method. This converts the value
-        returned from the database to an instance of this class.
-        """
-        return self.to_python(value)
-
-    def to_python(self, value):
-        """Overrides ``models.Field`` method. This is used to convert
-        bytes (from serialization etc) to an instance of this class"""
-        if value is None:
-            return None
-        elif isinstance(value, oauth2client.client.Credentials):
-            return value
-        else:
-            try:
-                return jsonpickle.decode(
-                    base64.b64decode(encoding.smart_bytes(value)).decode())
-            except ValueError:
-                return pickle.loads(
-                    base64.b64decode(encoding.smart_bytes(value)))
-
-    def get_prep_value(self, value):
-        """Overrides ``models.Field`` method. This is used to convert
-        the value from an instances of this class to bytes that can be
-        inserted into the database.
-        """
-        if value is None:
-            return None
-        else:
-            return encoding.smart_text(
-                base64.b64encode(jsonpickle.encode(value).encode()))
-
-    def value_to_string(self, obj):
-        """Convert the field value from the provided model to a string.
-
-        Used during model serialization.
-
-        Args:
-            obj: db.Model, model object
-
-        Returns:
-            string, the serialized field value
-        """
-        value = self._get_val_from_obj(obj)
-        return self.get_prep_value(value)

src/oauth2client/contrib/django_util/signals.py

@@ -1,28 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Signals for Google OAuth2 Helper.
-
-This module contains signals for Google OAuth2 Helper. Currently it only
-contains one, which fires when an OAuth2 authorization flow has completed.
-"""
-
-import django.dispatch
-
-"""Signal that fires when  OAuth2 Flow has completed.
-It passes the Django request object and the OAuth2 credentials object to the
- receiver.
-"""
-oauth2_authorized = django.dispatch.Signal(
-    providing_args=["request", "credentials"])

src/oauth2client/contrib/django_util/site.py

@@ -1,26 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains Django URL patterns used for OAuth2 flow."""
-
-from django.conf import urls
-
-from oauth2client.contrib.django_util import views
-
-urlpatterns = [
-    urls.url(r'oauth2callback/', views.oauth2_callback, name="callback"),
-    urls.url(r'oauth2authorize/', views.oauth2_authorize, name="authorize")
-]
-
-urls = (urlpatterns, "google_oauth", "google_oauth")

src/oauth2client/contrib/django_util/storage.py

@@ -1,81 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains a storage module that stores credentials using the Django ORM."""
-
-from oauth2client import client
-
-
-class DjangoORMStorage(client.Storage):
-    """Store and retrieve a single credential to and from the Django datastore.
-
-    This Storage helper presumes the Credentials
-    have been stored as a CredentialsField
-    on a db model class.
-    """
-
-    def __init__(self, model_class, key_name, key_value, property_name):
-        """Constructor for Storage.
-
-        Args:
-            model: string, fully qualified name of db.Model model class.
-            key_name: string, key name for the entity that has the credentials
-            key_value: string, key value for the entity that has the
-               credentials.
-            property_name: string, name of the property that is an
-                           CredentialsProperty.
-        """
-        super(DjangoORMStorage, self).__init__()
-        self.model_class = model_class
-        self.key_name = key_name
-        self.key_value = key_value
-        self.property_name = property_name
-
-    def locked_get(self):
-        """Retrieve stored credential from the Django ORM.
-
-        Returns:
-            oauth2client.Credentials retrieved from the Django ORM, associated
-             with the ``model``, ``key_value``->``key_name`` pair used to query
-             for the model, and ``property_name`` identifying the
-             ``CredentialsProperty`` field, all of which are defined in the
-             constructor for this Storage object.
-
-        """
-        query = {self.key_name: self.key_value}
-        entities = self.model_class.objects.filter(**query)
-        if len(entities) > 0:
-            credential = getattr(entities[0], self.property_name)
-            if getattr(credential, 'set_store', None) is not None:
-                credential.set_store(self)
-            return credential
-        else:
-            return None
-
-    def locked_put(self, credentials):
-        """Write a Credentials to the Django datastore.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-        """
-        entity, _ = self.model_class.objects.get_or_create(
-            **{self.key_name: self.key_value})
-
-        setattr(entity, self.property_name, credentials)
-        entity.save()
-
-    def locked_delete(self):
-        """Delete Credentials from the datastore."""
-        query = {self.key_name: self.key_value}
-        self.model_class.objects.filter(**query).delete()

src/oauth2client/contrib/django_util/views.py

@@ -1,191 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This module contains the views used by the OAuth2 flows.
-
-Their are two views used by the OAuth2 flow, the authorize and the callback
-view. The authorize view kicks off the three-legged OAuth flow, and the
-callback view validates the flow and if successful stores the credentials
-in the configured storage."""
-
-import hashlib
-import json
-import os
-
-from django import http
-from django import shortcuts
-from django.conf import settings
-from django.core import urlresolvers
-from django.shortcuts import redirect
-import jsonpickle
-from six.moves.urllib import parse
-
-from oauth2client import client
-from oauth2client.contrib import django_util
-from oauth2client.contrib.django_util import get_storage
-from oauth2client.contrib.django_util import signals
-
-_CSRF_KEY = 'google_oauth2_csrf_token'
-_FLOW_KEY = 'google_oauth2_flow_{0}'
-
-
-def _make_flow(request, scopes, return_url=None):
-    """Creates a Web Server Flow
-
-    Args:
-        request: A Django request object.
-        scopes: the request oauth2 scopes.
-        return_url: The URL to return to after the flow is complete. Defaults
-            to the path of the current request.
-
-    Returns:
-        An OAuth2 flow object that has been stored in the session.
-    """
-    # Generate a CSRF token to prevent malicious requests.
-    csrf_token = hashlib.sha256(os.urandom(1024)).hexdigest()
-
-    request.session[_CSRF_KEY] = csrf_token
-
-    state = json.dumps({
-        'csrf_token': csrf_token,
-        'return_url': return_url,
-    })
-
-    flow = client.OAuth2WebServerFlow(
-        client_id=django_util.oauth2_settings.client_id,
-        client_secret=django_util.oauth2_settings.client_secret,
-        scope=scopes,
-        state=state,
-        redirect_uri=request.build_absolute_uri(
-            urlresolvers.reverse("google_oauth:callback")))
-
-    flow_key = _FLOW_KEY.format(csrf_token)
-    request.session[flow_key] = jsonpickle.encode(flow)
-    return flow
-
-
-def _get_flow_for_token(csrf_token, request):
-    """ Looks up the flow in session to recover information about requested
-    scopes.
-
-    Args:
-        csrf_token: The token passed in the callback request that should
-            match the one previously generated and stored in the request on the
-            initial authorization view.
-
-    Returns:
-        The OAuth2 Flow object associated with this flow based on the
-        CSRF token.
-    """
-    flow_pickle = request.session.get(_FLOW_KEY.format(csrf_token), None)
-    return None if flow_pickle is None else jsonpickle.decode(flow_pickle)
-
-
-def oauth2_callback(request):
-    """ View that handles the user's return from OAuth2 provider.
-
-    This view verifies the CSRF state and OAuth authorization code, and on
-    success stores the credentials obtained in the storage provider,
-    and redirects to the return_url specified in the authorize view and
-    stored in the session.
-
-    Args:
-        request: Django request.
-
-    Returns:
-         A redirect response back to the return_url.
-    """
-    if 'error' in request.GET:
-        reason = request.GET.get(
-            'error_description', request.GET.get('error', ''))
-        return http.HttpResponseBadRequest(
-            'Authorization failed {0}'.format(reason))
-
-    try:
-        encoded_state = request.GET['state']
-        code = request.GET['code']
-    except KeyError:
-        return http.HttpResponseBadRequest(
-            'Request missing state or authorization code')
-
-    try:
-        server_csrf = request.session[_CSRF_KEY]
-    except KeyError:
-        return http.HttpResponseBadRequest(
-            'No existing session for this flow.')
-
-    try:
-        state = json.loads(encoded_state)
-        client_csrf = state['csrf_token']
-        return_url = state['return_url']
-    except (ValueError, KeyError):
-        return http.HttpResponseBadRequest('Invalid state parameter.')
-
-    if client_csrf != server_csrf:
-        return http.HttpResponseBadRequest('Invalid CSRF token.')
-
-    flow = _get_flow_for_token(client_csrf, request)
-
-    if not flow:
-        return http.HttpResponseBadRequest('Missing Oauth2 flow.')
-
-    try:
-        credentials = flow.step2_exchange(code)
-    except client.FlowExchangeError as exchange_error:
-        return http.HttpResponseBadRequest(
-            'An error has occurred: {0}'.format(exchange_error))
-
-    get_storage(request).put(credentials)
-
-    signals.oauth2_authorized.send(sender=signals.oauth2_authorized,
-                                   request=request, credentials=credentials)
-
-    return shortcuts.redirect(return_url)
-
-
-def oauth2_authorize(request):
-    """ View to start the OAuth2 Authorization flow.
-
-     This view starts the OAuth2 authorization flow. If scopes is passed in
-     as a  GET URL parameter, it will authorize those scopes, otherwise the
-     default scopes specified in settings. The return_url can also be
-     specified as a GET parameter, otherwise the referer header will be
-     checked, and if that isn't found it will return to the root path.
-
-    Args:
-       request: The Django request object.
-
-    Returns:
-         A redirect to Google OAuth2 Authorization.
-    """
-    return_url = request.GET.get('return_url', None)
-    if not return_url:
-        return_url = request.META.get('HTTP_REFERER', '/')
-
-    scopes = request.GET.getlist('scopes', django_util.oauth2_settings.scopes)
-    # Model storage (but not session storage) requires a logged in user
-    if django_util.oauth2_settings.storage_model:
-        if not request.user.is_authenticated():
-            return redirect('{0}?next={1}'.format(
-                settings.LOGIN_URL, parse.quote(request.get_full_path())))
-        # This checks for the case where we ended up here because of a logged
-        # out user but we had credentials for it in the first place
-        else:
-            user_oauth = django_util.UserOAuth2(request, scopes, return_url)
-            if user_oauth.has_credentials():
-                return redirect(return_url)
-
-    flow = _make_flow(request=request, scopes=scopes, return_url=return_url)
-    auth_url = flow.step1_get_authorize_url()
-    return shortcuts.redirect(auth_url)

src/oauth2client/contrib/flask_util.py

@@ -1,557 +0,0 @@
-# Copyright 2015 Google Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for the Flask web framework
-
-Provides a Flask extension that makes using OAuth2 web server flow easier.
-The extension includes views that handle the entire auth flow and a
-``@required`` decorator to automatically ensure that user credentials are
-available.
-
-
-Configuration
-=============
-
-To configure, you'll need a set of OAuth2 web application credentials from the
-`Google Developer's Console <https://console.developers.google.com/project/_/\
-apiui/credential>`__.
-
-.. code-block:: python
-
-    from oauth2client.contrib.flask_util import UserOAuth2
-
-    app = Flask(__name__)
-
-    app.config['SECRET_KEY'] = 'your-secret-key'
-
-    app.config['GOOGLE_OAUTH2_CLIENT_SECRETS_FILE'] = 'client_secrets.json'
-
-    # or, specify the client id and secret separately
-    app.config['GOOGLE_OAUTH2_CLIENT_ID'] = 'your-client-id'
-    app.config['GOOGLE_OAUTH2_CLIENT_SECRET'] = 'your-client-secret'
-
-    oauth2 = UserOAuth2(app)
-
-
-Usage
-=====
-
-Once configured, you can use the :meth:`UserOAuth2.required` decorator to
-ensure that credentials are available within a view.
-
-.. code-block:: python
-   :emphasize-lines: 3,7,10
-
-    # Note that app.route should be the outermost decorator.
-    @app.route('/needs_credentials')
-    @oauth2.required
-    def example():
-        # http is authorized with the user's credentials and can be used
-        # to make http calls.
-        http = oauth2.http()
-
-        # Or, you can access the credentials directly
-        credentials = oauth2.credentials
-
-If you want credentials to be optional for a view, you can leave the decorator
-off and use :meth:`UserOAuth2.has_credentials` to check.
-
-.. code-block:: python
-   :emphasize-lines: 3
-
-    @app.route('/optional')
-    def optional():
-        if oauth2.has_credentials():
-            return 'Credentials found!'
-        else:
-            return 'No credentials!'
-
-
-When credentials are available, you can use :attr:`UserOAuth2.email` and
-:attr:`UserOAuth2.user_id` to access information from the `ID Token
-<https://developers.google.com/identity/protocols/OpenIDConnect?hl=en>`__, if
-available.
-
-.. code-block:: python
-   :emphasize-lines: 4
-
-    @app.route('/info')
-    @oauth2.required
-    def info():
-        return "Hello, {} ({})".format(oauth2.email, oauth2.user_id)
-
-
-URLs & Trigging Authorization
-=============================
-
-The extension will add two new routes to your application:
-
-    * ``"oauth2.authorize"`` -> ``/oauth2authorize``
-    * ``"oauth2.callback"`` -> ``/oauth2callback``
-
-When configuring your OAuth2 credentials on the Google Developer's Console, be
-sure to add ``http[s]://[your-app-url]/oauth2callback`` as an authorized
-callback url.
-
-Typically you don't not need to use these routes directly, just be sure to
-decorate any views that require credentials with ``@oauth2.required``. If
-needed, you can trigger authorization at any time by redirecting the user
-to the URL returned by :meth:`UserOAuth2.authorize_url`.
-
-.. code-block:: python
-   :emphasize-lines: 3
-
-    @app.route('/login')
-    def login():
-        return oauth2.authorize_url("/")
-
-
-Incremental Auth
-================
-
-This extension also supports `Incremental Auth <https://developers.google.com\
-/identity/protocols/OAuth2WebServer?hl=en#incrementalAuth>`__. To enable it,
-configure the extension with ``include_granted_scopes``.
-
-.. code-block:: python
-
-    oauth2 = UserOAuth2(app, include_granted_scopes=True)
-
-Then specify any additional scopes needed on the decorator, for example:
-
-.. code-block:: python
-   :emphasize-lines: 2,7
-
-    @app.route('/drive')
-    @oauth2.required(scopes=["https://www.googleapis.com/auth/drive"])
-    def requires_drive():
-        ...
-
-    @app.route('/calendar')
-    @oauth2.required(scopes=["https://www.googleapis.com/auth/calendar"])
-    def requires_calendar():
-        ...
-
-The decorator will ensure that the the user has authorized all specified scopes
-before allowing them to access the view, and will also ensure that credentials
-do not lose any previously authorized scopes.
-
-
-Storage
-=======
-
-By default, the extension uses a Flask session-based storage solution. This
-means that credentials are only available for the duration of a session. It
-also means that with Flask's default configuration, the credentials will be
-visible in the session cookie. It's highly recommended to use database-backed
-session and to use https whenever handling user credentials.
-
-If you need the credentials to be available longer than a user session or
-available outside of a request context, you will need to implement your own
-:class:`oauth2client.Storage`.
-"""
-
-from functools import wraps
-import hashlib
-import json
-import os
-import pickle
-
-try:
-    from flask import Blueprint
-    from flask import _app_ctx_stack
-    from flask import current_app
-    from flask import redirect
-    from flask import request
-    from flask import session
-    from flask import url_for
-    import markupsafe
-except ImportError:  # pragma: NO COVER
-    raise ImportError('The flask utilities require flask 0.9 or newer.')
-
-import six.moves.http_client as httplib
-
-from oauth2client import client
-from oauth2client import clientsecrets
-from oauth2client import transport
-from oauth2client.contrib import dictionary_storage
-
-
-_DEFAULT_SCOPES = ('email',)
-_CREDENTIALS_KEY = 'google_oauth2_credentials'
-_FLOW_KEY = 'google_oauth2_flow_{0}'
-_CSRF_KEY = 'google_oauth2_csrf_token'
-
-
-def _get_flow_for_token(csrf_token):
-    """Retrieves the flow instance associated with a given CSRF token from
-    the Flask session."""
-    flow_pickle = session.pop(
-        _FLOW_KEY.format(csrf_token), None)
-
-    if flow_pickle is None:
-        return None
-    else:
-        return pickle.loads(flow_pickle)
-
-
-class UserOAuth2(object):
-    """Flask extension for making OAuth 2.0 easier.
-
-    Configuration values:
-
-        * ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` path to a client secrets json
-          file, obtained from the credentials screen in the Google Developers
-          console.
-        * ``GOOGLE_OAUTH2_CLIENT_ID`` the oauth2 credentials' client ID. This
-          is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` is not
-          specified.
-        * ``GOOGLE_OAUTH2_CLIENT_SECRET`` the oauth2 credentials' client
-          secret. This is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE``
-          is not specified.
-
-    If app is specified, all arguments will be passed along to init_app.
-
-    If no app is specified, then you should call init_app in your application
-    factory to finish initialization.
-    """
-
-    def __init__(self, app=None, *args, **kwargs):
-        self.app = app
-        if app is not None:
-            self.init_app(app, *args, **kwargs)
-
-    def init_app(self, app, scopes=None, client_secrets_file=None,
-                 client_id=None, client_secret=None, authorize_callback=None,
-                 storage=None, **kwargs):
-        """Initialize this extension for the given app.
-
-        Arguments:
-            app: A Flask application.
-            scopes: Optional list of scopes to authorize.
-            client_secrets_file: Path to a file containing client secrets. You
-                can also specify the GOOGLE_OAUTH2_CLIENT_SECRETS_FILE config
-                value.
-            client_id: If not specifying a client secrets file, specify the
-                OAuth2 client id. You can also specify the
-                GOOGLE_OAUTH2_CLIENT_ID config value. You must also provide a
-                client secret.
-            client_secret: The OAuth2 client secret. You can also specify the
-                GOOGLE_OAUTH2_CLIENT_SECRET config value.
-            authorize_callback: A function that is executed after successful
-                user authorization.
-            storage: A oauth2client.client.Storage subclass for storing the
-                credentials. By default, this is a Flask session based storage.
-            kwargs: Any additional args are passed along to the Flow
-                constructor.
-        """
-        self.app = app
-        self.authorize_callback = authorize_callback
-        self.flow_kwargs = kwargs
-
-        if storage is None:
-            storage = dictionary_storage.DictionaryStorage(
-                session, key=_CREDENTIALS_KEY)
-        self.storage = storage
-
-        if scopes is None:
-            scopes = app.config.get('GOOGLE_OAUTH2_SCOPES', _DEFAULT_SCOPES)
-        self.scopes = scopes
-
-        self._load_config(client_secrets_file, client_id, client_secret)
-
-        app.register_blueprint(self._create_blueprint())
-
-    def _load_config(self, client_secrets_file, client_id, client_secret):
-        """Loads oauth2 configuration in order of priority.
-
-        Priority:
-            1. Config passed to the constructor or init_app.
-            2. Config passed via the GOOGLE_OAUTH2_CLIENT_SECRETS_FILE app
-               config.
-            3. Config passed via the GOOGLE_OAUTH2_CLIENT_ID and
-               GOOGLE_OAUTH2_CLIENT_SECRET app config.
-
-        Raises:
-            ValueError if no config could be found.
-        """
-        if client_id and client_secret:
-            self.client_id, self.client_secret = client_id, client_secret
-            return
-
-        if client_secrets_file:
-            self._load_client_secrets(client_secrets_file)
-            return
-
-        if 'GOOGLE_OAUTH2_CLIENT_SECRETS_FILE' in self.app.config:
-            self._load_client_secrets(
-                self.app.config['GOOGLE_OAUTH2_CLIENT_SECRETS_FILE'])
-            return
-
-        try:
-            self.client_id, self.client_secret = (
-                self.app.config['GOOGLE_OAUTH2_CLIENT_ID'],
-                self.app.config['GOOGLE_OAUTH2_CLIENT_SECRET'])
-        except KeyError:
-            raise ValueError(
-                'OAuth2 configuration could not be found. Either specify the '
-                'client_secrets_file or client_id and client_secret or set '
-                'the app configuration variables '
-                'GOOGLE_OAUTH2_CLIENT_SECRETS_FILE or '
-                'GOOGLE_OAUTH2_CLIENT_ID and GOOGLE_OAUTH2_CLIENT_SECRET.')
-
-    def _load_client_secrets(self, filename):
-        """Loads client secrets from the given filename."""
-        client_type, client_info = clientsecrets.loadfile(filename)
-        if client_type != clientsecrets.TYPE_WEB:
-            raise ValueError(
-                'The flow specified in {0} is not supported.'.format(
-                    client_type))
-
-        self.client_id = client_info['client_id']
-        self.client_secret = client_info['client_secret']
-
-    def _make_flow(self, return_url=None, **kwargs):
-        """Creates a Web Server Flow"""
-        # Generate a CSRF token to prevent malicious requests.
-        csrf_token = hashlib.sha256(os.urandom(1024)).hexdigest()
-
-        session[_CSRF_KEY] = csrf_token
-
-        state = json.dumps({
-            'csrf_token': csrf_token,
-            'return_url': return_url
-        })
-
-        kw = self.flow_kwargs.copy()
-        kw.update(kwargs)
-
-        extra_scopes = kw.pop('scopes', [])
-        scopes = set(self.scopes).union(set(extra_scopes))
-
-        flow = client.OAuth2WebServerFlow(
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            scope=scopes,
-            state=state,
-            redirect_uri=url_for('oauth2.callback', _external=True),
-            **kw)
-
-        flow_key = _FLOW_KEY.format(csrf_token)
-        session[flow_key] = pickle.dumps(flow)
-
-        return flow
-
-    def _create_blueprint(self):
-        bp = Blueprint('oauth2', __name__)
-        bp.add_url_rule('/oauth2authorize', 'authorize', self.authorize_view)
-        bp.add_url_rule('/oauth2callback', 'callback', self.callback_view)
-
-        return bp
-
-    def authorize_view(self):
-        """Flask view that starts the authorization flow.
-
-        Starts flow by redirecting the user to the OAuth2 provider.
-        """
-        args = request.args.to_dict()
-
-        # Scopes will be passed as mutliple args, and to_dict() will only
-        # return one. So, we use getlist() to get all of the scopes.
-        args['scopes'] = request.args.getlist('scopes')
-
-        return_url = args.pop('return_url', None)
-        if return_url is None:
-            return_url = request.referrer or '/'
-
-        flow = self._make_flow(return_url=return_url, **args)
-        auth_url = flow.step1_get_authorize_url()
-
-        return redirect(auth_url)
-
-    def callback_view(self):
-        """Flask view that handles the user's return from OAuth2 provider.
-
-        On return, exchanges the authorization code for credentials and stores
-        the credentials.
-        """
-        if 'error' in request.args:
-            reason = request.args.get(
-                'error_description', request.args.get('error', ''))
-            reason = markupsafe.escape(reason)
-            return ('Authorization failed: {0}'.format(reason),
-                    httplib.BAD_REQUEST)
-
-        try:
-            encoded_state = request.args['state']
-            server_csrf = session[_CSRF_KEY]
-            code = request.args['code']
-        except KeyError:
-            return 'Invalid request', httplib.BAD_REQUEST
-
-        try:
-            state = json.loads(encoded_state)
-            client_csrf = state['csrf_token']
-            return_url = state['return_url']
-        except (ValueError, KeyError):
-            return 'Invalid request state', httplib.BAD_REQUEST
-
-        if client_csrf != server_csrf:
-            return 'Invalid request state', httplib.BAD_REQUEST
-
-        flow = _get_flow_for_token(server_csrf)
-
-        if flow is None:
-            return 'Invalid request state', httplib.BAD_REQUEST
-
-        # Exchange the auth code for credentials.
-        try:
-            credentials = flow.step2_exchange(code)
-        except client.FlowExchangeError as exchange_error:
-            current_app.logger.exception(exchange_error)
-            content = 'An error occurred: {0}'.format(exchange_error)
-            return content, httplib.BAD_REQUEST
-
-        # Save the credentials to the storage.
-        self.storage.put(credentials)
-
-        if self.authorize_callback:
-            self.authorize_callback(credentials)
-
-        return redirect(return_url)
-
-    @property
-    def credentials(self):
-        """The credentials for the current user or None if unavailable."""
-        ctx = _app_ctx_stack.top
-
-        if not hasattr(ctx, _CREDENTIALS_KEY):
-            ctx.google_oauth2_credentials = self.storage.get()
-
-        return ctx.google_oauth2_credentials
-
-    def has_credentials(self):
-        """Returns True if there are valid credentials for the current user."""
-        if not self.credentials:
-            return False
-        # Is the access token expired? If so, do we have an refresh token?
-        elif (self.credentials.access_token_expired and
-                not self.credentials.refresh_token):
-            return False
-        else:
-            return True
-
-    @property
-    def email(self):
-        """Returns the user's email address or None if there are no credentials.
-
-        The email address is provided by the current credentials' id_token.
-        This should not be used as unique identifier as the user can change
-        their email. If you need a unique identifier, use user_id.
-        """
-        if not self.credentials:
-            return None
-        try:
-            return self.credentials.id_token['email']
-        except KeyError:
-            current_app.logger.error(
-                'Invalid id_token {0}'.format(self.credentials.id_token))
-
-    @property
-    def user_id(self):
-        """Returns the a unique identifier for the user
-
-        Returns None if there are no credentials.
-
-        The id is provided by the current credentials' id_token.
-        """
-        if not self.credentials:
-            return None
-        try:
-            return self.credentials.id_token['sub']
-        except KeyError:
-            current_app.logger.error(
-                'Invalid id_token {0}'.format(self.credentials.id_token))
-
-    def authorize_url(self, return_url, **kwargs):
-        """Creates a URL that can be used to start the authorization flow.
-
-        When the user is directed to the URL, the authorization flow will
-        begin. Once complete, the user will be redirected to the specified
-        return URL.
-
-        Any kwargs are passed into the flow constructor.
-        """
-        return url_for('oauth2.authorize', return_url=return_url, **kwargs)
-
-    def required(self, decorated_function=None, scopes=None,
-                 **decorator_kwargs):
-        """Decorator to require OAuth2 credentials for a view.
-
-        If credentials are not available for the current user, then they will
-        be redirected to the authorization flow. Once complete, the user will
-        be redirected back to the original page.
-        """
-
-        def curry_wrapper(wrapped_function):
-            @wraps(wrapped_function)
-            def required_wrapper(*args, **kwargs):
-                return_url = decorator_kwargs.pop('return_url', request.url)
-
-                requested_scopes = set(self.scopes)
-                if scopes is not None:
-                    requested_scopes |= set(scopes)
-                if self.has_credentials():
-                    requested_scopes |= self.credentials.scopes
-
-                requested_scopes = list(requested_scopes)
-
-                # Does the user have credentials and does the credentials have
-                # all of the needed scopes?
-                if (self.has_credentials() and
-                        self.credentials.has_scopes(requested_scopes)):
-                    return wrapped_function(*args, **kwargs)
-                # Otherwise, redirect to authorization
-                else:
-                    auth_url = self.authorize_url(
-                        return_url,
-                        scopes=requested_scopes,
-                        **decorator_kwargs)
-
-                    return redirect(auth_url)
-
-            return required_wrapper
-
-        if decorated_function:
-            return curry_wrapper(decorated_function)
-        else:
-            return curry_wrapper
-
-    def http(self, *args, **kwargs):
-        """Returns an authorized http instance.
-
-        Can only be called if there are valid credentials for the user, such
-        as inside of a view that is decorated with @required.
-
-        Args:
-            *args: Positional arguments passed to httplib2.Http constructor.
-            **kwargs: Positional arguments passed to httplib2.Http constructor.
-
-        Raises:
-            ValueError if no credentials are available.
-        """
-        if not self.credentials:
-            raise ValueError('No credentials available.')
-        return self.credentials.authorize(
-            transport.get_http_object(*args, **kwargs))

src/oauth2client/contrib/gce.py

@@ -1,156 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for Google Compute Engine
-
-Utilities for making it easier to use OAuth 2.0 on Google Compute Engine.
-"""
-
-import logging
-import warnings
-
-from six.moves import http_client
-
-from oauth2client import client
-from oauth2client.contrib import _metadata
-
-
-logger = logging.getLogger(__name__)
-
-_SCOPES_WARNING = """\
-You have requested explicit scopes to be used with a GCE service account.
-Using this argument will have no effect on the actual scopes for tokens
-requested. These scopes are set at VM instance creation time and
-can't be overridden in the request.
-"""
-
-
-class AppAssertionCredentials(client.AssertionCredentials):
-    """Credentials object for Compute Engine Assertion Grants
-
-    This object will allow a Compute Engine instance to identify itself to
-    Google and other OAuth 2.0 servers that can verify assertions. It can be
-    used for the purpose of accessing data stored under an account assigned to
-    the Compute Engine instance itself.
-
-    This credential does not require a flow to instantiate because it
-    represents a two legged flow, and therefore has all of the required
-    information to generate and refresh its own access tokens.
-
-    Note that :attr:`service_account_email` and :attr:`scopes`
-    will both return None until the credentials have been refreshed.
-    To check whether credentials have previously been refreshed use
-    :attr:`invalid`.
-    """
-
-    def __init__(self, email=None, *args, **kwargs):
-        """Constructor for AppAssertionCredentials
-
-        Args:
-            email: an email that specifies the service account to use.
-                   Only necessary if using custom service accounts
-                   (see https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#createdefaultserviceaccount).
-        """
-        if 'scopes' in kwargs:
-            warnings.warn(_SCOPES_WARNING)
-            kwargs['scopes'] = None
-
-        # Assertion type is no longer used, but still in the
-        # parent class signature.
-        super(AppAssertionCredentials, self).__init__(None, *args, **kwargs)
-
-        self.service_account_email = email
-        self.scopes = None
-        self.invalid = True
-
-    @classmethod
-    def from_json(cls, json_data):
-        raise NotImplementedError(
-            'Cannot serialize credentials for GCE service accounts.')
-
-    def to_json(self):
-        raise NotImplementedError(
-            'Cannot serialize credentials for GCE service accounts.')
-
-    def retrieve_scopes(self, http):
-        """Retrieves the canonical list of scopes for this access token.
-
-        Overrides client.Credentials.retrieve_scopes. Fetches scopes info
-        from the metadata server.
-
-        Args:
-            http: httplib2.Http, an http object to be used to make the refresh
-                  request.
-
-        Returns:
-            A set of strings containing the canonical list of scopes.
-        """
-        self._retrieve_info(http)
-        return self.scopes
-
-    def _retrieve_info(self, http):
-        """Retrieves service account info for invalid credentials.
-
-        Args:
-            http: an object to be used to make HTTP requests.
-        """
-        if self.invalid:
-            info = _metadata.get_service_account_info(
-                http,
-                service_account=self.service_account_email or 'default')
-            self.invalid = False
-            self.service_account_email = info['email']
-            self.scopes = info['scopes']
-
-    def _refresh(self, http):
-        """Refreshes the access token.
-
-        Skip all the storage hoops and just refresh using the API.
-
-        Args:
-            http: an object to be used to make HTTP requests.
-
-        Raises:
-            HttpAccessTokenRefreshError: When the refresh fails.
-        """
-        try:
-            self._retrieve_info(http)
-            self.access_token, self.token_expiry = _metadata.get_token(
-                http, service_account=self.service_account_email)
-        except http_client.HTTPException as err:
-            raise client.HttpAccessTokenRefreshError(str(err))
-
-    @property
-    def serialization_data(self):
-        raise NotImplementedError(
-            'Cannot serialize credentials for GCE service accounts.')
-
-    def create_scoped_required(self):
-        return False
-
-    def sign_blob(self, blob):
-        """Cryptographically sign a blob (of bytes).
-
-        This method is provided to support a common interface, but
-        the actual key used for a Google Compute Engine service account
-        is not available, so it can't be used to sign content.
-
-        Args:
-            blob: bytes, Message to be signed.
-
-        Raises:
-            NotImplementedError, always.
-        """
-        raise NotImplementedError(
-            'Compute Engine service accounts cannot sign blobs')

src/oauth2client/contrib/keyring_storage.py

@@ -1,95 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""A keyring based Storage.
-
-A Storage for Credentials that uses the keyring module.
-"""
-
-import threading
-
-import keyring
-
-from oauth2client import client
-
-
-class Storage(client.Storage):
-    """Store and retrieve a single credential to and from the keyring.
-
-    To use this module you must have the keyring module installed. See
-    <http://pypi.python.org/pypi/keyring/>. This is an optional module and is
-    not installed with oauth2client by default because it does not work on all
-    the platforms that oauth2client supports, such as Google App Engine.
-
-    The keyring module <http://pypi.python.org/pypi/keyring/> is a
-    cross-platform library for access the keyring capabilities of the local
-    system. The user will be prompted for their keyring password when this
-    module is used, and the manner in which the user is prompted will vary per
-    platform.
-
-    Usage::
-
-        from oauth2client import keyring_storage
-
-        s = keyring_storage.Storage('name_of_application', 'user1')
-        credentials = s.get()
-
-    """
-
-    def __init__(self, service_name, user_name):
-        """Constructor.
-
-        Args:
-            service_name: string, The name of the service under which the
-                          credentials are stored.
-            user_name: string, The name of the user to store credentials for.
-        """
-        super(Storage, self).__init__(lock=threading.Lock())
-        self._service_name = service_name
-        self._user_name = user_name
-
-    def locked_get(self):
-        """Retrieve Credential from file.
-
-        Returns:
-            oauth2client.client.Credentials
-        """
-        credentials = None
-        content = keyring.get_password(self._service_name, self._user_name)
-
-        if content is not None:
-            try:
-                credentials = client.Credentials.new_from_json(content)
-                credentials.set_store(self)
-            except ValueError:
-                pass
-
-        return credentials
-
-    def locked_put(self, credentials):
-        """Write Credentials to file.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-        """
-        keyring.set_password(self._service_name, self._user_name,
-                             credentials.to_json())
-
-    def locked_delete(self):
-        """Delete Credentials file.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-        """
-        keyring.set_password(self._service_name, self._user_name, '')

src/oauth2client/contrib/multiprocess_file_storage.py

@@ -1,355 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Multiprocess file credential storage.
-
-This module provides file-based storage that supports multiple credentials and
-cross-thread and process access.
-
-This module supersedes the functionality previously found in `multistore_file`.
-
-This module provides :class:`MultiprocessFileStorage` which:
-    * Is tied to a single credential via a user-specified key. This key can be
-      used to distinguish between multiple users, client ids, and/or scopes.
-    * Can be safely accessed and refreshed across threads and processes.
-
-Process & thread safety guarantees the following behavior:
-    * If one thread or process refreshes a credential, subsequent refreshes
-      from other processes will re-fetch the credentials from the file instead
-      of performing an http request.
-    * If two processes or threads attempt to refresh concurrently, only one
-      will be able to acquire the lock and refresh, with the deadlock caveat
-      below.
-    * The interprocess lock will not deadlock, instead, the if a process can
-      not acquire the interprocess lock within ``INTERPROCESS_LOCK_DEADLINE``
-      it will allow refreshing the credential but will not write the updated
-      credential to disk, This logic happens during every lock cycle - if the
-      credentials are refreshed again it will retry locking and writing as
-      normal.
-
-Usage
-=====
-
-Before using the storage, you need to decide how you want to key the
-credentials. A few common strategies include:
-
-    * If you're storing credentials for multiple users in a single file, use
-      a unique identifier for each user as the key.
-    * If you're storing credentials for multiple client IDs in a single file,
-      use the client ID as the key.
-    * If you're storing multiple credentials for one user, use the scopes as
-      the key.
-    * If you have a complicated setup, use a compound key. For example, you
-      can use a combination of the client ID and scopes as the key.
-
-Create an instance of :class:`MultiprocessFileStorage` for each credential you
-want to store, for example::
-
-    filename = 'credentials'
-    key = '{}-{}'.format(client_id, user_id)
-    storage = MultiprocessFileStorage(filename, key)
-
-To store the credentials::
-
-    storage.put(credentials)
-
-If you're going to continue to use the credentials after storing them, be sure
-to call :func:`set_store`::
-
-    credentials.set_store(storage)
-
-To retrieve the credentials::
-
-    storage.get(credentials)
-
-"""
-
-import base64
-import json
-import logging
-import os
-import threading
-
-import fasteners
-from six import iteritems
-
-from oauth2client import _helpers
-from oauth2client import client
-
-
-#: The maximum amount of time, in seconds, to wait when acquire the
-#: interprocess lock before falling back to read-only mode.
-INTERPROCESS_LOCK_DEADLINE = 1
-
-logger = logging.getLogger(__name__)
-_backends = {}
-_backends_lock = threading.Lock()
-
-
-def _create_file_if_needed(filename):
-    """Creates the an empty file if it does not already exist.
-
-    Returns:
-        True if the file was created, False otherwise.
-    """
-    if os.path.exists(filename):
-        return False
-    else:
-        # Equivalent to "touch".
-        open(filename, 'a+b').close()
-        logger.info('Credential file {0} created'.format(filename))
-        return True
-
-
-def _load_credentials_file(credentials_file):
-    """Load credentials from the given file handle.
-
-    The file is expected to be in this format:
-
-        {
-            "file_version": 2,
-            "credentials": {
-                "key": "base64 encoded json representation of credentials."
-            }
-        }
-
-    This function will warn and return empty credentials instead of raising
-    exceptions.
-
-    Args:
-        credentials_file: An open file handle.
-
-    Returns:
-        A dictionary mapping user-defined keys to an instance of
-        :class:`oauth2client.client.Credentials`.
-    """
-    try:
-        credentials_file.seek(0)
-        data = json.load(credentials_file)
-    except Exception:
-        logger.warning(
-            'Credentials file could not be loaded, will ignore and '
-            'overwrite.')
-        return {}
-
-    if data.get('file_version') != 2:
-        logger.warning(
-            'Credentials file is not version 2, will ignore and '
-            'overwrite.')
-        return {}
-
-    credentials = {}
-
-    for key, encoded_credential in iteritems(data.get('credentials', {})):
-        try:
-            credential_json = base64.b64decode(encoded_credential)
-            credential = client.Credentials.new_from_json(credential_json)
-            credentials[key] = credential
-        except:
-            logger.warning(
-                'Invalid credential {0} in file, ignoring.'.format(key))
-
-    return credentials
-
-
-def _write_credentials_file(credentials_file, credentials):
-    """Writes credentials to a file.
-
-    Refer to :func:`_load_credentials_file` for the format.
-
-    Args:
-        credentials_file: An open file handle, must be read/write.
-        credentials: A dictionary mapping user-defined keys to an instance of
-            :class:`oauth2client.client.Credentials`.
-    """
-    data = {'file_version': 2, 'credentials': {}}
-
-    for key, credential in iteritems(credentials):
-        credential_json = credential.to_json()
-        encoded_credential = _helpers._from_bytes(base64.b64encode(
-            _helpers._to_bytes(credential_json)))
-        data['credentials'][key] = encoded_credential
-
-    credentials_file.seek(0)
-    json.dump(data, credentials_file)
-    credentials_file.truncate()
-
-
-class _MultiprocessStorageBackend(object):
-    """Thread-local backend for multiprocess storage.
-
-    Each process has only one instance of this backend per file. All threads
-    share a single instance of this backend. This ensures that all threads
-    use the same thread lock and process lock when accessing the file.
-    """
-
-    def __init__(self, filename):
-        self._file = None
-        self._filename = filename
-        self._process_lock = fasteners.InterProcessLock(
-            '{0}.lock'.format(filename))
-        self._thread_lock = threading.Lock()
-        self._read_only = False
-        self._credentials = {}
-
-    def _load_credentials(self):
-        """(Re-)loads the credentials from the file."""
-        if not self._file:
-            return
-
-        loaded_credentials = _load_credentials_file(self._file)
-        self._credentials.update(loaded_credentials)
-
-        logger.debug('Read credential file')
-
-    def _write_credentials(self):
-        if self._read_only:
-            logger.debug('In read-only mode, not writing credentials.')
-            return
-
-        _write_credentials_file(self._file, self._credentials)
-        logger.debug('Wrote credential file {0}.'.format(self._filename))
-
-    def acquire_lock(self):
-        self._thread_lock.acquire()
-        locked = self._process_lock.acquire(timeout=INTERPROCESS_LOCK_DEADLINE)
-
-        if locked:
-            _create_file_if_needed(self._filename)
-            self._file = open(self._filename, 'r+')
-            self._read_only = False
-
-        else:
-            logger.warn(
-                'Failed to obtain interprocess lock for credentials. '
-                'If a credential is being refreshed, other processes may '
-                'not see the updated access token and refresh as well.')
-            if os.path.exists(self._filename):
-                self._file = open(self._filename, 'r')
-            else:
-                self._file = None
-            self._read_only = True
-
-        self._load_credentials()
-
-    def release_lock(self):
-        if self._file is not None:
-            self._file.close()
-            self._file = None
-
-        if not self._read_only:
-            self._process_lock.release()
-
-        self._thread_lock.release()
-
-    def _refresh_predicate(self, credentials):
-        if credentials is None:
-            return True
-        elif credentials.invalid:
-            return True
-        elif credentials.access_token_expired:
-            return True
-        else:
-            return False
-
-    def locked_get(self, key):
-        # Check if the credential is already in memory.
-        credentials = self._credentials.get(key, None)
-
-        # Use the refresh predicate to determine if the entire store should be
-        # reloaded. This basically checks if the credentials are invalid
-        # or expired. This covers the situation where another process has
-        # refreshed the credentials and this process doesn't know about it yet.
-        # In that case, this process won't needlessly refresh the credentials.
-        if self._refresh_predicate(credentials):
-            self._load_credentials()
-            credentials = self._credentials.get(key, None)
-
-        return credentials
-
-    def locked_put(self, key, credentials):
-        self._load_credentials()
-        self._credentials[key] = credentials
-        self._write_credentials()
-
-    def locked_delete(self, key):
-        self._load_credentials()
-        self._credentials.pop(key, None)
-        self._write_credentials()
-
-
-def _get_backend(filename):
-    """A helper method to get or create a backend with thread locking.
-
-    This ensures that only one backend is used per-file per-process, so that
-    thread and process locks are appropriately shared.
-
-    Args:
-        filename: The full path to the credential storage file.
-
-    Returns:
-        An instance of :class:`_MultiprocessStorageBackend`.
-    """
-    filename = os.path.abspath(filename)
-
-    with _backends_lock:
-        if filename not in _backends:
-            _backends[filename] = _MultiprocessStorageBackend(filename)
-        return _backends[filename]
-
-
-class MultiprocessFileStorage(client.Storage):
-    """Multiprocess file credential storage.
-
-    Args:
-      filename: The path to the file where credentials will be stored.
-      key: An arbitrary string used to uniquely identify this set of
-          credentials. For example, you may use the user's ID as the key or
-          a combination of the client ID and user ID.
-    """
-    def __init__(self, filename, key):
-        self._key = key
-        self._backend = _get_backend(filename)
-
-    def acquire_lock(self):
-        self._backend.acquire_lock()
-
-    def release_lock(self):
-        self._backend.release_lock()
-
-    def locked_get(self):
-        """Retrieves the current credentials from the store.
-
-        Returns:
-            An instance of :class:`oauth2client.client.Credentials` or `None`.
-        """
-        credential = self._backend.locked_get(self._key)
-
-        if credential is not None:
-            credential.set_store(self)
-
-        return credential
-
-    def locked_put(self, credentials):
-        """Writes the given credentials to the store.
-
-        Args:
-            credentials: an instance of
-                :class:`oauth2client.client.Credentials`.
-        """
-        return self._backend.locked_put(self._key, credentials)
-
-    def locked_delete(self):
-        """Deletes the current credentials from the store."""
-        return self._backend.locked_delete(self._key)

src/oauth2client/contrib/sqlalchemy.py

@@ -1,173 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""OAuth 2.0 utilities for SQLAlchemy.
-
-Utilities for using OAuth 2.0 in conjunction with a SQLAlchemy.
-
-Configuration
-=============
-
-In order to use this storage, you'll need to create table
-with :class:`oauth2client.contrib.sqlalchemy.CredentialsType` column.
-It's recommended to either put this column on some sort of user info
-table or put the column in a table with a belongs-to relationship to
-a user info table.
-
-Here's an example of a simple table with a :class:`CredentialsType`
-column that's related to a user table by the `user_id` key.
-
-.. code-block:: python
-
-    from sqlalchemy import Column, ForeignKey, Integer
-    from sqlalchemy.ext.declarative import declarative_base
-    from sqlalchemy.orm import relationship
-
-    from oauth2client.contrib.sqlalchemy import CredentialsType
-
-
-    Base = declarative_base()
-
-
-    class Credentials(Base):
-        __tablename__ = 'credentials'
-
-        user_id = Column(Integer, ForeignKey('user.id'))
-        credentials = Column(CredentialsType)
-
-
-    class User(Base):
-        id = Column(Integer, primary_key=True)
-        # bunch of other columns
-        credentials = relationship('Credentials')
-
-
-Usage
-=====
-
-With tables ready, you are now able to store credentials in database.
-We will reuse tables defined above.
-
-.. code-block:: python
-
-    from sqlalchemy.orm import Session
-
-    from oauth2client.client import OAuth2Credentials
-    from oauth2client.contrib.sql_alchemy import Storage
-
-    session = Session()
-    user = session.query(User).first()
-    storage = Storage(
-        session=session,
-        model_class=Credentials,
-        # This is the key column used to identify
-        # the row that stores the credentials.
-        key_name='user_id',
-        key_value=user.id,
-        property_name='credentials',
-    )
-
-    # Store
-    credentials = OAuth2Credentials(...)
-    storage.put(credentials)
-
-    # Retrieve
-    credentials = storage.get()
-
-    # Delete
-    storage.delete()
-
-"""
-
-from __future__ import absolute_import
-
-import sqlalchemy.types
-
-from oauth2client import client
-
-
-class CredentialsType(sqlalchemy.types.PickleType):
-    """Type representing credentials.
-
-    Alias for :class:`sqlalchemy.types.PickleType`.
-    """
-
-
-class Storage(client.Storage):
-    """Store and retrieve a single credential to and from SQLAlchemy.
-    This helper presumes the Credentials
-    have been stored as a Credentials column
-    on a db model class.
-    """
-
-    def __init__(self, session, model_class, key_name,
-                 key_value, property_name):
-        """Constructor for Storage.
-
-        Args:
-            session: An instance of :class:`sqlalchemy.orm.Session`.
-            model_class: SQLAlchemy declarative mapping.
-            key_name: string, key name for the entity that has the credentials
-            key_value: key value for the entity that has the credentials
-            property_name: A string indicating which property on the
-                           ``model_class`` to store the credentials.
-                           This property must be a
-                           :class:`CredentialsType` column.
-        """
-        super(Storage, self).__init__()
-
-        self.session = session
-        self.model_class = model_class
-        self.key_name = key_name
-        self.key_value = key_value
-        self.property_name = property_name
-
-    def locked_get(self):
-        """Retrieve stored credential.
-
-        Returns:
-            A :class:`oauth2client.Credentials` instance or `None`.
-        """
-        filters = {self.key_name: self.key_value}
-        query = self.session.query(self.model_class).filter_by(**filters)
-        entity = query.first()
-
-        if entity:
-            credential = getattr(entity, self.property_name)
-            if credential and hasattr(credential, 'set_store'):
-                credential.set_store(self)
-            return credential
-        else:
-            return None
-
-    def locked_put(self, credentials):
-        """Write a credentials to the SQLAlchemy datastore.
-
-        Args:
-            credentials: :class:`oauth2client.Credentials`
-        """
-        filters = {self.key_name: self.key_value}
-        query = self.session.query(self.model_class).filter_by(**filters)
-        entity = query.first()
-
-        if not entity:
-            entity = self.model_class(**filters)
-
-        setattr(entity, self.property_name, credentials)
-        self.session.add(entity)
-
-    def locked_delete(self):
-        """Delete credentials from the SQLAlchemy datastore."""
-        filters = {self.key_name: self.key_value}
-        self.session.query(self.model_class).filter_by(**filters).delete()

src/oauth2client/contrib/xsrfutil.py

@@ -1,101 +0,0 @@
-# Copyright 2014 the Melange authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helper methods for creating & verifying XSRF tokens."""
-
-import base64
-import binascii
-import hmac
-import time
-
-from oauth2client import _helpers
-
-
-# Delimiter character
-DELIMITER = b':'
-
-# 1 hour in seconds
-DEFAULT_TIMEOUT_SECS = 60 * 60
-
-
-@_helpers.positional(2)
-def generate_token(key, user_id, action_id='', when=None):
-    """Generates a URL-safe token for the given user, action, time tuple.
-
-    Args:
-        key: secret key to use.
-        user_id: the user ID of the authenticated user.
-        action_id: a string identifier of the action they requested
-                   authorization for.
-        when: the time in seconds since the epoch at which the user was
-              authorized for this action. If not set the current time is used.
-
-    Returns:
-        A string XSRF protection token.
-    """
-    digester = hmac.new(_helpers._to_bytes(key, encoding='utf-8'))
-    digester.update(_helpers._to_bytes(str(user_id), encoding='utf-8'))
-    digester.update(DELIMITER)
-    digester.update(_helpers._to_bytes(action_id, encoding='utf-8'))
-    digester.update(DELIMITER)
-    when = _helpers._to_bytes(str(when or int(time.time())), encoding='utf-8')
-    digester.update(when)
-    digest = digester.digest()
-
-    token = base64.urlsafe_b64encode(digest + DELIMITER + when)
-    return token
-
-
-@_helpers.positional(3)
-def validate_token(key, token, user_id, action_id="", current_time=None):
-    """Validates that the given token authorizes the user for the action.
-
-    Tokens are invalid if the time of issue is too old or if the token
-    does not match what generateToken outputs (i.e. the token was forged).
-
-    Args:
-        key: secret key to use.
-        token: a string of the token generated by generateToken.
-        user_id: the user ID of the authenticated user.
-        action_id: a string identifier of the action they requested
-                   authorization for.
-
-    Returns:
-        A boolean - True if the user is authorized for the action, False
-        otherwise.
-    """
-    if not token:
-        return False
-    try:
-        decoded = base64.urlsafe_b64decode(token)
-        token_time = int(decoded.split(DELIMITER)[-1])
-    except (TypeError, ValueError, binascii.Error):
-        return False
-    if current_time is None:
-        current_time = time.time()
-    # If the token is too old it's not valid.
-    if current_time - token_time > DEFAULT_TIMEOUT_SECS:
-        return False
-
-    # The given token should match the generated one with the same time.
-    expected_token = generate_token(key, user_id, action_id=action_id,
-                                    when=token_time)
-    if len(token) != len(expected_token):
-        return False
-
-    # Perform constant time comparison to avoid timing attacks
-    different = 0
-    for x, y in zip(bytearray(token), bytearray(expected_token)):
-        different |= x ^ y
-    return not different

src/oauth2client/crypt.py

@@ -1,250 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Crypto-related routines for oauth2client."""
-
-import json
-import logging
-import time
-
-from oauth2client import _helpers
-from oauth2client import _pure_python_crypt
-
-
-RsaSigner = _pure_python_crypt.RsaSigner
-RsaVerifier = _pure_python_crypt.RsaVerifier
-
-CLOCK_SKEW_SECS = 300  # 5 minutes in seconds
-AUTH_TOKEN_LIFETIME_SECS = 300  # 5 minutes in seconds
-MAX_TOKEN_LIFETIME_SECS = 86400  # 1 day in seconds
-
-logger = logging.getLogger(__name__)
-
-
-class AppIdentityError(Exception):
-    """Error to indicate crypto failure."""
-
-
-def _bad_pkcs12_key_as_pem(*args, **kwargs):
-    raise NotImplementedError('pkcs12_key_as_pem requires OpenSSL.')
-
-
-try:
-    from oauth2client import _openssl_crypt
-    OpenSSLSigner = _openssl_crypt.OpenSSLSigner
-    OpenSSLVerifier = _openssl_crypt.OpenSSLVerifier
-    pkcs12_key_as_pem = _openssl_crypt.pkcs12_key_as_pem
-except ImportError:  # pragma: NO COVER
-    OpenSSLVerifier = None
-    OpenSSLSigner = None
-    pkcs12_key_as_pem = _bad_pkcs12_key_as_pem
-
-try:
-    from oauth2client import _pycrypto_crypt
-    PyCryptoSigner = _pycrypto_crypt.PyCryptoSigner
-    PyCryptoVerifier = _pycrypto_crypt.PyCryptoVerifier
-except ImportError:  # pragma: NO COVER
-    PyCryptoVerifier = None
-    PyCryptoSigner = None
-
-
-if OpenSSLSigner:
-    Signer = OpenSSLSigner
-    Verifier = OpenSSLVerifier
-elif PyCryptoSigner:  # pragma: NO COVER
-    Signer = PyCryptoSigner
-    Verifier = PyCryptoVerifier
-else:  # pragma: NO COVER
-    Signer = RsaSigner
-    Verifier = RsaVerifier
-
-
-def make_signed_jwt(signer, payload, key_id=None):
-    """Make a signed JWT.
-
-    See http://self-issued.info/docs/draft-jones-json-web-token.html.
-
-    Args:
-        signer: crypt.Signer, Cryptographic signer.
-        payload: dict, Dictionary of data to convert to JSON and then sign.
-        key_id: string, (Optional) Key ID header.
-
-    Returns:
-        string, The JWT for the payload.
-    """
-    header = {'typ': 'JWT', 'alg': 'RS256'}
-    if key_id is not None:
-        header['kid'] = key_id
-
-    segments = [
-        _helpers._urlsafe_b64encode(_helpers._json_encode(header)),
-        _helpers._urlsafe_b64encode(_helpers._json_encode(payload)),
-    ]
-    signing_input = b'.'.join(segments)
-
-    signature = signer.sign(signing_input)
-    segments.append(_helpers._urlsafe_b64encode(signature))
-
-    logger.debug(str(segments))
-
-    return b'.'.join(segments)
-
-
-def _verify_signature(message, signature, certs):
-    """Verifies signed content using a list of certificates.
-
-    Args:
-        message: string or bytes, The message to verify.
-        signature: string or bytes, The signature on the message.
-        certs: iterable, certificates in PEM format.
-
-    Raises:
-        AppIdentityError: If none of the certificates can verify the message
-                          against the signature.
-    """
-    for pem in certs:
-        verifier = Verifier.from_string(pem, is_x509_cert=True)
-        if verifier.verify(message, signature):
-            return
-
-    # If we have not returned, no certificate confirms the signature.
-    raise AppIdentityError('Invalid token signature')
-
-
-def _check_audience(payload_dict, audience):
-    """Checks audience field from a JWT payload.
-
-    Does nothing if the passed in ``audience`` is null.
-
-    Args:
-        payload_dict: dict, A dictionary containing a JWT payload.
-        audience: string or NoneType, an audience to check for in
-                  the JWT payload.
-
-    Raises:
-        AppIdentityError: If there is no ``'aud'`` field in the payload
-                          dictionary but there is an ``audience`` to check.
-        AppIdentityError: If the ``'aud'`` field in the payload dictionary
-                          does not match the ``audience``.
-    """
-    if audience is None:
-        return
-
-    audience_in_payload = payload_dict.get('aud')
-    if audience_in_payload is None:
-        raise AppIdentityError(
-            'No aud field in token: {0}'.format(payload_dict))
-    if audience_in_payload != audience:
-        raise AppIdentityError('Wrong recipient, {0} != {1}: {2}'.format(
-            audience_in_payload, audience, payload_dict))
-
-
-def _verify_time_range(payload_dict):
-    """Verifies the issued at and expiration from a JWT payload.
-
-    Makes sure the current time (in UTC) falls between the issued at and
-    expiration for the JWT (with some skew allowed for via
-    ``CLOCK_SKEW_SECS``).
-
-    Args:
-        payload_dict: dict, A dictionary containing a JWT payload.
-
-    Raises:
-        AppIdentityError: If there is no ``'iat'`` field in the payload
-                          dictionary.
-        AppIdentityError: If there is no ``'exp'`` field in the payload
-                          dictionary.
-        AppIdentityError: If the JWT expiration is too far in the future (i.e.
-                          if the expiration would imply a token lifetime
-                          longer than what is allowed.)
-        AppIdentityError: If the token appears to have been issued in the
-                          future (up to clock skew).
-        AppIdentityError: If the token appears to have expired in the past
-                          (up to clock skew).
-    """
-    # Get the current time to use throughout.
-    now = int(time.time())
-
-    # Make sure issued at and expiration are in the payload.
-    issued_at = payload_dict.get('iat')
-    if issued_at is None:
-        raise AppIdentityError(
-            'No iat field in token: {0}'.format(payload_dict))
-    expiration = payload_dict.get('exp')
-    if expiration is None:
-        raise AppIdentityError(
-            'No exp field in token: {0}'.format(payload_dict))
-
-    # Make sure the expiration gives an acceptable token lifetime.
-    if expiration >= now + MAX_TOKEN_LIFETIME_SECS:
-        raise AppIdentityError(
-            'exp field too far in future: {0}'.format(payload_dict))
-
-    # Make sure (up to clock skew) that the token wasn't issued in the future.
-    earliest = issued_at - CLOCK_SKEW_SECS
-    if now < earliest:
-        raise AppIdentityError('Token used too early, {0} < {1}: {2}'.format(
-            now, earliest, payload_dict))
-    # Make sure (up to clock skew) that the token isn't already expired.
-    latest = expiration + CLOCK_SKEW_SECS
-    if now > latest:
-        raise AppIdentityError('Token used too late, {0} > {1}: {2}'.format(
-            now, latest, payload_dict))
-
-
-def verify_signed_jwt_with_certs(jwt, certs, audience=None):
-    """Verify a JWT against public certs.
-
-    See http://self-issued.info/docs/draft-jones-json-web-token.html.
-
-    Args:
-        jwt: string, A JWT.
-        certs: dict, Dictionary where values of public keys in PEM format.
-        audience: string, The audience, 'aud', that this JWT should contain. If
-                  None then the JWT's 'aud' parameter is not verified.
-
-    Returns:
-        dict, The deserialized JSON payload in the JWT.
-
-    Raises:
-        AppIdentityError: if any checks are failed.
-    """
-    jwt = _helpers._to_bytes(jwt)
-
-    if jwt.count(b'.') != 2:
-        raise AppIdentityError(
-            'Wrong number of segments in token: {0}'.format(jwt))
-
-    header, payload, signature = jwt.split(b'.')
-    message_to_sign = header + b'.' + payload
-    signature = _helpers._urlsafe_b64decode(signature)
-
-    # Parse token.
-    payload_bytes = _helpers._urlsafe_b64decode(payload)
-    try:
-        payload_dict = json.loads(_helpers._from_bytes(payload_bytes))
-    except:
-        raise AppIdentityError('Can\'t parse token: {0}'.format(payload_bytes))
-
-    # Verify that the signature matches the message.
-    _verify_signature(message_to_sign, signature, certs.values())
-
-    # Verify the issued at and created times in the payload.
-    _verify_time_range(payload_dict)
-
-    # Check audience.
-    _check_audience(payload_dict, audience)
-
-    return payload_dict

src/oauth2client/file.py

@@ -1,95 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for OAuth.
-
-Utilities for making it easier to work with OAuth 2.0
-credentials.
-"""
-
-import os
-import threading
-
-from oauth2client import _helpers
-from oauth2client import client
-
-
-class Storage(client.Storage):
-    """Store and retrieve a single credential to and from a file."""
-
-    def __init__(self, filename):
-        super(Storage, self).__init__(lock=threading.Lock())
-        self._filename = filename
-
-    def locked_get(self):
-        """Retrieve Credential from file.
-
-        Returns:
-            oauth2client.client.Credentials
-
-        Raises:
-            IOError if the file is a symbolic link.
-        """
-        credentials = None
-        _helpers.validate_file(self._filename)
-        try:
-            f = open(self._filename, 'rb')
-            content = f.read()
-            f.close()
-        except IOError:
-            return credentials
-
-        try:
-            credentials = client.Credentials.new_from_json(content)
-            credentials.set_store(self)
-        except ValueError:
-            pass
-
-        return credentials
-
-    def _create_file_if_needed(self):
-        """Create an empty file if necessary.
-
-        This method will not initialize the file. Instead it implements a
-        simple version of "touch" to ensure the file has been created.
-        """
-        if not os.path.exists(self._filename):
-            old_umask = os.umask(0o177)
-            try:
-                open(self._filename, 'a+b').close()
-            finally:
-                os.umask(old_umask)
-
-    def locked_put(self, credentials):
-        """Write Credentials to file.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-
-        Raises:
-            IOError if the file is a symbolic link.
-        """
-        self._create_file_if_needed()
-        _helpers.validate_file(self._filename)
-        f = open(self._filename, 'w')
-        f.write(credentials.to_json())
-        f.close()
-
-    def locked_delete(self):
-        """Delete Credentials file.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-        """
-        os.unlink(self._filename)

src/oauth2client/service_account.py

@@ -1,685 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""oauth2client Service account credentials class."""
-
-import base64
-import copy
-import datetime
-import json
-import time
-
-import oauth2client
-from oauth2client import _helpers
-from oauth2client import client
-from oauth2client import crypt
-from oauth2client import transport
-
-
-_PASSWORD_DEFAULT = 'notasecret'
-_PKCS12_KEY = '_private_key_pkcs12'
-_PKCS12_ERROR = r"""
-This library only implements PKCS#12 support via the pyOpenSSL library.
-Either install pyOpenSSL, or please convert the .p12 file
-to .pem format:
-    $ cat key.p12 | \
-    >   openssl pkcs12 -nodes -nocerts -passin pass:notasecret | \
-    >   openssl rsa > key.pem
-"""
-
-
-class ServiceAccountCredentials(client.AssertionCredentials):
-    """Service Account credential for OAuth 2.0 signed JWT grants.
-
-    Supports
-
-    * JSON keyfile (typically contains a PKCS8 key stored as
-      PEM text)
-    * ``.p12`` key (stores PKCS12 key and certificate)
-
-    Makes an assertion to server using a signed JWT assertion in exchange
-    for an access token.
-
-    This credential does not require a flow to instantiate because it
-    represents a two legged flow, and therefore has all of the required
-    information to generate and refresh its own access tokens.
-
-    Args:
-        service_account_email: string, The email associated with the
-                               service account.
-        signer: ``crypt.Signer``, A signer which can be used to sign content.
-        scopes: List or string, (Optional) Scopes to use when acquiring
-                an access token.
-        private_key_id: string, (Optional) Private key identifier. Typically
-                        only used with a JSON keyfile. Can be sent in the
-                        header of a JWT token assertion.
-        client_id: string, (Optional) Client ID for the project that owns the
-                   service account.
-        user_agent: string, (Optional) User agent to use when sending
-                    request.
-        token_uri: string, URI for token endpoint. For convenience defaults
-                   to Google's endpoints but any OAuth 2.0 provider can be
-                   used.
-        revoke_uri: string, URI for revoke endpoint.  For convenience defaults
-                   to Google's endpoints but any OAuth 2.0 provider can be
-                   used.
-        kwargs: dict, Extra key-value pairs (both strings) to send in the
-                payload body when making an assertion.
-    """
-
-    MAX_TOKEN_LIFETIME_SECS = 3600
-    """Max lifetime of the token (one hour, in seconds)."""
-
-    NON_SERIALIZED_MEMBERS = (
-        frozenset(['_signer']) |
-        client.AssertionCredentials.NON_SERIALIZED_MEMBERS)
-    """Members that aren't serialized when object is converted to JSON."""
-
-    # Can be over-ridden by factory constructors. Used for
-    # serialization/deserialization purposes.
-    _private_key_pkcs8_pem = None
-    _private_key_pkcs12 = None
-    _private_key_password = None
-
-    def __init__(self,
-                 service_account_email,
-                 signer,
-                 scopes='',
-                 private_key_id=None,
-                 client_id=None,
-                 user_agent=None,
-                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
-                 **kwargs):
-
-        super(ServiceAccountCredentials, self).__init__(
-            None, user_agent=user_agent, token_uri=token_uri,
-            revoke_uri=revoke_uri)
-
-        self._service_account_email = service_account_email
-        self._signer = signer
-        self._scopes = _helpers.scopes_to_string(scopes)
-        self._private_key_id = private_key_id
-        self.client_id = client_id
-        self._user_agent = user_agent
-        self._kwargs = kwargs
-
-    def _to_json(self, strip, to_serialize=None):
-        """Utility function that creates JSON repr. of a credentials object.
-
-        Over-ride is needed since PKCS#12 keys will not in general be JSON
-        serializable.
-
-        Args:
-            strip: array, An array of names of members to exclude from the
-                   JSON.
-            to_serialize: dict, (Optional) The properties for this object
-                          that will be serialized. This allows callers to
-                          modify before serializing.
-
-        Returns:
-            string, a JSON representation of this instance, suitable to pass to
-            from_json().
-        """
-        if to_serialize is None:
-            to_serialize = copy.copy(self.__dict__)
-        pkcs12_val = to_serialize.get(_PKCS12_KEY)
-        if pkcs12_val is not None:
-            to_serialize[_PKCS12_KEY] = base64.b64encode(pkcs12_val)
-        return super(ServiceAccountCredentials, self)._to_json(
-            strip, to_serialize=to_serialize)
-
-    @classmethod
-    def _from_parsed_json_keyfile(cls, keyfile_dict, scopes,
-                                  token_uri=None, revoke_uri=None):
-        """Helper for factory constructors from JSON keyfile.
-
-        Args:
-            keyfile_dict: dict-like object, The parsed dictionary-like object
-                          containing the contents of the JSON keyfile.
-            scopes: List or string, Scopes to use when acquiring an
-                    access token.
-            token_uri: string, URI for OAuth 2.0 provider token endpoint.
-                       If unset and not present in keyfile_dict, defaults
-                       to Google's endpoints.
-            revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint.
-                       If unset and not present in keyfile_dict, defaults
-                       to Google's endpoints.
-
-        Returns:
-            ServiceAccountCredentials, a credentials object created from
-            the keyfile contents.
-
-        Raises:
-            ValueError, if the credential type is not :data:`SERVICE_ACCOUNT`.
-            KeyError, if one of the expected keys is not present in
-                the keyfile.
-        """
-        creds_type = keyfile_dict.get('type')
-        if creds_type != client.SERVICE_ACCOUNT:
-            raise ValueError('Unexpected credentials type', creds_type,
-                             'Expected', client.SERVICE_ACCOUNT)
-
-        service_account_email = keyfile_dict['client_email']
-        private_key_pkcs8_pem = keyfile_dict['private_key']
-        private_key_id = keyfile_dict['private_key_id']
-        client_id = keyfile_dict['client_id']
-        if not token_uri:
-            token_uri = keyfile_dict.get('token_uri',
-                                         oauth2client.GOOGLE_TOKEN_URI)
-        if not revoke_uri:
-            revoke_uri = keyfile_dict.get('revoke_uri',
-                                          oauth2client.GOOGLE_REVOKE_URI)
-
-        signer = crypt.Signer.from_string(private_key_pkcs8_pem)
-        credentials = cls(service_account_email, signer, scopes=scopes,
-                          private_key_id=private_key_id,
-                          client_id=client_id, token_uri=token_uri,
-                          revoke_uri=revoke_uri)
-        credentials._private_key_pkcs8_pem = private_key_pkcs8_pem
-        return credentials
-
-    @classmethod
-    def from_json_keyfile_name(cls, filename, scopes='',
-                               token_uri=None, revoke_uri=None):
-
-        """Factory constructor from JSON keyfile by name.
-
-        Args:
-            filename: string, The location of the keyfile.
-            scopes: List or string, (Optional) Scopes to use when acquiring an
-                    access token.
-            token_uri: string, URI for OAuth 2.0 provider token endpoint.
-                       If unset and not present in the key file, defaults
-                       to Google's endpoints.
-            revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint.
-                       If unset and not present in the key file, defaults
-                       to Google's endpoints.
-
-        Returns:
-            ServiceAccountCredentials, a credentials object created from
-            the keyfile.
-
-        Raises:
-            ValueError, if the credential type is not :data:`SERVICE_ACCOUNT`.
-            KeyError, if one of the expected keys is not present in
-                the keyfile.
-        """
-        with open(filename, 'r') as file_obj:
-            client_credentials = json.load(file_obj)
-        return cls._from_parsed_json_keyfile(client_credentials, scopes,
-                                             token_uri=token_uri,
-                                             revoke_uri=revoke_uri)
-
-    @classmethod
-    def from_json_keyfile_dict(cls, keyfile_dict, scopes='',
-                               token_uri=None, revoke_uri=None):
-        """Factory constructor from parsed JSON keyfile.
-
-        Args:
-            keyfile_dict: dict-like object, The parsed dictionary-like object
-                          containing the contents of the JSON keyfile.
-            scopes: List or string, (Optional) Scopes to use when acquiring an
-                    access token.
-            token_uri: string, URI for OAuth 2.0 provider token endpoint.
-                       If unset and not present in keyfile_dict, defaults
-                       to Google's endpoints.
-            revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint.
-                       If unset and not present in keyfile_dict, defaults
-                       to Google's endpoints.
-
-        Returns:
-            ServiceAccountCredentials, a credentials object created from
-            the keyfile.
-
-        Raises:
-            ValueError, if the credential type is not :data:`SERVICE_ACCOUNT`.
-            KeyError, if one of the expected keys is not present in
-                the keyfile.
-        """
-        return cls._from_parsed_json_keyfile(keyfile_dict, scopes,
-                                             token_uri=token_uri,
-                                             revoke_uri=revoke_uri)
-
-    @classmethod
-    def _from_p12_keyfile_contents(cls, service_account_email,
-                                   private_key_pkcs12,
-                                   private_key_password=None, scopes='',
-                                   token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                                   revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
-        """Factory constructor from JSON keyfile.
-
-        Args:
-            service_account_email: string, The email associated with the
-                                   service account.
-            private_key_pkcs12: string, The contents of a PKCS#12 keyfile.
-            private_key_password: string, (Optional) Password for PKCS#12
-                                  private key. Defaults to ``notasecret``.
-            scopes: List or string, (Optional) Scopes to use when acquiring an
-                    access token.
-            token_uri: string, URI for token endpoint. For convenience defaults
-                       to Google's endpoints but any OAuth 2.0 provider can be
-                       used.
-            revoke_uri: string, URI for revoke endpoint. For convenience
-                        defaults to Google's endpoints but any OAuth 2.0
-                        provider can be used.
-
-        Returns:
-            ServiceAccountCredentials, a credentials object created from
-            the keyfile.
-
-        Raises:
-            NotImplementedError if pyOpenSSL is not installed / not the
-            active crypto library.
-        """
-        if private_key_password is None:
-            private_key_password = _PASSWORD_DEFAULT
-        if crypt.Signer is not crypt.OpenSSLSigner:
-            raise NotImplementedError(_PKCS12_ERROR)
-        signer = crypt.Signer.from_string(private_key_pkcs12,
-                                          private_key_password)
-        credentials = cls(service_account_email, signer, scopes=scopes,
-                          token_uri=token_uri, revoke_uri=revoke_uri)
-        credentials._private_key_pkcs12 = private_key_pkcs12
-        credentials._private_key_password = private_key_password
-        return credentials
-
-    @classmethod
-    def from_p12_keyfile(cls, service_account_email, filename,
-                         private_key_password=None, scopes='',
-                         token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                         revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
-
-        """Factory constructor from JSON keyfile.
-
-        Args:
-            service_account_email: string, The email associated with the
-                                   service account.
-            filename: string, The location of the PKCS#12 keyfile.
-            private_key_password: string, (Optional) Password for PKCS#12
-                                  private key. Defaults to ``notasecret``.
-            scopes: List or string, (Optional) Scopes to use when acquiring an
-                    access token.
-            token_uri: string, URI for token endpoint. For convenience defaults
-                       to Google's endpoints but any OAuth 2.0 provider can be
-                       used.
-            revoke_uri: string, URI for revoke endpoint. For convenience
-                        defaults to Google's endpoints but any OAuth 2.0
-                        provider can be used.
-
-        Returns:
-            ServiceAccountCredentials, a credentials object created from
-            the keyfile.
-
-        Raises:
-            NotImplementedError if pyOpenSSL is not installed / not the
-            active crypto library.
-        """
-        with open(filename, 'rb') as file_obj:
-            private_key_pkcs12 = file_obj.read()
-        return cls._from_p12_keyfile_contents(
-            service_account_email, private_key_pkcs12,
-            private_key_password=private_key_password, scopes=scopes,
-            token_uri=token_uri, revoke_uri=revoke_uri)
-
-    @classmethod
-    def from_p12_keyfile_buffer(cls, service_account_email, file_buffer,
-                                private_key_password=None, scopes='',
-                                token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                                revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
-        """Factory constructor from JSON keyfile.
-
-        Args:
-            service_account_email: string, The email associated with the
-                                   service account.
-            file_buffer: stream, A buffer that implements ``read()``
-                         and contains the PKCS#12 key contents.
-            private_key_password: string, (Optional) Password for PKCS#12
-                                  private key. Defaults to ``notasecret``.
-            scopes: List or string, (Optional) Scopes to use when acquiring an
-                    access token.
-            token_uri: string, URI for token endpoint. For convenience defaults
-                       to Google's endpoints but any OAuth 2.0 provider can be
-                       used.
-            revoke_uri: string, URI for revoke endpoint. For convenience
-                        defaults to Google's endpoints but any OAuth 2.0
-                        provider can be used.
-
-        Returns:
-            ServiceAccountCredentials, a credentials object created from
-            the keyfile.
-
-        Raises:
-            NotImplementedError if pyOpenSSL is not installed / not the
-            active crypto library.
-        """
-        private_key_pkcs12 = file_buffer.read()
-        return cls._from_p12_keyfile_contents(
-            service_account_email, private_key_pkcs12,
-            private_key_password=private_key_password, scopes=scopes,
-            token_uri=token_uri, revoke_uri=revoke_uri)
-
-    def _generate_assertion(self):
-        """Generate the assertion that will be used in the request."""
-        now = int(time.time())
-        payload = {
-            'aud': self.token_uri,
-            'scope': self._scopes,
-            'iat': now,
-            'exp': now + self.MAX_TOKEN_LIFETIME_SECS,
-            'iss': self._service_account_email,
-        }
-        payload.update(self._kwargs)
-        return crypt.make_signed_jwt(self._signer, payload,
-                                     key_id=self._private_key_id)
-
-    def sign_blob(self, blob):
-        """Cryptographically sign a blob (of bytes).
-
-        Implements abstract method
-        :meth:`oauth2client.client.AssertionCredentials.sign_blob`.
-
-        Args:
-            blob: bytes, Message to be signed.
-
-        Returns:
-            tuple, A pair of the private key ID used to sign the blob and
-            the signed contents.
-        """
-        return self._private_key_id, self._signer.sign(blob)
-
-    @property
-    def service_account_email(self):
-        """Get the email for the current service account.
-
-        Returns:
-            string, The email associated with the service account.
-        """
-        return self._service_account_email
-
-    @property
-    def serialization_data(self):
-        # NOTE: This is only useful for JSON keyfile.
-        return {
-            'type': 'service_account',
-            'client_email': self._service_account_email,
-            'private_key_id': self._private_key_id,
-            'private_key': self._private_key_pkcs8_pem,
-            'client_id': self.client_id,
-        }
-
-    @classmethod
-    def from_json(cls, json_data):
-        """Deserialize a JSON-serialized instance.
-
-        Inverse to :meth:`to_json`.
-
-        Args:
-            json_data: dict or string, Serialized JSON (as a string or an
-                       already parsed dictionary) representing a credential.
-
-        Returns:
-            ServiceAccountCredentials from the serialized data.
-        """
-        if not isinstance(json_data, dict):
-            json_data = json.loads(_helpers._from_bytes(json_data))
-
-        private_key_pkcs8_pem = None
-        pkcs12_val = json_data.get(_PKCS12_KEY)
-        password = None
-        if pkcs12_val is None:
-            private_key_pkcs8_pem = json_data['_private_key_pkcs8_pem']
-            signer = crypt.Signer.from_string(private_key_pkcs8_pem)
-        else:
-            # NOTE: This assumes that private_key_pkcs8_pem is not also
-            #       in the serialized data. This would be very incorrect
-            #       state.
-            pkcs12_val = base64.b64decode(pkcs12_val)
-            password = json_data['_private_key_password']
-            signer = crypt.Signer.from_string(pkcs12_val, password)
-
-        credentials = cls(
-            json_data['_service_account_email'],
-            signer,
-            scopes=json_data['_scopes'],
-            private_key_id=json_data['_private_key_id'],
-            client_id=json_data['client_id'],
-            user_agent=json_data['_user_agent'],
-            **json_data['_kwargs']
-        )
-        if private_key_pkcs8_pem is not None:
-            credentials._private_key_pkcs8_pem = private_key_pkcs8_pem
-        if pkcs12_val is not None:
-            credentials._private_key_pkcs12 = pkcs12_val
-        if password is not None:
-            credentials._private_key_password = password
-        credentials.invalid = json_data['invalid']
-        credentials.access_token = json_data['access_token']
-        credentials.token_uri = json_data['token_uri']
-        credentials.revoke_uri = json_data['revoke_uri']
-        token_expiry = json_data.get('token_expiry', None)
-        if token_expiry is not None:
-            credentials.token_expiry = datetime.datetime.strptime(
-                token_expiry, client.EXPIRY_FORMAT)
-        return credentials
-
-    def create_scoped_required(self):
-        return not self._scopes
-
-    def create_scoped(self, scopes):
-        result = self.__class__(self._service_account_email,
-                                self._signer,
-                                scopes=scopes,
-                                private_key_id=self._private_key_id,
-                                client_id=self.client_id,
-                                user_agent=self._user_agent,
-                                **self._kwargs)
-        result.token_uri = self.token_uri
-        result.revoke_uri = self.revoke_uri
-        result._private_key_pkcs8_pem = self._private_key_pkcs8_pem
-        result._private_key_pkcs12 = self._private_key_pkcs12
-        result._private_key_password = self._private_key_password
-        return result
-
-    def create_with_claims(self, claims):
-        """Create credentials that specify additional claims.
-
-        Args:
-            claims: dict, key-value pairs for claims.
-
-        Returns:
-            ServiceAccountCredentials, a copy of the current service account
-            credentials with updated claims to use when obtaining access
-            tokens.
-        """
-        new_kwargs = dict(self._kwargs)
-        new_kwargs.update(claims)
-        result = self.__class__(self._service_account_email,
-                                self._signer,
-                                scopes=self._scopes,
-                                private_key_id=self._private_key_id,
-                                client_id=self.client_id,
-                                user_agent=self._user_agent,
-                                **new_kwargs)
-        result.token_uri = self.token_uri
-        result.revoke_uri = self.revoke_uri
-        result._private_key_pkcs8_pem = self._private_key_pkcs8_pem
-        result._private_key_pkcs12 = self._private_key_pkcs12
-        result._private_key_password = self._private_key_password
-        return result
-
-    def create_delegated(self, sub):
-        """Create credentials that act as domain-wide delegation of authority.
-
-        Use the ``sub`` parameter as the subject to delegate on behalf of
-        that user.
-
-        For example::
-
-          >>> account_sub = 'foo@email.com'
-          >>> delegate_creds = creds.create_delegated(account_sub)
-
-        Args:
-            sub: string, An email address that this service account will
-                 act on behalf of (via domain-wide delegation).
-
-        Returns:
-            ServiceAccountCredentials, a copy of the current service account
-            updated to act on behalf of ``sub``.
-        """
-        return self.create_with_claims({'sub': sub})
-
-
-def _datetime_to_secs(utc_time):
-    # TODO(issue 298): use time_delta.total_seconds()
-    # time_delta.total_seconds() not supported in Python 2.6
-    epoch = datetime.datetime(1970, 1, 1)
-    time_delta = utc_time - epoch
-    return time_delta.days * 86400 + time_delta.seconds
-
-
-class _JWTAccessCredentials(ServiceAccountCredentials):
-    """Self signed JWT credentials.
-
-    Makes an assertion to server using a self signed JWT from service account
-    credentials.  These credentials do NOT use OAuth 2.0 and instead
-    authenticate directly.
-    """
-    _MAX_TOKEN_LIFETIME_SECS = 3600
-    """Max lifetime of the token (one hour, in seconds)."""
-
-    def __init__(self,
-                 service_account_email,
-                 signer,
-                 scopes=None,
-                 private_key_id=None,
-                 client_id=None,
-                 user_agent=None,
-                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
-                 additional_claims=None):
-        if additional_claims is None:
-            additional_claims = {}
-        super(_JWTAccessCredentials, self).__init__(
-            service_account_email,
-            signer,
-            private_key_id=private_key_id,
-            client_id=client_id,
-            user_agent=user_agent,
-            token_uri=token_uri,
-            revoke_uri=revoke_uri,
-            **additional_claims)
-
-    def authorize(self, http):
-        """Authorize an httplib2.Http instance with a JWT assertion.
-
-        Unless specified, the 'aud' of the assertion will be the base
-        uri of the request.
-
-        Args:
-            http: An instance of ``httplib2.Http`` or something that acts
-                  like it.
-        Returns:
-            A modified instance of http that was passed in.
-        Example::
-            h = httplib2.Http()
-            h = credentials.authorize(h)
-        """
-        transport.wrap_http_for_jwt_access(self, http)
-        return http
-
-    def get_access_token(self, http=None, additional_claims=None):
-        """Create a signed jwt.
-
-        Args:
-            http: unused
-            additional_claims: dict, additional claims to add to
-                the payload of the JWT.
-        Returns:
-            An AccessTokenInfo with the signed jwt
-        """
-        if additional_claims is None:
-            if self.access_token is None or self.access_token_expired:
-                self.refresh(None)
-            return client.AccessTokenInfo(
-              access_token=self.access_token, expires_in=self._expires_in())
-        else:
-            # Create a 1 time token
-            token, unused_expiry = self._create_token(additional_claims)
-            return client.AccessTokenInfo(
-              access_token=token, expires_in=self._MAX_TOKEN_LIFETIME_SECS)
-
-    def revoke(self, http):
-        """Cannot revoke JWTAccessCredentials tokens."""
-        pass
-
-    def create_scoped_required(self):
-        # JWTAccessCredentials are unscoped by definition
-        return True
-
-    def create_scoped(self, scopes, token_uri=oauth2client.GOOGLE_TOKEN_URI,
-                      revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
-        # Returns an OAuth2 credentials with the given scope
-        result = ServiceAccountCredentials(self._service_account_email,
-                                           self._signer,
-                                           scopes=scopes,
-                                           private_key_id=self._private_key_id,
-                                           client_id=self.client_id,
-                                           user_agent=self._user_agent,
-                                           token_uri=token_uri,
-                                           revoke_uri=revoke_uri,
-                                           **self._kwargs)
-        if self._private_key_pkcs8_pem is not None:
-            result._private_key_pkcs8_pem = self._private_key_pkcs8_pem
-        if self._private_key_pkcs12 is not None:
-            result._private_key_pkcs12 = self._private_key_pkcs12
-        if self._private_key_password is not None:
-            result._private_key_password = self._private_key_password
-        return result
-
-    def refresh(self, http):
-        """Refreshes the access_token.
-
-        The HTTP object is unused since no request needs to be made to
-        get a new token, it can just be generated locally.
-
-        Args:
-            http: unused HTTP object
-        """
-        self._refresh(None)
-
-    def _refresh(self, http):
-        """Refreshes the access_token.
-
-        Args:
-            http: unused HTTP object
-        """
-        self.access_token, self.token_expiry = self._create_token()
-
-    def _create_token(self, additional_claims=None):
-        now = client._UTCNOW()
-        lifetime = datetime.timedelta(seconds=self._MAX_TOKEN_LIFETIME_SECS)
-        expiry = now + lifetime
-        payload = {
-            'iat': _datetime_to_secs(now),
-            'exp': _datetime_to_secs(expiry),
-            'iss': self._service_account_email,
-            'sub': self._service_account_email
-        }
-        payload.update(self._kwargs)
-        if additional_claims is not None:
-            payload.update(additional_claims)
-        jwt = crypt.make_signed_jwt(self._signer, payload,
-                                    key_id=self._private_key_id)
-        return jwt.decode('ascii'), expiry

src/oauth2client/tools.py

@@ -1,256 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Command-line tools for authenticating via OAuth 2.0
-
-Do the OAuth 2.0 Web Server dance for a command line application. Stores the
-generated credentials in a common file that is used by other example apps in
-the same directory.
-"""
-
-from __future__ import print_function
-
-import logging
-import socket
-import sys
-
-from six.moves import BaseHTTPServer
-from six.moves import http_client
-from six.moves import input
-from six.moves import urllib
-
-from oauth2client import _helpers
-from oauth2client import client
-
-
-__all__ = ['argparser', 'run_flow', 'message_if_missing']
-
-_CLIENT_SECRETS_MESSAGE = """WARNING: Please configure OAuth 2.0
-
-To make this sample run you will need to populate the client_secrets.json file
-found at:
-
-   {file_path}
-
-with information from the APIs Console <https://code.google.com/apis/console>.
-
-"""
-
-_FAILED_START_MESSAGE = """
-Failed to start a local webserver listening on either port 8080
-or port 8090. Please check your firewall settings and locally
-running programs that may be blocking or using those ports.
-
-Falling back to --noauth_local_webserver and continuing with
-authorization.
-"""
-
-_BROWSER_OPENED_MESSAGE = """
-Your browser has been opened to visit:
-
-    {address}
-
-If your browser is on a different machine then exit and re-run this
-application with the command-line parameter
-
-  --noauth_local_webserver
-"""
-
-_GO_TO_LINK_MESSAGE = """
-Go to the following link in your browser:
-
-    {address}
-"""
-
-
-def _CreateArgumentParser():
-    try:
-        import argparse
-    except ImportError:  # pragma: NO COVER
-        return None
-    parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument('--auth_host_name', default='localhost',
-                        help='Hostname when running a local web server.')
-    parser.add_argument('--noauth_local_webserver', action='store_true',
-                        default=False, help='Do not run a local web server.')
-    parser.add_argument('--auth_host_port', default=[8080, 8090], type=int,
-                        nargs='*', help='Port web server should listen on.')
-    parser.add_argument(
-        '--logging_level', default='ERROR',
-        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
-        help='Set the logging level of detail.')
-    return parser
-
-
-# argparser is an ArgumentParser that contains command-line options expected
-# by tools.run(). Pass it in as part of the 'parents' argument to your own
-# ArgumentParser.
-argparser = _CreateArgumentParser()
-
-
-class ClientRedirectServer(BaseHTTPServer.HTTPServer):
-    """A server to handle OAuth 2.0 redirects back to localhost.
-
-    Waits for a single request and parses the query parameters
-    into query_params and then stops serving.
-    """
-    query_params = {}
-
-
-class ClientRedirectHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-    """A handler for OAuth 2.0 redirects back to localhost.
-
-    Waits for a single request and parses the query parameters
-    into the servers query_params and then stops serving.
-    """
-
-    def do_GET(self):
-        """Handle a GET request.
-
-        Parses the query parameters and prints a message
-        if the flow has completed. Note that we can't detect
-        if an error occurred.
-        """
-        self.send_response(http_client.OK)
-        self.send_header('Content-type', 'text/html')
-        self.end_headers()
-        parts = urllib.parse.urlparse(self.path)
-        query = _helpers.parse_unique_urlencoded(parts.query)
-        self.server.query_params = query
-        self.wfile.write(
-            b'<html><head><title>Authentication Status</title></head>')
-        self.wfile.write(
-            b'<body><p>The authentication flow has completed.</p>')
-        self.wfile.write(b'</body></html>')
-
-    def log_message(self, format, *args):
-        """Do not log messages to stdout while running as cmd. line program."""
-
-
-@_helpers.positional(3)
-def run_flow(flow, storage, flags=None, http=None):
-    """Core code for a command-line application.
-
-    The ``run()`` function is called from your application and runs
-    through all the steps to obtain credentials. It takes a ``Flow``
-    argument and attempts to open an authorization server page in the
-    user's default web browser. The server asks the user to grant your
-    application access to the user's data. If the user grants access,
-    the ``run()`` function returns new credentials. The new credentials
-    are also stored in the ``storage`` argument, which updates the file
-    associated with the ``Storage`` object.
-
-    It presumes it is run from a command-line application and supports the
-    following flags:
-
-        ``--auth_host_name`` (string, default: ``localhost``)
-           Host name to use when running a local web server to handle
-           redirects during OAuth authorization.
-
-        ``--auth_host_port`` (integer, default: ``[8080, 8090]``)
-           Port to use when running a local web server to handle redirects
-           during OAuth authorization. Repeat this option to specify a list
-           of values.
-
-        ``--[no]auth_local_webserver`` (boolean, default: ``True``)
-           Run a local web server to handle redirects during OAuth
-           authorization.
-
-    The tools module defines an ``ArgumentParser`` the already contains the
-    flag definitions that ``run()`` requires. You can pass that
-    ``ArgumentParser`` to your ``ArgumentParser`` constructor::
-
-        parser = argparse.ArgumentParser(
-            description=__doc__,
-            formatter_class=argparse.RawDescriptionHelpFormatter,
-            parents=[tools.argparser])
-        flags = parser.parse_args(argv)
-
-    Args:
-        flow: Flow, an OAuth 2.0 Flow to step through.
-        storage: Storage, a ``Storage`` to store the credential in.
-        flags: ``argparse.Namespace``, (Optional) The command-line flags. This
-               is the object returned from calling ``parse_args()`` on
-               ``argparse.ArgumentParser`` as described above. Defaults
-               to ``argparser.parse_args()``.
-        http: An instance of ``httplib2.Http.request`` or something that
-              acts like it.
-
-    Returns:
-        Credentials, the obtained credential.
-    """
-    if flags is None:
-        flags = argparser.parse_args()
-    logging.getLogger().setLevel(getattr(logging, flags.logging_level))
-    if not flags.noauth_local_webserver:
-        success = False
-        port_number = 0
-        for port in flags.auth_host_port:
-            port_number = port
-            try:
-                httpd = ClientRedirectServer((flags.auth_host_name, port),
-                                             ClientRedirectHandler)
-            except socket.error:
-                pass
-            else:
-                success = True
-                break
-        flags.noauth_local_webserver = not success
-        if not success:
-            print(_FAILED_START_MESSAGE)
-
-    if not flags.noauth_local_webserver:
-        oauth_callback = 'http://{host}:{port}/'.format(
-            host=flags.auth_host_name, port=port_number)
-    else:
-        oauth_callback = client.OOB_CALLBACK_URN
-    flow.redirect_uri = oauth_callback
-    authorize_url = flow.step1_get_authorize_url()
-
-    if not flags.noauth_local_webserver:
-        import webbrowser
-        webbrowser.open(authorize_url, new=1, autoraise=True)
-        print(_BROWSER_OPENED_MESSAGE.format(address=authorize_url))
-    else:
-        print(_GO_TO_LINK_MESSAGE.format(address=authorize_url))
-
-    code = None
-    if not flags.noauth_local_webserver:
-        httpd.handle_request()
-        if 'error' in httpd.query_params:
-            sys.exit('Authentication request was rejected.')
-        if 'code' in httpd.query_params:
-            code = httpd.query_params['code']
-        else:
-            print('Failed to find "code" in the query parameters '
-                  'of the redirect.')
-            sys.exit('Try running with --noauth_local_webserver.')
-    else:
-        code = input('Enter verification code: ').strip()
-
-    try:
-        credential = flow.step2_exchange(code, http=http)
-    except client.FlowExchangeError as e:
-        sys.exit('Authentication has failed: {0}'.format(e))
-
-    storage.put(credential)
-    credential.set_store(storage)
-    print('Authentication successful.')
-
-    return credential
-
-
-def message_if_missing(filename):
-    """Helpful message to display if the CLIENT_SECRETS file is missing."""
-    return _CLIENT_SECRETS_MESSAGE.format(file_path=filename)

src/oauth2client/transport.py

@@ -1,285 +0,0 @@
-# Copyright 2016 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-import httplib2
-import six
-from six.moves import http_client
-
-from oauth2client import _helpers
-
-
-_LOGGER = logging.getLogger(__name__)
-# Properties present in file-like streams / buffers.
-_STREAM_PROPERTIES = ('read', 'seek', 'tell')
-
-# Google Data client libraries may need to set this to [401, 403].
-REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,)
-
-
-class MemoryCache(object):
-    """httplib2 Cache implementation which only caches locally."""
-
-    def __init__(self):
-        self.cache = {}
-
-    def get(self, key):
-        return self.cache.get(key)
-
-    def set(self, key, value):
-        self.cache[key] = value
-
-    def delete(self, key):
-        self.cache.pop(key, None)
-
-
-def get_cached_http():
-    """Return an HTTP object which caches results returned.
-
-    This is intended to be used in methods like
-    oauth2client.client.verify_id_token(), which calls to the same URI
-    to retrieve certs.
-
-    Returns:
-        httplib2.Http, an HTTP object with a MemoryCache
-    """
-    return _CACHED_HTTP
-
-
-def get_http_object(*args, **kwargs):
-    """Return a new HTTP object.
-
-    Args:
-        *args: tuple, The positional arguments to be passed when
-               contructing a new HTTP object.
-        **kwargs: dict, The keyword arguments to be passed when
-                  contructing a new HTTP object.
-
-    Returns:
-        httplib2.Http, an HTTP object.
-    """
-    return httplib2.Http(*args, **kwargs)
-
-
-def _initialize_headers(headers):
-    """Creates a copy of the headers.
-
-    Args:
-        headers: dict, request headers to copy.
-
-    Returns:
-        dict, the copied headers or a new dictionary if the headers
-        were None.
-    """
-    return {} if headers is None else dict(headers)
-
-
-def _apply_user_agent(headers, user_agent):
-    """Adds a user-agent to the headers.
-
-    Args:
-        headers: dict, request headers to add / modify user
-                 agent within.
-        user_agent: str, the user agent to add.
-
-    Returns:
-        dict, the original headers passed in, but modified if the
-        user agent is not None.
-    """
-    if user_agent is not None:
-        if 'user-agent' in headers:
-            headers['user-agent'] = (user_agent + ' ' + headers['user-agent'])
-        else:
-            headers['user-agent'] = user_agent
-
-    return headers
-
-
-def clean_headers(headers):
-    """Forces header keys and values to be strings, i.e not unicode.
-
-    The httplib module just concats the header keys and values in a way that
-    may make the message header a unicode string, which, if it then tries to
-    contatenate to a binary request body may result in a unicode decode error.
-
-    Args:
-        headers: dict, A dictionary of headers.
-
-    Returns:
-        The same dictionary but with all the keys converted to strings.
-    """
-    clean = {}
-    try:
-        for k, v in six.iteritems(headers):
-            if not isinstance(k, six.binary_type):
-                k = str(k)
-            if not isinstance(v, six.binary_type):
-                v = str(v)
-            clean[_helpers._to_bytes(k)] = _helpers._to_bytes(v)
-    except UnicodeEncodeError:
-        from oauth2client.client import NonAsciiHeaderError
-        raise NonAsciiHeaderError(k, ': ', v)
-    return clean
-
-
-def wrap_http_for_auth(credentials, http):
-    """Prepares an HTTP object's request method for auth.
-
-    Wraps HTTP requests with logic to catch auth failures (typically
-    identified via a 401 status code). In the event of failure, tries
-    to refresh the token used and then retry the original request.
-
-    Args:
-        credentials: Credentials, the credentials used to identify
-                     the authenticated user.
-        http: httplib2.Http, an http object to be used to make
-              auth requests.
-    """
-    orig_request_method = http.request
-
-    # The closure that will replace 'httplib2.Http.request'.
-    def new_request(uri, method='GET', body=None, headers=None,
-                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
-                    connection_type=None):
-        if not credentials.access_token:
-            _LOGGER.info('Attempting refresh to obtain '
-                         'initial access_token')
-            credentials._refresh(orig_request_method)
-
-        # Clone and modify the request headers to add the appropriate
-        # Authorization header.
-        headers = _initialize_headers(headers)
-        credentials.apply(headers)
-        _apply_user_agent(headers, credentials.user_agent)
-
-        body_stream_position = None
-        # Check if the body is a file-like stream.
-        if all(getattr(body, stream_prop, None) for stream_prop in
-               _STREAM_PROPERTIES):
-            body_stream_position = body.tell()
-
-        resp, content = request(orig_request_method, uri, method, body,
-                                clean_headers(headers),
-                                redirections, connection_type)
-
-        # A stored token may expire between the time it is retrieved and
-        # the time the request is made, so we may need to try twice.
-        max_refresh_attempts = 2
-        for refresh_attempt in range(max_refresh_attempts):
-            if resp.status not in REFRESH_STATUS_CODES:
-                break
-            _LOGGER.info('Refreshing due to a %s (attempt %s/%s)',
-                         resp.status, refresh_attempt + 1,
-                         max_refresh_attempts)
-            credentials._refresh(orig_request_method)
-            credentials.apply(headers)
-            if body_stream_position is not None:
-                body.seek(body_stream_position)
-
-            resp, content = request(orig_request_method, uri, method, body,
-                                    clean_headers(headers),
-                                    redirections, connection_type)
-
-        return resp, content
-
-    # Replace the request method with our own closure.
-    http.request = new_request
-
-    # Set credentials as a property of the request method.
-    http.request.credentials = credentials
-
-
-def wrap_http_for_jwt_access(credentials, http):
-    """Prepares an HTTP object's request method for JWT access.
-
-    Wraps HTTP requests with logic to catch auth failures (typically
-    identified via a 401 status code). In the event of failure, tries
-    to refresh the token used and then retry the original request.
-
-    Args:
-        credentials: _JWTAccessCredentials, the credentials used to identify
-                     a service account that uses JWT access tokens.
-        http: httplib2.Http, an http object to be used to make
-              auth requests.
-    """
-    orig_request_method = http.request
-    wrap_http_for_auth(credentials, http)
-    # The new value of ``http.request`` set by ``wrap_http_for_auth``.
-    authenticated_request_method = http.request
-
-    # The closure that will replace 'httplib2.Http.request'.
-    def new_request(uri, method='GET', body=None, headers=None,
-                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
-                    connection_type=None):
-        if 'aud' in credentials._kwargs:
-            # Preemptively refresh token, this is not done for OAuth2
-            if (credentials.access_token is None or
-                    credentials.access_token_expired):
-                credentials.refresh(None)
-            return request(authenticated_request_method, uri,
-                           method, body, headers, redirections,
-                           connection_type)
-        else:
-            # If we don't have an 'aud' (audience) claim,
-            # create a 1-time token with the uri root as the audience
-            headers = _initialize_headers(headers)
-            _apply_user_agent(headers, credentials.user_agent)
-            uri_root = uri.split('?', 1)[0]
-            token, unused_expiry = credentials._create_token({'aud': uri_root})
-
-            headers['Authorization'] = 'Bearer ' + token
-            return request(orig_request_method, uri, method, body,
-                           clean_headers(headers),
-                           redirections, connection_type)
-
-    # Replace the request method with our own closure.
-    http.request = new_request
-
-    # Set credentials as a property of the request method.
-    http.request.credentials = credentials
-
-
-def request(http, uri, method='GET', body=None, headers=None,
-            redirections=httplib2.DEFAULT_MAX_REDIRECTS,
-            connection_type=None):
-    """Make an HTTP request with an HTTP object and arguments.
-
-    Args:
-        http: httplib2.Http, an http object to be used to make requests.
-        uri: string, The URI to be requested.
-        method: string, The HTTP method to use for the request. Defaults
-                to 'GET'.
-        body: string, The payload / body in HTTP request. By default
-              there is no payload.
-        headers: dict, Key-value pairs of request headers. By default
-                 there are no headers.
-        redirections: int, The number of allowed 203 redirects for
-                      the request. Defaults to 5.
-        connection_type: httplib.HTTPConnection, a subclass to be used for
-                         establishing connection. If not set, the type
-                         will be determined from the ``uri``.
-
-    Returns:
-        tuple, a pair of a httplib2.Response with the status code and other
-        headers and the bytes of the content returned.
-    """
-    # NOTE: Allowing http or http.request is temporary (See Issue 601).
-    http_callable = getattr(http, 'request', http)
-    return http_callable(uri, method=method, body=body, headers=headers,
-                         redirections=redirections,
-                         connection_type=connection_type)
-
-
-_CACHED_HTTP = httplib2.Http(MemoryCache())

src/passlib/__init__.py

@@ -1,3 +0,0 @@
-"""passlib - suite of password hashing & generation routines"""
-
-__version__ = '1.7.1'

src/passlib/_setup/__init__.py

@@ -1 +0,0 @@
-"""passlib.setup - helpers used by passlib's setup.py script"""

src/passlib/_setup/docdist.py

@@ -1,87 +0,0 @@
-"""custom command to build doc.zip file"""
-#=============================================================================
-# imports
-#=============================================================================
-# core
-import os
-from distutils import dir_util
-from distutils.cmd import Command
-from distutils.errors import *
-from distutils.spawn import spawn
-# local
-__all__ = [
-    "docdist"
-]
-#=============================================================================
-# command
-#=============================================================================
-class docdist(Command):
-
-    description = "create zip file containing standalone html docs"
-
-    user_options = [
-        ('build-dir=', None, 'Build directory'),
-        ('dist-dir=', 'd',
-         "directory to put the source distribution archive(s) in "
-         "[default: dist]"),
-        ('format=', 'f',
-         "archive format to create (tar, ztar, gztar, zip)"),
-        ('sign', 's', 'sign files using gpg'),
-        ('identity=', 'i', 'GPG identity used to sign files'),
-    ]
-
-    def initialize_options(self):
-        self.build_dir = None
-        self.dist_dir = None
-        self.format = None
-        self.keep_temp = False
-        self.sign = False
-        self.identity = None
-
-    def finalize_options(self):
-        if self.identity and not self.sign:
-            raise DistutilsOptionError(
-                "Must use --sign for --identity to have meaning"
-            )
-        if self.build_dir is None:
-            cmd = self.get_finalized_command('build')
-            self.build_dir = os.path.join(cmd.build_base, 'docdist')
-        if not self.dist_dir:
-            self.dist_dir = "dist"
-        if not self.format:
-            self.format = "zip"
-
-    def run(self):
-        # call build sphinx to build docs
-        self.run_command("build_sphinx")
-        cmd = self.get_finalized_command("build_sphinx")
-        source_dir = cmd.builder_target_dir
-
-        # copy to directory with appropriate name
-        dist = self.distribution
-        arc_name = "%s-docs-%s" % (dist.get_name(), dist.get_version())
-        tmp_dir = os.path.join(self.build_dir, arc_name)
-        if os.path.exists(tmp_dir):
-            dir_util.remove_tree(tmp_dir, dry_run=self.dry_run)
-        self.copy_tree(source_dir, tmp_dir, preserve_symlinks=True)
-
-        # make archive from dir
-        arc_base = os.path.join(self.dist_dir, arc_name)
-        self.arc_filename = self.make_archive(arc_base, self.format,
-                                              self.build_dir)
-
-        # Sign if requested
-        if self.sign:
-            gpg_args = ["gpg", "--detach-sign", "-a", self.arc_filename]
-            if self.identity:
-                gpg_args[2:2] = ["--local-user", self.identity]
-            spawn(gpg_args,
-                  dry_run=self.dry_run)
-
-        # cleanup
-        if not self.keep_temp:
-            dir_util.remove_tree(tmp_dir, dry_run=self.dry_run)
-
-#=============================================================================
-# eof
-#=============================================================================

src/passlib/_setup/stamp.py

@@ -1,57 +0,0 @@
-"""update version string during build"""
-#=============================================================================
-# imports
-#=============================================================================
-from __future__ import with_statement
-# core
-import os
-import re
-import time
-from distutils.dist import Distribution
-# pkg
-# local
-__all__ = [
-    "stamp_source",
-    "stamp_distutils_output",
-]
-#=============================================================================
-# helpers
-#=============================================================================
-def get_command_class(opts, name):
-    return opts['cmdclass'].get(name) or Distribution().get_command_class(name)
-
-def stamp_source(base_dir, version, dry_run=False):
-    """update version string in passlib dist"""
-    path = os.path.join(base_dir, "passlib", "__init__.py")
-    with open(path) as fh:
-        input = fh.read()
-    output, count = re.subn('(?m)^__version__\s*=.*$',
-                    '__version__ = ' + repr(version),
-                    input)
-    assert count == 1, "failed to replace version string"
-    if not dry_run:
-        os.unlink(path) # sdist likes to use hardlinks
-        with open(path, "w") as fh:
-            fh.write(output)
-
-def stamp_distutils_output(opts, version):
-
-    # subclass buildpy to update version string in source
-    _build_py = get_command_class(opts, "build_py")
-    class build_py(_build_py):
-        def build_packages(self):
-            _build_py.build_packages(self)
-            stamp_source(self.build_lib, version, self.dry_run)
-    opts['cmdclass']['build_py'] = build_py
-
-    # subclass sdist to do same thing
-    _sdist = get_command_class(opts, "sdist")
-    class sdist(_sdist):
-        def make_release_tree(self, base_dir, files):
-            _sdist.make_release_tree(self, base_dir, files)
-            stamp_source(base_dir, version, self.dry_run)
-    opts['cmdclass']['sdist'] = sdist
-
-#=============================================================================
-# eof
-#=============================================================================