deepblue/GoogleDriveManagement
1
0
Fork 0
mirror of https://github.com/GAM-team/GAM.git synced 2026-07-05 05:11:35 +00:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

Begin breaking apart gam.py into logical pieces (#1047)

Browse Source 
* Begin breaking apart gam.py into logical pieces

Start with one of the deepest parts of the stack, Google API request execution calls and associated errors. Critical information printing functions and application control logic are also broken out into their own components.

This change also adds unit tests for migrated content and makes code more PEP8 compliant.

This commit starts work on  jay0lee/GAM#147

* Add unit tests to Travis config

* Swap assert_called_once() with assertEqual() and Mock.call_count

Makes tests compatible with Python 3.5. assert_called_once() is only available in Python 3.6+
This commit is contained in:
ejochman
2019-12-07 06:50:56 -08:00
committed by Jay Lee
parent 4cc1a97a0a
commit a0c410be0e
11 changed files with 2042 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@@ -198,6 +198,8 @@ install:
- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
script:
# Discover and run all Python unit tests
- $python -m unittest discover -s ./ -p "*_test.py"
- $gam version extended
- $gam version | grep travis # travis should be part of the path (not /tmp or such)
# determine which Python version GAM is built with and ensure it's at least build version from above.

66
src/controlflow.py Normal file
View File

@@ -0,0 +1,66 @@
"""Methods related to the central control flow of an application."""
import random
import sys
import time
import display # TODO: Change to relative import when gam is setup as a package
from var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
from var import MESSAGE_INVALID_JSON
def system_error_exit(return_code, message):
"""Raises a system exit with the given return code and message.
Args:
return_code: Int, the return code to yield when the system exits.
message: An error message to print before the system exits.
"""
if message:
display.print_error(message)
sys.exit(return_code)
def csv_field_error_exit(field_name, field_names):
"""Raises a system exit when a CSV field is malformed.
Args:
field_name: The CSV field name for which a header does not exist in the
existing CSV headers.
field_names: The known list of CSV headers.
"""
system_error_exit(
2,
MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
','.join(field_names)))
def invalid_json_exit(file_name):
"""Raises a sysyem exit when invalid JSON content is encountered."""
system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
def wait_on_failure(current_attempt_num,
total_num_retries,
error_message,
error_print_threshold=3):
"""Executes an exponential backoff-style system sleep.
Args:
current_attempt_num: Int, the current number of retries.
total_num_retries: Int, the total number of times the current action will be
retried.
error_message: String, a message to be displayed that will give more context
around why the action is being retried.
error_print_threshold: Int, the number of attempts which will have their
error messages suppressed. Any current_attempt_num greater than
error_print_threshold will print the prescribed error.
"""
wait_on_fail = min(2**current_attempt_num,
60) + float(random.randint(1, 1000)) / 1000
if current_attempt_num > error_print_threshold:
sys.stderr.write(
'Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'
.format(error_message, int(wait_on_fail), current_attempt_num,
total_num_retries))
sys.stderr.flush()
time.sleep(wait_on_fail)

108
src/controlflow_test.py Normal file
View File

@@ -0,0 +1,108 @@
"""Tests for controlflow."""
import unittest
from unittest.mock import patch
import controlflow
class ControlFlowTest(unittest.TestCase):
def test_system_error_exit_raises_systemexit_error(self):
with self.assertRaises(SystemExit):
controlflow.system_error_exit(1, 'exit message')
def test_system_error_exit_raises_systemexit_with_return_code(self):
with self.assertRaises(SystemExit) as context_manager:
controlflow.system_error_exit(100, 'exit message')
self.assertEqual(context_manager.exception.code, 100)
@patch.object(controlflow.display, 'print_error')
def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
with self.assertRaises(SystemExit):
controlflow.system_error_exit(100, 'exit message')
self.assertIn('exit message', mock_print_err.call_args[0][0])
def test_csv_field_error_exit_raises_systemexit_error(self):
with self.assertRaises(SystemExit):
controlflow.csv_field_error_exit('aField',
['unusedField1', 'unusedField2'])
def test_csv_field_error_exit_exits_code_2(self):
with self.assertRaises(SystemExit) as context_manager:
controlflow.csv_field_error_exit('aField',
['unusedField1', 'unusedField2'])
self.assertEqual(context_manager.exception.code, 2)
@patch.object(controlflow.display, 'print_error')
def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
with self.assertRaises(SystemExit):
controlflow.csv_field_error_exit('aField',
['unusedField1', 'unusedField2'])
printed_message = mock_print_err.call_args[0][0]
self.assertIn('aField', printed_message)
self.assertIn('unusedField1', printed_message)
self.assertIn('unusedField2', printed_message)
def test_invalid_json_exit_raises_systemexit_error(self):
with self.assertRaises(SystemExit):
controlflow.invalid_json_exit('filename')
def test_invalid_json_exit_exit_exits_code_17(self):
with self.assertRaises(SystemExit) as context_manager:
controlflow.invalid_json_exit('filename')
self.assertEqual(context_manager.exception.code, 17)
@patch.object(controlflow.display, 'print_error')
def test_invalid_json_exit_prints_error_details(self, mock_print_err):
with self.assertRaises(SystemExit):
controlflow.invalid_json_exit('filename')
printed_message = mock_print_err.call_args[0][0]
self.assertIn('filename', printed_message)
@patch.object(controlflow.time, 'sleep')
def test_wait_on_failure_waits_exponentially(self, mock_sleep):
controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
sleep_calls = mock_sleep.call_args_list
self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
@patch.object(controlflow.time, 'sleep')
def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
total_attempts = 20
for attempt in range(1, total_attempts + 1):
controlflow.wait_on_failure(
attempt,
total_attempts,
'Attempt #%s' % attempt,
# Suppress messages while we make a lot of attempts.
error_print_threshold=total_attempts + 1)
# Wait time may be between 60 and 61 secs, due to rand addition.
self.assertLess(mock_sleep.call_args[0][0], 61)
# Prevent the system from actually sleeping and thus slowing down the test.
@patch.object(controlflow.time, 'sleep')
@patch.object(controlflow.sys.stderr, 'write')
def test_wait_on_failure_prints_errors(self, mock_stderr_write,
unused_mock_sleep):
message = 'An error message to display'
controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
self.assertIn(message, mock_stderr_write.call_args[0][0])
@patch.object(controlflow.time, 'sleep')
@patch.object(controlflow.sys.stderr, 'write')
def test_wait_on_failure_only_prints_after_threshold(self, mock_stderr_write,
unused_mock_sleep):
total_attempts = 5
threshold = 3
for attempt in range(1, total_attempts + 1):
controlflow.wait_on_failure(
attempt,
total_attempts,
'Attempt #%s' % attempt,
error_print_threshold=threshold)
self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

18
src/display.py Normal file
View File

@@ -0,0 +1,18 @@
"""Methods related to display of information to the user."""
import sys
import utils
from var import ERROR_PREFIX
from var import WARNING_PREFIX
def print_error(message):
"""Prints a one-line error message to stderr in a standard format."""
sys.stderr.write(
utils.convertUTF8('\n{0}{1}\n'.format(ERROR_PREFIX, message)))
def print_warning(message):
"""Prints a one-line warning message to stderr in a standard format."""
sys.stderr.write(
utils.convertUTF8('\n{0}{1}\n'.format(WARNING_PREFIX, message)))

59
src/display_test.py Normal file
View File

@@ -0,0 +1,59 @@
"""Tests for display."""
import unittest
from unittest.mock import patch
import display
from var import ERROR_PREFIX
from var import WARNING_PREFIX
class DisplayTest(unittest.TestCase):
@patch.object(display.sys.stderr, 'write')
def test_print_error_prints_to_stderr(self, mock_write):
message = 'test error'
display.print_error(message)
printed_message = mock_write.call_args[0][0]
self.assertIn(message, printed_message)
@patch.object(display.sys.stderr, 'write')
def test_print_error_prints_error_prefix(self, mock_write):
message = 'test error'
display.print_error(message)
printed_message = mock_write.call_args[0][0]
self.assertLess(
printed_message.find(ERROR_PREFIX), printed_message.find(message),
'The error prefix does not appear before the error message')
@patch.object(display.sys.stderr, 'write')
def test_print_error_ends_message_with_newline(self, mock_write):
message = 'test error'
display.print_error(message)
printed_message = mock_write.call_args[0][0]
self.assertRegex(printed_message, '\n$',
'The error message does not end in a newline.')
@patch.object(display.sys.stderr, 'write')
def test_print_warning_prints_to_stderr(self, mock_write):
message = 'test warning'
display.print_warning(message)
printed_message = mock_write.call_args[0][0]
self.assertIn(message, printed_message)
@patch.object(display.sys.stderr, 'write')
def test_print_warning_prints_error_prefix(self, mock_write):
message = 'test warning'
display.print_error(message)
printed_message = mock_write.call_args[0][0]
self.assertLess(
printed_message.find(WARNING_PREFIX), printed_message.find(message),
'The warning prefix does not appear before the error message')
@patch.object(display.sys.stderr, 'write')
def test_print_warning_ends_message_with_newline(self, mock_write):
message = 'test warning'
display.print_error(message)
printed_message = mock_write.call_args[0][0]
self.assertRegex(printed_message, '\n$',
'The warning message does not end in a newline.')

1920
src/gam.py
View File

File diff suppressed because it is too large Load Diff

168
src/gapi/__init__.py Normal file
View File

@@ -0,0 +1,168 @@
"""Methods related to execution of GAPI requests."""
import controlflow
import display
from gapi import errors
import googleapiclient.errors
import httplib2
from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION,
GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
MESSAGE_API_ACCESS_CONFIG, MESSAGE_API_ACCESS_DENIED,
MESSAGE_SERVICE_NOT_APPLICABLE)
import google.auth.exceptions
def create_http(cache=None,
timeout=None,
override_min_tls=None,
override_max_tls=None):
"""Creates a uniform HTTP transport object.
Args:
cache: The HTTP cache to use.
timeout: The cache timeout, in seconds.
override_min_tls: The minimum TLS version to require. If not provided, the
default is used.
override_max_tls: The maximum TLS version to require. If not provided, the
default is used.
Returns:
httplib2.Http with the specified options.
"""
tls_minimum_version = override_min_tls if override_min_tls else GC_Values[
GC_TLS_MIN_VERSION]
tls_maximum_version = override_max_tls if override_max_tls else GC_Values[
GC_TLS_MAX_VERSION]
return httplib2.Http(
ca_certs=GC_Values[GC_CA_FILE],
tls_maximum_version=tls_maximum_version,
tls_minimum_version=tls_minimum_version,
cache=cache,
timeout=timeout)
def call(service,
function,
silent_errors=False,
soft_errors=False,
throw_reasons=None,
retry_reasons=None,
**kwargs):
"""Executes a single request on a Google service function.
Args:
service: A Google service object for the desired API.
function: String, The name of a service request method to execute.
silent_errors: Bool, If True, error messages are suppressed when
encountered.
soft_errors: Bool, If True, writes non-fatal errors to stderr.
throw_reasons: A list of Google HTTP error reason strings indicating the
errors generated by this request should be re-thrown. All other HTTP
errors are consumed.
retry_reasons: A list of Google HTTP error reason strings indicating which
error should be retried, using exponential backoff techniques, when the
error reason is encountered.
**kwargs: Additional params to pass to the request method.
Returns:
A response object for the corresponding Google API call.
"""
if throw_reasons is None:
throw_reasons = []
if retry_reasons is None:
retry_reasons = []
method = getattr(service, function)
retries = 10
parameters = dict(
list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
for n in range(1, retries + 1):
try:
return method(**parameters).execute()
except googleapiclient.errors.HttpError as e:
http_status, reason, message = errors.get_gapi_error_detail(
e,
soft_errors=soft_errors,
silent_errors=silent_errors,
retry_on_http_error=n < 3)
if http_status == -1:
# The error detail indicated that we should retry this request
# We'll refresh credentials and make another pass
service._http.request.credentials.refresh(create_http())
continue
if http_status == 0:
return None
is_known_error_reason = reason in [r.value for r in errors.ErrorReason]
if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
message)
else:
raise e
if (n != retries) and (is_known_error_reason and errors.ErrorReason(
reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
controlflow.wait_on_failure(n, retries, reason)
continue
if soft_errors:
display.print_error('{0}: {1} - {2}{3}'.format(http_status, message,
reason,
['',
': Giving up.'][n > 1]))
return None
controlflow.system_error_exit(
int(http_status), '{0}: {1} - {2}'.format(http_status, message,
reason))
except google.auth.exceptions.RefreshError as e:
handle_oauth_token_error(
e, soft_errors or
errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
raise errors.GapiServiceNotAvailableError(str(e))
display.print_error('User {0}: {1}'.format(
GM_Globals[GM_CURRENT_API_USER], str(e)))
return None
except ValueError as e:
if service._http.cache is not None:
service._http.cache = None
continue
controlflow.system_error_exit(4, str(e))
except (httplib2.ServerNotFoundError, RuntimeError) as e:
if n != retries:
service._http.connections = {}
controlflow.wait_on_failure(n, retries, str(e))
continue
controlflow.system_error_exit(4, str(e))
except TypeError as e:
controlflow.system_error_exit(4, str(e))
# TODO: Make this private once all execution related items that use this method
# have been brought into this file
def handle_oauth_token_error(e, soft_errors):
"""On a token error, exits the application and writes a message to stderr.
Args:
e: google.auth.exceptions.RefreshError, The error to handle.
soft_errors: Boolean, if True, suppresses any applicable errors and instead
returns to the caller.
"""
token_error = str(e).replace('.', '')
if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
'Invalid response'):
if soft_errors:
return
if not GM_Globals[GM_CURRENT_API_USER]:
display.print_error(
MESSAGE_API_ACCESS_DENIED.format(
GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
else:
controlflow.system_error_exit(
19,
MESSAGE_SERVICE_NOT_APPLICABLE.format(
GM_Globals[GM_CURRENT_API_USER]))
controlflow.system_error_exit(18,
'Authentication Token Error - {0}'.format(e))

241
src/gapi/__init___test.py Normal file
View File

@@ -0,0 +1,241 @@
"""Tests for gapi."""
import json
import unittest
from unittest.mock import MagicMock
from unittest.mock import patch
from gam import SetGlobalVariables
import gapi
from gapi import errors
def create_http_error(status, reason, message):
"""Creates a HttpError object similar to most Google API Errors.
Args:
status: Int, the error's HTTP response status number.
reason: String, a camelCase reason for the HttpError being given.
message: String, a general error message describing the error that occurred.
Returns:
googleapiclient.errors.HttpError
"""
response = {
'status': status,
'content-type': 'application/json',
}
content = {
'error': {
'code': status,
'errors': [{
'reason': str(reason),
'message': message,
}]
}
}
content_bytes = json.dumps(content).encode('UTF-8')
return gapi.googleapiclient.errors.HttpError(response, content_bytes)
class CreateHttpTest(unittest.TestCase):
def setUp(self):
SetGlobalVariables()
super(CreateHttpTest, self).setUp()
def test_create_http_sets_default_values_on_http(self):
http = gapi.create_http()
self.assertIsNone(http.cache)
self.assertIsNone(http.timeout)
self.assertEqual(http.tls_minimum_version,
gapi.GC_Values[gapi.GC_TLS_MIN_VERSION])
self.assertEqual(http.tls_maximum_version,
gapi.GC_Values[gapi.GC_TLS_MAX_VERSION])
self.assertEqual(http.ca_certs, gapi.GC_Values[gapi.GC_CA_FILE])
def test_create_http_sets_tls_min_version(self):
http = gapi.create_http(override_min_tls=1111)
self.assertEqual(http.tls_minimum_version, 1111)
def test_create_http_sets_tls_max_version(self):
http = gapi.create_http(override_max_tls=9999)
self.assertEqual(http.tls_maximum_version, 9999)
def test_create_http_sets_cache(self):
fake_cache = {}
http = gapi.create_http(cache=fake_cache)
self.assertEqual(http.cache, fake_cache)
def test_create_http_sets_cache_timeout(self):
http = gapi.create_http(timeout=1234)
self.assertEqual(http.timeout, 1234)
class CallTest(unittest.TestCase):
def setUp(self):
SetGlobalVariables()
self.mock_service = MagicMock()
self.mock_method_name = 'mock_method'
self.mock_method = getattr(self.mock_service, self.mock_method_name)
super(CallTest, self).setUp()
def test_call_returns_basic_200_response(self):
response = gapi.call(self.mock_service, self.mock_method_name)
self.assertEqual(response, self.mock_method().execute.return_value)
def test_call_passes_target_method_params(self):
gapi.call(
self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
self.assertEqual(self.mock_method.call_count, 1)
method_kwargs = self.mock_method.call_args[1]
self.assertEqual(method_kwargs.get('my_param_1'), 1)
self.assertEqual(method_kwargs.get('my_param_2'), 2)
@patch.object(gapi.errors, 'get_gapi_error_detail')
def test_call_retries_with_soft_errors(self, mock_error_detail):
mock_error_detail.return_value = (-1, 'aReason', 'some message')
# Make the request fail first, then return the proper response on the retry.
fake_http_error = create_http_error(403, 'aReason', 'unused message')
fake_200_response = MagicMock()
self.mock_method.return_value.execute.side_effect = [
fake_http_error, fake_200_response
]
response = gapi.call(
self.mock_service, self.mock_method_name, soft_errors=True)
self.assertEqual(response, fake_200_response)
self.assertEqual(
self.mock_service._http.request.credentials.refresh.call_count, 1)
self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
def test_call_throws_for_provided_reason(self):
throw_reason = errors.ErrorReason.USER_NOT_FOUND
fake_http_error = create_http_error(404, throw_reason, 'forced throw')
self.mock_method.return_value.execute.side_effect = fake_http_error
gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
with self.assertRaises(gam_exception):
gapi.call(
self.mock_service,
self.mock_method_name,
throw_reasons=[throw_reason])
# Prevent wait_on_failure from performing actual backoff unnecessarily, since
# we're not actually testing over a network connection
@patch.object(gapi.controlflow, 'wait_on_failure')
def test_call_retries_request_for_default_retry_reasons(
self, mock_wait_on_failure):
# Test using one of the default retry reasons
default_throw_reason = errors.ErrorReason.BACKEND_ERROR
self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
fake_http_error = create_http_error(404, default_throw_reason, 'message')
fake_200_response = MagicMock()
# Fail once, then succeed on retry
self.mock_method.return_value.execute.side_effect = [
fake_http_error, fake_200_response
]
response = gapi.call(
self.mock_service, self.mock_method_name, retry_reasons=[])
self.assertEqual(response, fake_200_response)
self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
# Make sure a backoff technique was used for retry.
self.assertEqual(mock_wait_on_failure.call_count, 1)
# Prevent wait_on_failure from performing actual backoff unnecessarily, since
# we're not actually testing over a network connection
@patch.object(gapi.controlflow, 'wait_on_failure')
def test_call_retries_requests_for_provided_retry_reasons(
self, unused_mock_wait_on_failure):
retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
fake_retrieable_error1 = create_http_error(400, retry_reason1,
'Forced Error 1')
retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
fake_retrieable_error2 = create_http_error(400, retry_reason2,
'Forced Error 2')
non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
fake_non_retriable_error = create_http_error(
400, non_retriable_reason,
'This error should not cause the request to be retried')
# Fail once, then succeed on retry
self.mock_method.return_value.execute.side_effect = [
fake_retrieable_error1, fake_retrieable_error2, fake_non_retriable_error
]
with self.assertRaises(SystemExit):
# The third call should raise the SystemExit when non_retriable_error is
# raised.
gapi.call(
self.mock_service,
self.mock_method_name,
retry_reasons=[retry_reason1, retry_reason2])
self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
def test_call_exits_on_oauth_token_error(self):
# An error with any OAUTH2_TOKEN_ERROR
fake_token_error = gapi.google.auth.exceptions.RefreshError(
errors.OAUTH2_TOKEN_ERRORS[0])
self.mock_method.return_value.execute.side_effect = fake_token_error
with self.assertRaises(SystemExit):
gapi.call(self.mock_service, self.mock_method_name)
def test_call_exits_on_nonretriable_error(self):
error_reason = 'unknownReason'
fake_http_error = create_http_error(500, error_reason,
'Testing unretriable errors')
self.mock_method.return_value.execute.side_effect = fake_http_error
with self.assertRaises(SystemExit):
gapi.call(self.mock_service, self.mock_method_name)
def test_call_exits_on_request_valueerror(self):
self.mock_method.return_value.execute.side_effect = ValueError()
with self.assertRaises(SystemExit):
gapi.call(self.mock_service, self.mock_method_name)
def test_call_clears_bad_http_cache_on_request_failure(self):
self.mock_service._http.cache = 'something that is not None'
fake_200_response = MagicMock()
self.mock_method.return_value.execute.side_effect = [
ValueError(), fake_200_response
]
self.assertIsNotNone(self.mock_service._http.cache)
response = gapi.call(self.mock_service, self.mock_method_name)
self.assertEqual(response, fake_200_response)
# Assert the cache was cleared
self.assertIsNone(self.mock_service._http.cache)
# Prevent wait_on_failure from performing actual backoff unnecessarily, since
# we're not actually testing over a network connection
@patch.object(gapi.controlflow, 'wait_on_failure')
def test_call_retries_requests_with_backoff_on_servernotfounderror(
self, mock_wait_on_failure):
fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
fake_200_response = MagicMock()
# Fail once, then succeed on retry
self.mock_method.return_value.execute.side_effect = [
fake_servernotfounderror, fake_200_response
]
http_connections = self.mock_service._http.connections
response = gapi.call(self.mock_service, self.mock_method_name)
self.assertEqual(response, fake_200_response)
# HTTP cached connections should be cleared on receiving this error
self.assertNotEqual(http_connections, self.mock_service._http.connections)
self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
# Make sure a backoff technique was used for retry.
self.assertEqual(mock_wait_on_failure.call_count, 1)
if __name__ == '__main__':
unittest.main()

340
src/gapi/errors.py Normal file
View File

@@ -0,0 +1,340 @@
"""GAPI and OAuth Token related errors methods."""
import json
import controlflow
from enum import Enum
import googleapiclient.errors
from var import UTF8
import display # TODO: Change to relative import when gam is setup as a package
class GapiAbortedError(Exception):
pass
class GapiAuthErrorError(Exception):
pass
class GapiBadRequestError(Exception):
pass
class GapiConditionNotMetError(Exception):
pass
class GapiCyclicMembershipsNotAllowedError(Exception):
pass
class GapiDomainCannotUseApisError(Exception):
pass
class GapiDomainNotFoundError(Exception):
pass
class GapiDuplicateError(Exception):
pass
class GapiFailedPreconditionError(Exception):
pass
class GapiForbiddenError(Exception):
pass
class GapiGroupNotFoundError(Exception):
pass
class GapiInvalidError(Exception):
pass
class GapiInvalidArgumentError(Exception):
pass
class GapiInvalidMemberError(Exception):
pass
class GapiMemberNotFoundError(Exception):
pass
class GapiNotFoundError(Exception):
pass
class GapiNotImplementedError(Exception):
pass
class GapiPermissionDeniedError(Exception):
pass
class GapiResourceNotFoundError(Exception):
pass
class GapiServiceNotAvailableError(Exception):
pass
class GapiUserNotFoundError(Exception):
pass
# GAPI Error Reasons
class ErrorReason(Enum):
"""The reason why a non-200 HTTP response was returned from a GAPI."""
ABORTED = 'aborted'
AUTH_ERROR = 'authError'
BACKEND_ERROR = 'backendError'
BAD_REQUEST = 'badRequest'
CONDITION_NOT_MET = 'conditionNotMet'
CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
DOMAIN_NOT_FOUND = 'domainNotFound'
DUPLICATE = 'duplicate'
FAILED_PRECONDITION = 'failedPrecondition'
FORBIDDEN = 'forbidden'
GROUP_NOT_FOUND = 'groupNotFound'
INTERNAL_ERROR = 'internalError'
INVALID = 'invalid'
INVALID_ARGUMENT = 'invalidArgument'
INVALID_MEMBER = 'invalidMember'
MEMBER_NOT_FOUND = 'memberNotFound'
NOT_FOUND = 'notFound'
NOT_IMPLEMENTED = 'notImplemented'
PERMISSION_DENIED = 'permissionDenied'
QUOTA_EXCEEDED = 'quotaExceeded'
RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
RESOURCE_NOT_FOUND = 'resourceNotFound'
SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
SYSTEM_ERROR = 'systemError'
USER_NOT_FOUND = 'userNotFound'
USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
def __str__(self):
return str(self.value)
# Common sets of GAPI error reasons
DEFAULT_RETRY_REASONS = [
ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
ErrorReason.INTERNAL_ERROR
]
GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
GROUP_GET_THROW_REASONS = [
ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
ErrorReason.BAD_REQUEST
]
GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
MEMBERS_THROW_REASONS = [
ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
ErrorReason.FORBIDDEN
]
MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
# A map of GAPI error reasons to the corresponding GAM Python Exception
ERROR_REASON_TO_EXCEPTION = {
ErrorReason.ABORTED:
GapiAbortedError,
ErrorReason.AUTH_ERROR:
GapiAuthErrorError,
ErrorReason.BAD_REQUEST:
GapiBadRequestError,
ErrorReason.CONDITION_NOT_MET:
GapiConditionNotMetError,
ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
GapiCyclicMembershipsNotAllowedError,
ErrorReason.DOMAIN_CANNOT_USE_APIS:
GapiDomainCannotUseApisError,
ErrorReason.DOMAIN_NOT_FOUND:
GapiDomainNotFoundError,
ErrorReason.DUPLICATE:
GapiDuplicateError,
ErrorReason.FAILED_PRECONDITION:
GapiFailedPreconditionError,
ErrorReason.FORBIDDEN:
GapiForbiddenError,
ErrorReason.GROUP_NOT_FOUND:
GapiGroupNotFoundError,
ErrorReason.INVALID:
GapiInvalidError,
ErrorReason.INVALID_ARGUMENT:
GapiInvalidArgumentError,
ErrorReason.INVALID_MEMBER:
GapiInvalidMemberError,
ErrorReason.MEMBER_NOT_FOUND:
GapiMemberNotFoundError,
ErrorReason.NOT_FOUND:
GapiNotFoundError,
ErrorReason.NOT_IMPLEMENTED:
GapiNotImplementedError,
ErrorReason.PERMISSION_DENIED:
GapiPermissionDeniedError,
ErrorReason.RESOURCE_NOT_FOUND:
GapiResourceNotFoundError,
ErrorReason.SERVICE_NOT_AVAILABLE:
GapiServiceNotAvailableError,
ErrorReason.USER_NOT_FOUND:
GapiUserNotFoundError,
}
# OAuth Token Errors
OAUTH2_TOKEN_ERRORS = [
'access_denied',
'access_denied: Requested client not authorized',
'internal_failure: Backend Error',
'internal_failure: None',
'invalid_grant',
'invalid_grant: Bad Request',
'invalid_grant: Invalid email or User ID',
'invalid_grant: Not a valid email',
'invalid_grant: Invalid JWT: No valid verifier found for issuer',
'invalid_request: Invalid impersonation prn email address',
'unauthorized_client: Client is unauthorized to retrieve access tokens '
'using this method',
'unauthorized_client: Client is unauthorized to retrieve access tokens '
'using this method, or client not authorized for any of the scopes '
'requested',
'unauthorized_client: Unauthorized client or scope in request',
]
def _create_http_error_dict(status_code, reason, message):
"""Creates a basic error dict similar to most Google API Errors.
Args:
status_code: Int, the error's HTTP response status code.
reason: String, a camelCase reason for the HttpError being given.
message: String, a general error message describing the error that occurred.
Returns:
dict
"""
return {
'error': {
'code': status_code,
'errors': [{
'reason': str(reason),
'message': message,
}]
}
}
def get_gapi_error_detail(e,
soft_errors=False,
silent_errors=False,
retry_on_http_error=False):
"""Extracts error detail from a non-200 GAPI Response.
Args:
e: googleapiclient.HttpError, The HTTP Error received.
soft_errors: Boolean, If true, causes error messages to be surpressed,
rather than sending them to stderr.
silent_errors: Boolean, If true, suppresses and ignores any errors from
being displayed
retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
code, indicating that the request can be retried. TODO: Remove this param,
as it seems to be outside the scope of this method.
Returns:
A tuple containing the HTTP Response code, GAPI error reason, and error
message.
"""
try:
error = json.loads(e.content.decode(UTF8))
except ValueError:
error_content = e.content.decode(UTF8) if isinstance(e.content,
bytes) else e.content
if (e.resp['status'] == '503') and (
error_content == 'Quota exceeded for the current request'):
return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
if (e.resp['status'] == '403') and (
error_content.startswith('Request rate higher than configured')):
return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
'Domain not found')
elif (e.resp['status'] == '400') and (
'InvalidSsoSigningKey' in error_content):
error = _create_http_error_dict(400, ErrorReason.INVALID.value,
'InvalidSsoSigningKey')
elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
error = _create_http_error_dict(400, ErrorReason.INVALID.value,
'UnknownError')
elif retry_on_http_error:
return (-1, None, None)
elif soft_errors:
if not silent_errors:
display.print_error(error_content)
return (0, None, None)
else:
controlflow.system_error_exit(5, error_content)
# END: ValueError catch
if 'error' in error:
http_status = error['error']['code']
try:
message = error['error']['errors'][0]['message']
except KeyError:
message = error['error']['message']
else:
if 'error_description' in error:
if error['error_description'] == 'Invalid Value':
message = error['error_description']
http_status = 400
error = _create_http_error_dict(400, ErrorReason.INVALID.value, message)
else:
controlflow.system_error_exit(4, str(error))
else:
controlflow.system_error_exit(4, str(error))
# Extract the error reason
try:
reason = error['error']['errors'][0]['reason']
if reason == 'notFound':
if 'userKey' in message:
reason = ErrorReason.USER_NOT_FOUND.value
elif 'groupKey' in message:
reason = ErrorReason.GROUP_NOT_FOUND.value
elif 'memberKey' in message:
reason = ErrorReason.MEMBER_NOT_FOUND.value
elif 'Domain not found' in message:
reason = ErrorReason.DOMAIN_NOT_FOUND.value
elif 'Resource Not Found' in message:
reason = ErrorReason.RESOURCE_NOT_FOUND.value
elif reason == 'invalid':
if 'userId' in message:
reason = ErrorReason.USER_NOT_FOUND.value
elif 'memberKey' in message:
reason = ErrorReason.INVALID_MEMBER.value
elif reason == 'failedPrecondition':
if 'Bad Request' in message:
reason = ErrorReason.BAD_REQUEST.value
elif 'Mail service not enabled' in message:
reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
elif reason == 'required':
if 'memberKey' in message:
reason = ErrorReason.MEMBER_NOT_FOUND.value
elif reason == 'conditionNotMet':
if 'Cyclic memberships not allowed' in message:
reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
except KeyError:
reason = '{0}'.format(http_status)
return (http_status, reason, message)

209
src/gapi/errors_test.py Normal file
View File

@@ -0,0 +1,209 @@
"""Python unit tests for gapi.errors"""
import json
import unittest
from unittest.mock import patch
import googleapiclient.errors
from gapi import errors
def create_simple_http_error(status, reason, message):
content = errors._create_http_error_dict(status, reason, message)
return create_http_error(status, content)
def create_http_error(status, content):
response = {
'status': status,
'content-type': 'application/json',
}
content_as_bytes = json.dumps(content).encode('UTF-8')
return googleapiclient.errors.HttpError(response, content_as_bytes)
class ErrorsTest(unittest.TestCase):
def test_get_gapi_error_detail_quota_exceeded(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_detail_invalid_domain(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_detail_invalid_signing_key(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_detail_unknown_error(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_retry_http_error(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_prints_soft_errors(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_exits_on_unrecoverable_errors(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_quota_exceeded_for_current_request(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_quota_exceeded_high_request_rate(self):
# TODO: Add test logic once the opening ValueError exception case has a
# repro case (i.e. an Exception type/format that will cause it to raise).
pass
def test_get_gapi_error_extracts_user_not_found(self):
err = create_simple_http_error(404, 'notFound',
'Resource Not Found: userKey.')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 404)
self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
self.assertEqual(message, 'Resource Not Found: userKey.')
def test_get_gapi_error_extracts_group_not_found(self):
err = create_simple_http_error(404, 'notFound',
'Resource Not Found: groupKey.')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 404)
self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
self.assertEqual(message, 'Resource Not Found: groupKey.')
def test_get_gapi_error_extracts_member_not_found(self):
err = create_simple_http_error(404, 'notFound',
'Resource Not Found: memberKey.')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 404)
self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
self.assertEqual(message, 'Resource Not Found: memberKey.')
def test_get_gapi_error_extracts_domain_not_found(self):
err = create_simple_http_error(404, 'notFound', 'Domain not found.')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 404)
self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
self.assertEqual(message, 'Domain not found.')
def test_get_gapi_error_extracts_generic_resource_not_found(self):
err = create_simple_http_error(404, 'notFound',
'Resource Not Found: unknownResource.')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 404)
self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
self.assertEqual(message, 'Resource Not Found: unknownResource.')
def test_get_gapi_error_extracts_invalid_userid(self):
err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 400)
self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
self.assertEqual(message, 'Invalid Input: userId')
def test_get_gapi_error_extracts_invalid_member(self):
err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 400)
self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
self.assertEqual(message, 'Invalid Input: memberKey')
def test_get_gapi_error_extracts_bad_request(self):
err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 400)
self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
self.assertEqual(message, 'Bad Request')
def test_get_gapi_error_extracts_service_not_available(self):
err = create_simple_http_error(400, 'failedPrecondition',
'Mail service not enabled')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 400)
self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
self.assertEqual(message, 'Mail service not enabled')
def test_get_gapi_error_extracts_required_member_not_found(self):
err = create_simple_http_error(400, 'required',
'Missing required field: memberKey')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 400)
self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
self.assertEqual(message, 'Missing required field: memberKey')
def test_get_gapi_error_extracts_cyclic_memberships_error(self):
err = create_simple_http_error(400, 'conditionNotMet',
'Cyclic memberships not allowed')
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, 400)
self.assertEqual(reason,
errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
self.assertEqual(message, 'Cyclic memberships not allowed')
def test_get_gapi_error_extracts_single_error_with_message(self):
status_code = 999
response = {'status': status_code}
# This error does not have an "errors" key describing each error.
content = {'error': {'code': status_code, 'message': 'unknown error'}}
content_as_bytes = json.dumps(content).encode('UTF-8')
err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, status_code)
self.assertEqual(reason, str(status_code))
self.assertEqual(message, content['error']['message'])
def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
self):
status_code = 999
response = {'status': status_code}
# This error only has an error_description_field and an unknown description.
content = {'error_description': 'something errored'}
content_as_bytes = json.dumps(content).encode('UTF-8')
err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)
with self.assertRaises(SystemExit) as context:
errors.get_gapi_error_detail(err)
self.assertEqual(4, context.exception.code)
def test_get_gapi_error_exits_on_invalid_error_description(self):
status_code = 400
response = {'status': status_code}
content = {'error_description': 'Invalid Value'}
content_as_bytes = json.dumps(content).encode('UTF-8')
err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)
http_status, reason, message = errors.get_gapi_error_detail(err)
self.assertEqual(http_status, status_code)
self.assertEqual(reason, errors.ErrorReason.INVALID.value)
self.assertEqual(message, 'Invalid Value')
def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
status_code = 900
response = {'status': status_code}
content = {'notErrorContentThatIsExpected': 'foo'}
content_as_bytes = json.dumps(content).encode('UTF-8')
err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)
with self.assertRaises(SystemExit) as context:
errors.get_gapi_error_detail(err)
self.assertEqual(4, context.exception.code)
if __name__ == '__main__':
unittest.main()

52
src/var.py
View File

@@ -951,58 +951,6 @@ MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}.
MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON = 'Please run\n\ngam create project\ngam user <user> check serviceaccount\n\nto create and configure a service account.'
MESSAGE_UPDATE_GAM_TO_64BIT = "You're running a 32-bit version of GAM on a 64-bit version of Windows, upgrade to a windows-x86_64 version of GAM"
MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from Google by %s'
# oauth errors
OAUTH2_TOKEN_ERRORS = [
'access_denied',
'access_denied: Requested client not authorized',
'internal_failure: Backend Error',
'internal_failure: None',
'invalid_grant',
'invalid_grant: Bad Request',
'invalid_grant: Invalid email or User ID',
'invalid_grant: Not a valid email',
'invalid_grant: Invalid JWT: No valid verifier found for issuer',
'invalid_request: Invalid impersonation prn email address',
'unauthorized_client: Client is unauthorized to retrieve access tokens using this method',
'unauthorized_client: Client is unauthorized to retrieve access tokens using this method, or client not authorized for any of the scopes requested',
'unauthorized_client: Unauthorized client or scope in request',
]
#
# callGAPI throw reasons
GAPI_ABORTED = 'aborted'
GAPI_AUTH_ERROR = 'authError'
GAPI_BACKEND_ERROR = 'backendError'
GAPI_BAD_REQUEST = 'badRequest'
GAPI_CONDITION_NOT_MET = 'conditionNotMet'
GAPI_CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
GAPI_DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
GAPI_DOMAIN_NOT_FOUND = 'domainNotFound'
GAPI_DUPLICATE = 'duplicate'
GAPI_FAILED_PRECONDITION = 'failedPrecondition'
GAPI_FORBIDDEN = 'forbidden'
GAPI_GROUP_NOT_FOUND = 'groupNotFound'
GAPI_INTERNAL_ERROR = 'internalError'
GAPI_INVALID = 'invalid'
GAPI_INVALID_ARGUMENT = 'invalidArgument'
GAPI_INVALID_MEMBER = 'invalidMember'
GAPI_MEMBER_NOT_FOUND = 'memberNotFound'
GAPI_NOT_FOUND = 'notFound'
GAPI_NOT_IMPLEMENTED = 'notImplemented'
GAPI_PERMISSION_DENIED = 'permissionDenied'
GAPI_QUOTA_EXCEEDED = 'quotaExceeded'
GAPI_RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
GAPI_RESOURCE_NOT_FOUND = 'resourceNotFound'
GAPI_SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
GAPI_SYSTEM_ERROR = 'systemError'
GAPI_USER_NOT_FOUND = 'userNotFound'
GAPI_USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
#
GAPI_DEFAULT_RETRY_REASONS = [GAPI_QUOTA_EXCEEDED, GAPI_RATE_LIMIT_EXCEEDED, GAPI_USER_RATE_LIMIT_EXCEEDED, GAPI_BACKEND_ERROR, GAPI_INTERNAL_ERROR]
GAPI_GMAIL_THROW_REASONS = [GAPI_SERVICE_NOT_AVAILABLE]
GAPI_GROUP_GET_THROW_REASONS = [GAPI_GROUP_NOT_FOUND, GAPI_DOMAIN_NOT_FOUND, GAPI_DOMAIN_CANNOT_USE_APIS, GAPI_FORBIDDEN, GAPI_BAD_REQUEST]
GAPI_GROUP_GET_RETRY_REASONS = [GAPI_INVALID, GAPI_SYSTEM_ERROR]
GAPI_MEMBERS_THROW_REASONS = [GAPI_GROUP_NOT_FOUND, GAPI_DOMAIN_NOT_FOUND, GAPI_DOMAIN_CANNOT_USE_APIS, GAPI_INVALID, GAPI_FORBIDDEN]
GAPI_MEMBERS_RETRY_REASONS = [GAPI_SYSTEM_ERROR]
USER_ADDRESS_TYPES = ['home', 'work', 'other']
USER_EMAIL_TYPES = ['home', 'work', 'other']