mirror of
https://github.com/GAM-team/GAM.git
synced 2026-06-28 09:51:36 +00:00
Merge branch 'master' of https://github.com/jay0lee/GAM
This commit is contained in:
Jay Lee
533
src/gam.py
533
src/gam.py
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,7 @@
|
||||
"""Methods related to execution of GAPI requests."""
|
||||
|
||||
import sys
|
||||
|
||||
import controlflow
|
||||
import display
|
||||
from gapi import errors
|
||||
@@ -8,8 +10,8 @@ import httplib2
|
||||
from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION,
|
||||
GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
|
||||
GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
|
||||
MESSAGE_API_ACCESS_CONFIG, MESSAGE_API_ACCESS_DENIED,
|
||||
MESSAGE_SERVICE_NOT_APPLICABLE)
|
||||
MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
|
||||
MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
|
||||
import google.auth.exceptions
|
||||
|
||||
|
||||
@@ -138,6 +140,163 @@ def call(service,
|
||||
controlflow.system_error_exit(4, str(e))
|
||||
|
||||
|
||||
def get_items(service,
|
||||
function,
|
||||
items='items',
|
||||
throw_reasons=None,
|
||||
retry_reasons=None,
|
||||
**kwargs):
|
||||
"""Gets a single page of items from a Google service function that is paged.
|
||||
|
||||
Args:
|
||||
service: A Google service object for the desired API.
|
||||
function: String, The name of a service request method to execute.
|
||||
items: String, the name of the resulting "items" field within the service
|
||||
method's response object.
|
||||
throw_reasons: A list of Google HTTP error reason strings indicating the
|
||||
errors generated by this request should be re-thrown. All other HTTP
|
||||
errors are consumed.
|
||||
retry_reasons: A list of Google HTTP error reason strings indicating which
|
||||
error should be retried, using exponential backoff techniques, when the
|
||||
error reason is encountered.
|
||||
**kwargs: Additional params to pass to the request method.
|
||||
|
||||
Returns:
|
||||
The list of items in the first page of a response.
|
||||
"""
|
||||
results = call(
|
||||
service,
|
||||
function,
|
||||
throw_reasons=throw_reasons,
|
||||
retry_reasons=retry_reasons,
|
||||
**kwargs)
|
||||
if results:
|
||||
return results.get(items, [])
|
||||
return []
|
||||
|
||||
|
||||
def _get_max_page_size_for_api_call(service, function, **kwargs):
|
||||
"""Gets the maximum number of results supported for a single API call.
|
||||
|
||||
Args:
|
||||
service: A Google service object for the desired API.
|
||||
function: String, The name of the service method to check for max page size.
|
||||
**kwargs: Additional params that will be passed to the request method.
|
||||
|
||||
Returns:
|
||||
Int, A value from discovery if it exists, otherwise value from
|
||||
MAX_RESULTS_API_EXCEPTIONS, otherwise None
|
||||
"""
|
||||
method = getattr(service, function)
|
||||
api_id = method(**kwargs).methodId
|
||||
for resource in service._rootDesc.get('resources', {}).values():
|
||||
for a_method in resource.get('methods', {}).values():
|
||||
if a_method.get('id') == api_id:
|
||||
if not a_method.get('parameters') or a_method['parameters'].get(
|
||||
'pageSize') or not a_method['parameters'].get('maxResults'):
|
||||
# Make sure API call supports maxResults. For now we don't care to
|
||||
# set pageSize since all known pageSize API calls have
|
||||
# default pageSize == max pageSize.
|
||||
return None
|
||||
known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
|
||||
max_results = a_method['parameters']['maxResults'].get(
|
||||
'maximum', known_api_max)
|
||||
return {'maxResults': max_results}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_all_pages(service,
|
||||
function,
|
||||
items='items',
|
||||
page_message=None,
|
||||
message_attribute=None,
|
||||
soft_errors=False,
|
||||
throw_reasons=None,
|
||||
retry_reasons=None,
|
||||
**kwargs):
|
||||
"""Aggregates and returns all pages of a Google service function response.
|
||||
|
||||
All pages of items are aggregated and returned as a single list.
|
||||
|
||||
Args:
|
||||
service: A Google service object for the desired API.
|
||||
function: String, The name of a service request method to execute.
|
||||
items: String, the name of the resulting "items" field within the method's
|
||||
response object. The items in this field will be aggregated across all
|
||||
pages and returned.
|
||||
page_message: String, a message to be displayed to the user during paging.
|
||||
Template strings allow for dynamic content to be inserted during paging.
|
||||
Supported template strings:
|
||||
%%total_items%% : The current number of items discovered across all
|
||||
pages.
|
||||
%%first_item%% : In conjunction with `message_attribute` arg, will
|
||||
display a unique property of the first item in the current page.
|
||||
%%last_item%% : In conjunction with `message_attribute` arg, will
|
||||
display a unique property of the last item in the current page.
|
||||
message_attribute: String, the name of a signature field within a single
|
||||
returned item which identifies that unique item. This field is used with
|
||||
`page_message` to templatize a paging status message.
|
||||
soft_errors: Bool, If True, writes non-fatal errors to stderr.
|
||||
throw_reasons: A list of Google HTTP error reason strings indicating the
|
||||
errors generated by this request should be re-thrown. All other HTTP
|
||||
errors are consumed.
|
||||
retry_reasons: A list of Google HTTP error reason strings indicating which
|
||||
error should be retried, using exponential backoff techniques, when the
|
||||
error reason is encountered.
|
||||
**kwargs: Additional params to pass to the request method.
|
||||
|
||||
Returns:
|
||||
A list of all items received from all paged responses.
|
||||
"""
|
||||
if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
|
||||
page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
|
||||
if page_key:
|
||||
kwargs.update(page_key)
|
||||
all_items = []
|
||||
page_token = None
|
||||
total_items = 0
|
||||
while True:
|
||||
page = call(
|
||||
service,
|
||||
function,
|
||||
soft_errors=soft_errors,
|
||||
throw_reasons=throw_reasons,
|
||||
retry_reasons=retry_reasons,
|
||||
pageToken=page_token,
|
||||
**kwargs)
|
||||
if page:
|
||||
page_token = page.get('nextPageToken')
|
||||
page_items = page.get(items, [])
|
||||
num_page_items = len(page_items)
|
||||
total_items += num_page_items
|
||||
all_items.extend(page_items)
|
||||
else:
|
||||
page_token = None
|
||||
num_page_items = 0
|
||||
|
||||
# Show a paging message to the user that indicates paging progress
|
||||
if page_message:
|
||||
show_message = page_message.replace('%%total_items%%', str(total_items))
|
||||
if message_attribute:
|
||||
first_item = page_items[0] if num_page_items > 0 else {}
|
||||
last_item = page_items[-1] if num_page_items > 1 else first_item
|
||||
show_message = show_message.replace(
|
||||
'%%first_item%%', str(first_item.get(message_attribute, '')))
|
||||
show_message = show_message.replace(
|
||||
'%%last_item%%', str(last_item.get(message_attribute, '')))
|
||||
sys.stderr.write('\r')
|
||||
sys.stderr.flush()
|
||||
sys.stderr.write(show_message)
|
||||
|
||||
if not page_token:
|
||||
# End the paging status message and return all items.
|
||||
if page_message and (page_message[-1] != '\n'):
|
||||
sys.stderr.write('\r\n')
|
||||
sys.stderr.flush()
|
||||
return all_items
|
||||
|
||||
|
||||
# TODO: Make this private once all execution related items that use this method
|
||||
# have been brought into this file
|
||||
def handle_oauth_token_error(e, soft_errors):
|
||||
|
||||
@@ -72,14 +72,48 @@ class CreateHttpTest(unittest.TestCase):
|
||||
self.assertEqual(http.timeout, 1234)
|
||||
|
||||
|
||||
class CallTest(unittest.TestCase):
|
||||
class GapiTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
SetGlobalVariables()
|
||||
self.mock_service = MagicMock()
|
||||
self.mock_method_name = 'mock_method'
|
||||
self.mock_method = getattr(self.mock_service, self.mock_method_name)
|
||||
super(CallTest, self).setUp()
|
||||
|
||||
self.simple_3_page_response = [
|
||||
{
|
||||
'items': [{
|
||||
'position': 'page1,item1'
|
||||
}, {
|
||||
'position': 'page1,item2'
|
||||
}, {
|
||||
'position': 'page1,item3'
|
||||
}],
|
||||
'nextPageToken': 'page2'
|
||||
},
|
||||
{
|
||||
'items': [{
|
||||
'position': 'page2,item1'
|
||||
}, {
|
||||
'position': 'page2,item2'
|
||||
}, {
|
||||
'position': 'page2,item3'
|
||||
}],
|
||||
'nextPageToken': 'page3'
|
||||
},
|
||||
{
|
||||
'items': [{
|
||||
'position': 'page3,item1'
|
||||
}, {
|
||||
'position': 'page3,item2'
|
||||
}, {
|
||||
'position': 'page3,item3'
|
||||
}],
|
||||
},
|
||||
]
|
||||
self.empty_items_response = {'items': []}
|
||||
|
||||
super(GapiTest, self).setUp()
|
||||
|
||||
def test_call_returns_basic_200_response(self):
|
||||
response = gapi.call(self.mock_service, self.mock_method_name)
|
||||
@@ -236,6 +270,269 @@ class CallTest(unittest.TestCase):
|
||||
# Make sure a backoff technique was used for retry.
|
||||
self.assertEqual(mock_wait_on_failure.call_count, 1)
|
||||
|
||||
def test_get_items_calls_correct_service_function(self):
|
||||
gapi.get_items(self.mock_service, self.mock_method_name)
|
||||
self.assertTrue(self.mock_method.called)
|
||||
|
||||
def test_get_items_returns_one_page(self):
|
||||
fake_response = {'items': [{}, {}, {}]}
|
||||
self.mock_method.return_value.execute.return_value = fake_response
|
||||
page = gapi.get_items(self.mock_service, self.mock_method_name)
|
||||
self.assertEqual(page, fake_response['items'])
|
||||
|
||||
def test_get_items_non_default_page_field_name(self):
|
||||
field_name = 'things'
|
||||
fake_response = {field_name: [{}, {}, {}]}
|
||||
self.mock_method.return_value.execute.return_value = fake_response
|
||||
page = gapi.get_items(
|
||||
self.mock_service, self.mock_method_name, items=field_name)
|
||||
self.assertEqual(page, fake_response[field_name])
|
||||
|
||||
def test_get_items_passes_additional_kwargs_to_service(self):
|
||||
gapi.get_items(
|
||||
self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
|
||||
self.assertEqual(self.mock_method.call_count, 1)
|
||||
method_kwargs = self.mock_method.call_args[1]
|
||||
self.assertEqual(1, method_kwargs.get('my_param_1'))
|
||||
self.assertEqual(2, method_kwargs.get('my_param_2'))
|
||||
|
||||
def test_get_items_returns_empty_list_when_no_items_returned(self):
|
||||
non_items_response = {'noItemsInThisResponse': {}}
|
||||
self.mock_method.return_value.execute.return_value = non_items_response
|
||||
page = gapi.get_items(self.mock_service, self.mock_method_name)
|
||||
self.assertIsInstance(page, list)
|
||||
self.assertEqual(0, len(page))
|
||||
|
||||
def test_get_all_pages_returns_all_items(self):
|
||||
page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
|
||||
page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
|
||||
page_3 = {'items': ['3-1', '3-2', '3-3']}
|
||||
self.mock_method.return_value.execute.side_effect = [page_1, page_2, page_3]
|
||||
response_items = gapi.get_all_pages(self.mock_service,
|
||||
self.mock_method_name)
|
||||
self.assertListEqual(response_items,
|
||||
page_1['items'] + page_2['items'] + page_3['items'])
|
||||
|
||||
def test_get_all_pages_includes_next_pagetoken_in_request(self):
|
||||
page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
|
||||
page_2 = {'items': ['2-1', '2-2', '2-3']}
|
||||
self.mock_method.return_value.execute.side_effect = [page_1, page_2]
|
||||
|
||||
gapi.get_all_pages(self.mock_service, self.mock_method_name, pageSize=100)
|
||||
self.assertEqual(self.mock_method.call_count, 2)
|
||||
call_2_kwargs = self.mock_method.call_args_list[1][1]
|
||||
self.assertIn('pageToken', call_2_kwargs)
|
||||
self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
|
||||
|
||||
def test_get_all_pages_uses_default_max_page_size(self):
|
||||
sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
|
||||
sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
|
||||
self.mock_method.return_value.methodId = sample_api_id
|
||||
self.mock_service._rootDesc = {
|
||||
'resources': {
|
||||
'someResource': {
|
||||
'methods': {
|
||||
'someMethod': {
|
||||
'id': sample_api_id,
|
||||
'parameters': {
|
||||
'maxResults': {
|
||||
'maximum': sample_api_max_results
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.mock_method.return_value.execute.return_value = self.empty_items_response
|
||||
|
||||
gapi.get_all_pages(self.mock_service, self.mock_method_name)
|
||||
request_method_kwargs = self.mock_method.call_args[1]
|
||||
self.assertIn('maxResults', request_method_kwargs)
|
||||
self.assertEqual(request_method_kwargs['maxResults'],
|
||||
gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
|
||||
|
||||
def test_get_all_pages_max_page_size_overrided(self):
|
||||
self.mock_method.return_value.execute.return_value = self.empty_items_response
|
||||
|
||||
gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, pageSize=123456)
|
||||
request_method_kwargs = self.mock_method.call_args[1]
|
||||
self.assertIn('pageSize', request_method_kwargs)
|
||||
self.assertEqual(123456, request_method_kwargs['pageSize'])
|
||||
|
||||
@patch.object(gapi.sys.stderr, 'write')
|
||||
def test_get_all_pages_prints_paging_message(self, mock_write):
|
||||
self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
|
||||
|
||||
paging_message = 'A simple string displayed during paging'
|
||||
gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, page_message=paging_message)
|
||||
messages_written = [
|
||||
call_args[0][0] for call_args in mock_write.call_args_list
|
||||
]
|
||||
self.assertIn(paging_message, messages_written)
|
||||
|
||||
@patch.object(gapi.sys.stderr, 'write')
|
||||
def test_get_all_pages_prints_paging_message_inline(self, mock_write):
|
||||
self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
|
||||
|
||||
paging_message = 'A simple string displayed during paging'
|
||||
gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, page_message=paging_message)
|
||||
messages_written = [
|
||||
call_args[0][0] for call_args in mock_write.call_args_list
|
||||
]
|
||||
|
||||
# Make sure a return carriage was written between two pages
|
||||
paging_message_call_positions = [
|
||||
i for i, message in enumerate(messages_written)
|
||||
if message == paging_message
|
||||
]
|
||||
self.assertGreater(len(paging_message_call_positions), 1)
|
||||
printed_between_page_messages = messages_written[
|
||||
paging_message_call_positions[0]:paging_message_call_positions[1]]
|
||||
self.assertIn('\r', printed_between_page_messages)
|
||||
|
||||
@patch.object(gapi.sys.stderr, 'write')
|
||||
def test_get_all_pages_ends_paging_message_with_newline(self, mock_write):
|
||||
self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
|
||||
|
||||
paging_message = 'A simple string displayed during paging'
|
||||
gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, page_message=paging_message)
|
||||
messages_written = [
|
||||
call_args[0][0] for call_args in mock_write.call_args_list
|
||||
]
|
||||
last_page_message_index = len(
|
||||
messages_written) - messages_written[::-1].index(paging_message)
|
||||
last_carriage_return_index = len(
|
||||
messages_written) - messages_written[::-1].index('\r\n')
|
||||
self.assertGreater(last_carriage_return_index, last_page_message_index)
|
||||
|
||||
@patch.object(gapi.sys.stderr, 'write')
|
||||
def test_get_all_pages_prints_attribute_total_items_in_paging_message(
|
||||
self, mock_write):
|
||||
self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
|
||||
|
||||
paging_message = 'Total number of items discovered: %%total_items%%'
|
||||
gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, page_message=paging_message)
|
||||
|
||||
messages_written = [
|
||||
call_args[0][0] for call_args in mock_write.call_args_list
|
||||
]
|
||||
page_1_item_count = len(self.simple_3_page_response[0]['items'])
|
||||
page_1_message = paging_message.replace('%%total_items%%',
|
||||
str(page_1_item_count))
|
||||
self.assertIn(page_1_message, messages_written)
|
||||
|
||||
page_2_item_count = len(self.simple_3_page_response[1]['items'])
|
||||
page_2_message = paging_message.replace(
|
||||
'%%total_items%%', str(page_1_item_count + page_2_item_count))
|
||||
self.assertIn(page_2_message, messages_written)
|
||||
|
||||
page_3_item_count = len(self.simple_3_page_response[2]['items'])
|
||||
page_3_message = paging_message.replace(
|
||||
'%%total_items%%',
|
||||
str(page_1_item_count + page_2_item_count + page_3_item_count))
|
||||
self.assertIn(page_3_message, messages_written)
|
||||
|
||||
# Assert that the template text is always replaced.
|
||||
for message in messages_written:
|
||||
self.assertNotIn('%%total_items', message)
|
||||
|
||||
@patch.object(gapi.sys.stderr, 'write')
|
||||
def test_get_all_pages_prints_attribute_first_item_in_paging_message(
|
||||
self, mock_write):
|
||||
self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
|
||||
|
||||
paging_message = 'First item in page: %%first_item%%'
|
||||
gapi.get_all_pages(
|
||||
self.mock_service,
|
||||
self.mock_method_name,
|
||||
page_message=paging_message,
|
||||
message_attribute='position')
|
||||
|
||||
messages_written = [
|
||||
call_args[0][0] for call_args in mock_write.call_args_list
|
||||
]
|
||||
page_1_message = paging_message.replace(
|
||||
'%%first_item%%',
|
||||
self.simple_3_page_response[0]['items'][0]['position'])
|
||||
self.assertIn(page_1_message, messages_written)
|
||||
|
||||
page_2_message = paging_message.replace(
|
||||
'%%first_item%%',
|
||||
self.simple_3_page_response[1]['items'][0]['position'])
|
||||
self.assertIn(page_2_message, messages_written)
|
||||
|
||||
# Assert that the template text is always replaced.
|
||||
for message in messages_written:
|
||||
self.assertNotIn('%%first_item', message)
|
||||
|
||||
@patch.object(gapi.sys.stderr, 'write')
|
||||
def test_get_all_pages_prints_attribute_last_item_in_paging_message(
|
||||
self, mock_write):
|
||||
self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
|
||||
|
||||
paging_message = 'Last item in page: %%last_item%%'
|
||||
gapi.get_all_pages(
|
||||
self.mock_service,
|
||||
self.mock_method_name,
|
||||
page_message=paging_message,
|
||||
message_attribute='position')
|
||||
|
||||
messages_written = [
|
||||
call_args[0][0] for call_args in mock_write.call_args_list
|
||||
]
|
||||
page_1_message = paging_message.replace(
|
||||
'%%last_item%%',
|
||||
self.simple_3_page_response[0]['items'][-1]['position'])
|
||||
self.assertIn(page_1_message, messages_written)
|
||||
|
||||
page_2_message = paging_message.replace(
|
||||
'%%last_item%%',
|
||||
self.simple_3_page_response[1]['items'][-1]['position'])
|
||||
self.assertIn(page_2_message, messages_written)
|
||||
|
||||
# Assert that the template text is always replaced.
|
||||
for message in messages_written:
|
||||
self.assertNotIn('%%last_item', message)
|
||||
|
||||
def test_get_all_pages_prints_all_attributes_in_paging_message(self):
|
||||
pass
|
||||
|
||||
def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
|
||||
self.mock_method.return_value.execute.return_value = self.empty_items_response
|
||||
gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
|
||||
method_kwargs = self.mock_method.call_args[1]
|
||||
self.assertEqual(method_kwargs.get('my_param_1'), 1)
|
||||
self.assertEqual(method_kwargs.get('my_param_2'), 2)
|
||||
|
||||
@patch.object(gapi, 'call')
|
||||
def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
|
||||
throw_for = MagicMock()
|
||||
retry_for = MagicMock()
|
||||
mock_call.return_value = self.empty_items_response
|
||||
gapi.get_all_pages(
|
||||
self.mock_service,
|
||||
self.mock_method_name,
|
||||
throw_reasons=throw_for,
|
||||
retry_reasons=retry_for)
|
||||
method_kwargs = mock_call.call_args[1]
|
||||
self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
|
||||
self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
|
||||
|
||||
def test_get_all_pages_non_default_items_field_name(self):
|
||||
field_name = 'things'
|
||||
fake_response = {field_name: [{}, {}, {}]}
|
||||
self.mock_method.return_value.execute.return_value = fake_response
|
||||
page = gapi.get_all_pages(
|
||||
self.mock_service, self.mock_method_name, items=field_name)
|
||||
self.assertEqual(page, fake_response[field_name])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user