mirror of
https://github.com/GAM-team/GAM.git
synced 2025-07-10 22:53:34 +00:00
Consolidate callGAPIpages() implementation and add docstrings on all GAPI execution-related methods (#851)
* Consolidate callGAPIpages() implementation and add docstrings on all GAPI execution-related methods callGAPIpages was previously broken out into a few subroutines which contained a couple lines of code and weren't being used elsewhere in the code. The main "GAPI" execution methods were also missing documentation which made it hard to tell what they did and/or how to use their parameters. * Fix oauth2client library name
This commit is contained in:
ejochman
Jay Lee
185
src/gam.py
185
src/gam.py
@ -852,12 +852,32 @@ GAPI_REASON_EXCEPTION_MAP = {
|
||||
}
|
||||
|
||||
def callGAPI(service, function,
|
||||
silent_errors=False, soft_errors=False, throw_reasons=None, retry_reasons=None,
|
||||
silent_errors=False, soft_errors=False,
|
||||
throw_reasons=None, retry_reasons=None,
|
||||
**kwargs):
|
||||
"""Executes a single request on a Google service function.
|
||||
|
||||
Args:
|
||||
service: A Google service object for the desired API.
|
||||
function: String, The name of a service request method to execute.
|
||||
silent_errors: Bool, If True, error messages are suppressed when
|
||||
encountered.
|
||||
soft_errors: Bool, If True, writes non-fatal errors to stderr.
|
||||
throw_reasons: A list of Google HTTP error reason strings indicating the
|
||||
errors generated by this request should be re-thrown. All other HTTP
|
||||
errors are consumed.
|
||||
retry_reasons: A list of Google HTTP error reason strings indicating which
|
||||
error should be retried, using exponential backoff techniques, when the
|
||||
error reason is encountered.
|
||||
|
||||
Returns:
|
||||
The given Google service function's response object.
|
||||
"""
|
||||
if throw_reasons is None:
|
||||
throw_reasons = []
|
||||
if retry_reasons is None:
|
||||
retry_reasons = []
|
||||
|
||||
method = getattr(service, function)
|
||||
retries = 10
|
||||
parameters = dict(kwargs.items() + GM_Globals[GM_EXTRA_ARGS_DICT].items())
|
||||
@ -894,73 +914,122 @@ def callGAPI(service, function,
|
||||
service._http.cache = None
|
||||
continue
|
||||
systemErrorExit(4, str(e))
|
||||
except TypeError as e:
|
||||
systemErrorExit(4, str(e))
|
||||
except httplib2.ServerNotFoundError as e:
|
||||
except (TypeError, httplib2.ServerNotFoundError) as e:
|
||||
systemErrorExit(4, str(e))
|
||||
|
||||
def _processGAPIpagesResult(this_page, items, all_pages, total_items, page_message, message_attribute):
|
||||
if this_page:
|
||||
pageToken = this_page.get(u'nextPageToken')
|
||||
if items in this_page:
|
||||
page_items = len(this_page[items])
|
||||
total_items += page_items
|
||||
if all_pages is not None:
|
||||
all_pages.extend(this_page[items])
|
||||
else:
|
||||
this_page = {items: []}
|
||||
page_items = 0
|
||||
else:
|
||||
pageToken = None
|
||||
this_page = {items: []}
|
||||
page_items = 0
|
||||
if page_message:
|
||||
show_message = page_message.replace(u'%%num_items%%', str(page_items))
|
||||
show_message = show_message.replace(u'%%total_items%%', str(total_items))
|
||||
if message_attribute:
|
||||
try:
|
||||
show_message = show_message.replace(u'%%first_item%%', str(this_page[items][0][message_attribute]))
|
||||
show_message = show_message.replace(u'%%last_item%%', str(this_page[items][-1][message_attribute]))
|
||||
except (IndexError, KeyError):
|
||||
show_message = show_message.replace(u'%%first_item%%', u'')
|
||||
show_message = show_message.replace(u'%%last_item%%', u'')
|
||||
sys.stderr.write(u'\r')
|
||||
sys.stderr.flush()
|
||||
sys.stderr.write(show_message)
|
||||
return (pageToken, total_items)
|
||||
|
||||
def _finalizeGAPIpagesResult(page_message):
|
||||
if page_message and (page_message[-1] != u'\n'):
|
||||
sys.stderr.write(u'\r\n')
|
||||
sys.stderr.flush()
|
||||
|
||||
def callGAPIpages(service, function, items,
|
||||
def callGAPIpages(service, function, items=u'items',
|
||||
page_message=None, message_attribute=None,
|
||||
soft_errors=False, throw_reasons=None, retry_reasons=None,
|
||||
**kwargs):
|
||||
if throw_reasons is None:
|
||||
throw_reasons = []
|
||||
all_pages = list()
|
||||
"""Aggregates and returns all pages of a Google service function response.
|
||||
|
||||
All pages of items are aggregated and returned as a single list.
|
||||
|
||||
Args:
|
||||
service: A Google service object for the desired API.
|
||||
function: String, The name of a service request method to execute.
|
||||
items: String, the name of the resulting "items" field within the method's
|
||||
response object. The items in this field will be aggregated across all
|
||||
pages and returned.
|
||||
page_message: String, a message to be displayed to the user during paging.
|
||||
Template strings allow for dynamic content to be inserted during paging.
|
||||
|
||||
Supported template strings:
|
||||
%%num_items%% : The number of items in the current page.
|
||||
%%total_items%% : The current number of items discovered across all
|
||||
pages.
|
||||
%%first_item%% : In conjunction with `message_attribute` arg, will
|
||||
display a unique property of the first item in the
|
||||
current page.
|
||||
%%last_item%% : In conjunction with `message_attribute` arg, will
|
||||
display a unique property of the last item in the
|
||||
current page.
|
||||
|
||||
message_attribute: String, the name of a signature field within a single
|
||||
returned item which identifies that unique item. This field is used with
|
||||
`page_message` to templatize a paging status message.
|
||||
soft_errors: Bool, If True, writes non-fatal errors to stderr.
|
||||
throw_reasons: A list of Google HTTP error reason strings indicating the
|
||||
errors generated by this request should be re-thrown. All other HTTP
|
||||
errors are consumed.
|
||||
retry_reasons: A list of Google HTTP error reason strings indicating which
|
||||
error should be retried, using exponential backoff techniques, when the
|
||||
error reason is encountered.
|
||||
|
||||
Returns:
|
||||
A list of all items received from all paged responses.
|
||||
"""
|
||||
all_items = []
|
||||
page_token = None
|
||||
total_items = 0
|
||||
while True:
|
||||
this_page = callGAPI(service, function, soft_errors=soft_errors,
|
||||
throw_reasons=throw_reasons, retry_reasons=retry_reasons, **kwargs)
|
||||
pageToken, total_items = _processGAPIpagesResult(this_page, items, all_pages, total_items, page_message, message_attribute)
|
||||
if not pageToken:
|
||||
_finalizeGAPIpagesResult(page_message)
|
||||
return all_pages
|
||||
kwargs[u'pageToken'] = pageToken
|
||||
page = callGAPI(
|
||||
service,
|
||||
function,
|
||||
soft_errors=soft_errors,
|
||||
throw_reasons=throw_reasons,
|
||||
retry_reasons=retry_reasons,
|
||||
pageToken=page_token,
|
||||
**kwargs)
|
||||
if page:
|
||||
page_token = page.get(u'nextPageToken')
|
||||
page_items = page.get(items, [])
|
||||
num_page_items = len(page_items)
|
||||
total_items += num_page_items
|
||||
all_items.extend(page_items)
|
||||
else:
|
||||
page_token = None
|
||||
num_page_items = 0
|
||||
|
||||
def callGAPIitems(service, function, items,
|
||||
# Show a paging message to the user that indicates paging progress
|
||||
if page_message:
|
||||
show_message = page_message.replace(u'%%num_items%%', str(num_page_items))
|
||||
show_message = show_message.replace(u'%%total_items%%', str(total_items))
|
||||
if message_attribute:
|
||||
first_item = page_items[0] if num_page_items > 0 else {}
|
||||
last_item = page_items[-1] if num_page_items > 1 else first_item
|
||||
show_message = show_message.replace(
|
||||
u'%%first_item%%', str(first_item.get(message_attribute, u'')))
|
||||
show_message = show_message.replace(
|
||||
u'%%last_item%%', str(last_item.get(message_attribute, u'')))
|
||||
sys.stderr.write(u'\r')
|
||||
sys.stderr.flush()
|
||||
sys.stderr.write(show_message)
|
||||
|
||||
if not page_token:
|
||||
# End the paging status message and return all items.
|
||||
if page_message and (page_message[-1] != u'\n'):
|
||||
sys.stderr.write(u'\r\n')
|
||||
sys.stderr.flush()
|
||||
return all_items
|
||||
|
||||
def callGAPIitems(service, function, items=u'items',
|
||||
throw_reasons=None, retry_reasons=None,
|
||||
**kwargs):
|
||||
if throw_reasons is None:
|
||||
throw_reasons = []
|
||||
if retry_reasons is None:
|
||||
retry_reasons = []
|
||||
results = callGAPI(service, function,
|
||||
throw_reasons=throw_reasons, retry_reasons=retry_reasons,
|
||||
**kwargs)
|
||||
"""Gets a single page of items from a Google service function that is paged.
|
||||
|
||||
Args:
|
||||
service: A Google service object for the desired API.
|
||||
function: String, The name of a service request method to execute.
|
||||
items: String, the name of the resulting "items" field within the service
|
||||
method's response object.
|
||||
soft_errors: Bool, If True, writes non-fatal errors to stderr.
|
||||
throw_reasons: A list of Google HTTP error reason strings indicating the
|
||||
errors generated by this request should be re-thrown. All other HTTP
|
||||
errors are consumed.
|
||||
retry_reasons: A list of Google HTTP error reason strings indicating which
|
||||
error should be retried, using exponential backoff techniques, when the
|
||||
error reason is encountered.
|
||||
|
||||
Returns:
|
||||
The list of items in the first page of a response.
|
||||
"""
|
||||
results = callGAPI(
|
||||
service,
|
||||
function,
|
||||
throw_reasons=throw_reasons,
|
||||
retry_reasons=retry_reasons,
|
||||
**kwargs)
|
||||
if results:
|
||||
return results.get(items, [])
|
||||
return []
|
||||
|
||||
Reference in New Issue
Block a user