diff --git a/src/gam/cmd/admin.py b/src/gam/cmd/admin.py index 754a186a..54a9b538 100644 --- a/src/gam/cmd/admin.py +++ b/src/gam/cmd/admin.py @@ -12,7 +12,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import accessErrorExit -from gam.util.api import buildGAPIObject, callGAPI, callGAPIitems, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( UID_PATTERN, checkForExtraneousArguments, diff --git a/src/gam/cmd/alerts.py b/src/gam/cmd/alerts.py index 6fd190cc..d3261bc7 100644 --- a/src/gam/cmd/alerts.py +++ b/src/gam/cmd/alerts.py @@ -11,7 +11,9 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import _getAdminEmail, buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.api import _getAdminEmail +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, checkForExtraneousArguments, diff --git a/src/gam/cmd/aliases.py b/src/gam/cmd/aliases.py index b3518c2f..10b36014 100644 --- a/src/gam/cmd/aliases.py +++ b/src/gam/cmd/aliases.py @@ -10,7 +10,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkForExtraneousArguments, getAddCSVData, diff --git a/src/gam/cmd/analytics.py b/src/gam/cmd/analytics.py index 174021b5..2913460e 100644 --- a/src/gam/cmd/analytics.py +++ b/src/gam/cmd/analytics.py @@ -8,7 +8,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIServiceObject, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPIpages from gam.util.args import getArgument, getBoolean, getInteger, getString from gam.util.csv_pf import ( CSVPrintFile, diff --git a/src/gam/cmd/audit.py b/src/gam/cmd/audit.py index d39ed341..c12b720d 100644 --- a/src/gam/cmd/audit.py +++ b/src/gam/cmd/audit.py @@ -12,7 +12,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import callGData, getEmailAuditObject +from gam.util.api import getEmailAuditObject +from gam.util.api_call import callGData from gam.util.args import ( YYYYMMDD_HHMM_FORMAT, checkForExtraneousArguments, diff --git a/src/gam/cmd/browsers.py b/src/gam/cmd/browsers.py index f66a2707..22b87b4a 100644 --- a/src/gam/cmd/browsers.py +++ b/src/gam/cmd/browsers.py @@ -9,7 +9,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages, yieldGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages, yieldGAPIpages from gam.util.args import ( OrderBy, checkForExtraneousArguments, diff --git a/src/gam/cmd/caa.py b/src/gam/cmd/caa.py index 9c68dc12..40dabcc8 100644 --- a/src/gam/cmd/caa.py +++ b/src/gam/cmd/caa.py @@ -9,7 +9,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( AND_OR_CONJUNCTION_MAP, checkForExtraneousArguments, diff --git a/src/gam/cmd/calendar.py b/src/gam/cmd/calendar.py index 1b0b6b67..4bdb4f02 100644 --- a/src/gam/cmd/calendar.py +++ b/src/gam/cmd/calendar.py @@ -13,13 +13,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit, entityUnknownWarning -from gam.util.api import ( - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, - checkGAPIError, -) +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( CALENDAR_EVENT_COLOR_MAP, YYYYMMDD_FORMAT, diff --git a/src/gam/cmd/chat/members.py b/src/gam/cmd/chat/members.py index 6685b7f4..1002e603 100644 --- a/src/gam/cmd/chat/members.py +++ b/src/gam/cmd/chat/members.py @@ -11,7 +11,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import _getAdminEmail, buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( AND_OR_CONJUNCTION_MAP, OrderBy, diff --git a/src/gam/cmd/chat/setup.py b/src/gam/cmd/chat/setup.py index 113d182f..df47735c 100644 --- a/src/gam/cmd/chat/setup.py +++ b/src/gam/cmd/chat/setup.py @@ -15,13 +15,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import ( - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIitems, - callGAPIpages, -) +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( UTF8, checkArgumentPresent, diff --git a/src/gam/cmd/chat/spaces.py b/src/gam/cmd/chat/spaces.py index 82ad36cf..da6a9407 100644 --- a/src/gam/cmd/chat/spaces.py +++ b/src/gam/cmd/chat/spaces.py @@ -13,7 +13,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, SORF_TEXT_ARGUMENTS, diff --git a/src/gam/cmd/chromeapps.py b/src/gam/cmd/chromeapps.py index ab26a90a..e0bf1a91 100644 --- a/src/gam/cmd/chromeapps.py +++ b/src/gam/cmd/chromeapps.py @@ -10,7 +10,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit -from gam.util.api import buildGAPIObject, buildGAPIObjectNoAuthentication, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject, buildGAPIObjectNoAuthentication +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, YYYYMMDD_FORMAT, diff --git a/src/gam/cmd/chromepolicies.py b/src/gam/cmd/chromepolicies.py index 081ef8bf..6c2943dd 100644 --- a/src/gam/cmd/chromepolicies.py +++ b/src/gam/cmd/chromepolicies.py @@ -12,7 +12,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( FALSE_VALUES, TRUE_FALSE, diff --git a/src/gam/cmd/cidevices.py b/src/gam/cmd/cidevices.py index db094ef6..eae50baa 100644 --- a/src/gam/cmd/cidevices.py +++ b/src/gam/cmd/cidevices.py @@ -11,14 +11,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import ( - _getAdminEmail, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, - transportCreateRequest, -) +from gam.util.api import _getAdminEmail, buildGAPIObject, transportCreateRequest +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( NEVER_TIME_NOMS, OrderBy, diff --git a/src/gam/cmd/cigroups/groups.py b/src/gam/cmd/cigroups/groups.py index cb662b86..594f7e2c 100644 --- a/src/gam/cmd/cigroups/groups.py +++ b/src/gam/cmd/cigroups/groups.py @@ -14,7 +14,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( NEVER_TIME, _getOptionalIsSuspendedIsArchived, diff --git a/src/gam/cmd/cigroups/members.py b/src/gam/cmd/cigroups/members.py index a2ee50be..999e409c 100644 --- a/src/gam/cmd/cigroups/members.py +++ b/src/gam/cmd/cigroups/members.py @@ -24,7 +24,8 @@ from gam.cmd.groups.groups import ( from gam.cmd.cigroups.groups import ALL_CIGROUP_MEMBER_TYPES, CIGROUP_MEMBER_TYPES_MAP from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages, getHttpObj +from gam.util.api import buildGAPIObject, getHttpObj +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkArgumentPresent, checkForExtraneousArguments, diff --git a/src/gam/cmd/ciuserinvitations.py b/src/gam/cmd/ciuserinvitations.py index 374f12ce..2141c643 100644 --- a/src/gam/cmd/ciuserinvitations.py +++ b/src/gam/cmd/ciuserinvitations.py @@ -10,7 +10,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, checkForExtraneousArguments, diff --git a/src/gam/cmd/cloudstorage.py b/src/gam/cmd/cloudstorage.py index 028b6114..9a546e32 100644 --- a/src/gam/cmd/cloudstorage.py +++ b/src/gam/cmd/cloudstorage.py @@ -15,7 +15,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPIpages, checkGAPIError +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPIpages, checkGAPIError from gam.util.args import getArgument, getBoolean, getString from gam.util.display import ( ACTION_NOT_PERFORMED_RC, diff --git a/src/gam/cmd/contacts.py b/src/gam/cmd/contacts.py index 1ff09be5..0c246919 100644 --- a/src/gam/cmd/contacts.py +++ b/src/gam/cmd/contacts.py @@ -16,7 +16,8 @@ from gam.constants import NO_ENTITIES_FOUND_RC from gamlib import gdata as GDATA from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import callGData, callGDataPages, getContactsObject, getContactsQuery +from gam.util.api import getContactsObject, getContactsQuery +from gam.util.api_call import callGData, callGDataPages from gam.util.args import ( LANGUAGE_CODES_MAP, checkArgumentPresent, diff --git a/src/gam/cmd/courses/content.py b/src/gam/cmd/courses/content.py index e3a63dd1..b5a221a7 100644 --- a/src/gam/cmd/courses/content.py +++ b/src/gam/cmd/courses/content.py @@ -13,7 +13,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, getArgument, diff --git a/src/gam/cmd/courses/courses.py b/src/gam/cmd/courses/courses.py index 43b13b2c..cca093a9 100644 --- a/src/gam/cmd/courses/courses.py +++ b/src/gam/cmd/courses/courses.py @@ -14,7 +14,9 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( addCourseIdScope, getAddCSVData, diff --git a/src/gam/cmd/courses/guardians.py b/src/gam/cmd/courses/guardians.py index 09e0ee55..a6f867b4 100644 --- a/src/gam/cmd/courses/guardians.py +++ b/src/gam/cmd/courses/guardians.py @@ -15,7 +15,9 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( addCourseIdScope, checkForExtraneousArguments, diff --git a/src/gam/cmd/courses/participants.py b/src/gam/cmd/courses/participants.py index c5d92873..39dfd180 100644 --- a/src/gam/cmd/courses/participants.py +++ b/src/gam/cmd/courses/participants.py @@ -14,7 +14,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIObject, callGAPI, checkGAPIError, waitOnFailure +from gam.util.api import buildGAPIObject, waitOnFailure +from gam.util.api_call import callGAPI, checkGAPIError from gam.util.args import ( SORF_TEXT_ARGUMENTS, addCourseAliasScope, diff --git a/src/gam/cmd/cros.py b/src/gam/cmd/cros.py index c3051950..1bd40d5c 100644 --- a/src/gam/cmd/cros.py +++ b/src/gam/cmd/cros.py @@ -16,15 +16,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit -from gam.util.api import ( - _finalizeGAPIpagesResult, - _processGAPIpagesResult, - buildGAPIObject, - callGAPI, - callGAPIitems, - callGAPIpages, - checkGAPIError, -) +from gam.util.api import buildGAPIObject +from gam.util.api_call import _finalizeGAPIpagesResult, _processGAPIpagesResult +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages, checkGAPIError from gam.util.output import ISOformatTimeStamp from gam.util.args import ( SORTORDER_CHOICE_MAP, diff --git a/src/gam/cmd/customer.py b/src/gam/cmd/customer.py index 8383f294..f46ca687 100644 --- a/src/gam/cmd/customer.py +++ b/src/gam/cmd/customer.py @@ -9,7 +9,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import accessErrorExit -from gam.util.api import buildGAPIObject, callGAPI, callGAPIitems +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import ( LANGUAGE_CODES_MAP, YYYYMMDD_FORMAT, diff --git a/src/gam/cmd/datatransfer.py b/src/gam/cmd/datatransfer.py index ef420177..25c44736 100644 --- a/src/gam/cmd/datatransfer.py +++ b/src/gam/cmd/datatransfer.py @@ -11,7 +11,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkForExtraneousArguments, getAddCSVData, diff --git a/src/gam/cmd/delegates.py b/src/gam/cmd/delegates.py index e608900c..0b562b29 100644 --- a/src/gam/cmd/delegates.py +++ b/src/gam/cmd/delegates.py @@ -7,7 +7,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import checkForExtraneousArguments, getArgument from gam.util.csv_pf import CSVPrintFile from gam.util.display import ( diff --git a/src/gam/cmd/domains.py b/src/gam/cmd/domains.py index acf23b4d..ccb82e3e 100644 --- a/src/gam/cmd/domains.py +++ b/src/gam/cmd/domains.py @@ -12,7 +12,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import accessErrorExit -from gam.util.api import buildGAPIObject, callGAPI, callGAPIitems +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import checkForExtraneousArguments, formatLocalTimestamp, getArgument, getString from gam.util.csv_pf import ( CSVPrintFile, diff --git a/src/gam/cmd/drive/activity.py b/src/gam/cmd/drive/activity.py index 9722c969..6ac5295d 100644 --- a/src/gam/cmd/drive/activity.py +++ b/src/gam/cmd/drive/activity.py @@ -15,15 +15,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import ( - _getAdminEmail, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIitems, - callGAPIpages, - yieldGAPIpages, -) +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages, yieldGAPIpages from gam.util.args import ( StartEndTime, checkArgumentPresent, diff --git a/src/gam/cmd/drive/copymove/copymove_move.py b/src/gam/cmd/drive/copymove/copymove_move.py index ddb7dac9..fe5037f6 100644 --- a/src/gam/cmd/drive/copymove/copymove_move.py +++ b/src/gam/cmd/drive/copymove/copymove_move.py @@ -19,7 +19,7 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import callGAPI, callGAPIpages +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import getArgument, getBoolean from gam.util.display import ( entityActionFailedWarning, diff --git a/src/gam/cmd/drive/copymove/copymove_util.py b/src/gam/cmd/drive/copymove/copymove_util.py index f340d8fa..a8eeaf2f 100644 --- a/src/gam/cmd/drive/copymove/copymove_util.py +++ b/src/gam/cmd/drive/copymove/copymove_util.py @@ -18,7 +18,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( StartEndTime, formatLocalTime, diff --git a/src/gam/cmd/drive/core.py b/src/gam/cmd/drive/core.py index d17a1726..1d768ae6 100644 --- a/src/gam/cmd/drive/core.py +++ b/src/gam/cmd/drive/core.py @@ -18,7 +18,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages, getHttpObj +from gam.util.api import getHttpObj +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( LANGUAGE_CODES_MAP, checkArgumentPresent, diff --git a/src/gam/cmd/drive/fileinfo.py b/src/gam/cmd/drive/fileinfo.py index dfd4ec2c..9752ad42 100644 --- a/src/gam/cmd/drive/fileinfo.py +++ b/src/gam/cmd/drive/fileinfo.py @@ -65,7 +65,8 @@ from gam.cmd.drive.filelist import ( _checkUpdateLastModifiction, _getLastModificationPath, _initLastModification, _showLastModification, _updateLastModificationRow, ) -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages, yieldGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages, yieldGAPIpages from gam.util.args import ( OrderBy, formatFileSize, diff --git a/src/gam/cmd/drive/filelist.py b/src/gam/cmd/drive/filelist.py index bc2453ca..966b84a1 100644 --- a/src/gam/cmd/drive/filelist.py +++ b/src/gam/cmd/drive/filelist.py @@ -93,7 +93,7 @@ from gam.cmd.drive.filetree import ( initFileTree, noFileSelectFileIdEntity, ) -from gam.util.api import callGAPI, callGAPIitems, callGAPIpages, yieldGAPIpages +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages, yieldGAPIpages from gam.util.args import ( NEVER_TIME, OrderBy, diff --git a/src/gam/cmd/drive/filepaths.py b/src/gam/cmd/drive/filepaths.py index 5b3a1e56..05831b75 100644 --- a/src/gam/cmd/drive/filepaths.py +++ b/src/gam/cmd/drive/filepaths.py @@ -51,7 +51,7 @@ SHARED_WITHME = 'SharedWithMe' SHARED_DRIVES = 'SharedDrives' from gam.cmd.drive.core import DRIVE_LABEL_CHOICE_MAP # cross-module ref -from gam.util.api import callGAPI, callGAPIitems, callGAPIpages +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( OrderBy, StartEndTime, diff --git a/src/gam/cmd/drive/files.py b/src/gam/cmd/drive/files.py index 80f39fff..573491a5 100644 --- a/src/gam/cmd/drive/files.py +++ b/src/gam/cmd/drive/files.py @@ -15,7 +15,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIitems, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( getAddCSVData, getArgument, diff --git a/src/gam/cmd/drive/filetree.py b/src/gam/cmd/drive/filetree.py index c6200186..6dd11732 100644 --- a/src/gam/cmd/drive/filetree.py +++ b/src/gam/cmd/drive/filetree.py @@ -60,7 +60,7 @@ from gam.cmd.drive.core import ( getEscapedDriveFileName, initDriveFileEntity, ) -from gam.util.api import callGAPI, callGAPIpages +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( StartEndTime, checkArgumentPresent, diff --git a/src/gam/cmd/drive/labels.py b/src/gam/cmd/drive/labels.py index 64ef7dc7..7540e177 100644 --- a/src/gam/cmd/drive/labels.py +++ b/src/gam/cmd/drive/labels.py @@ -14,7 +14,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import _getAdminEmail, buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.api import _getAdminEmail +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( BCP47_LANGUAGE_CODES_MAP, getArgument, diff --git a/src/gam/cmd/drive/looker.py b/src/gam/cmd/drive/looker.py index f06f1b96..290a0ad3 100644 --- a/src/gam/cmd/drive/looker.py +++ b/src/gam/cmd/drive/looker.py @@ -13,7 +13,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, getArgument, diff --git a/src/gam/cmd/drive/permissions.py b/src/gam/cmd/drive/permissions.py index f50762c3..5962edeb 100644 --- a/src/gam/cmd/drive/permissions.py +++ b/src/gam/cmd/drive/permissions.py @@ -18,14 +18,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import ( - _getAdminEmail, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, - checkGAPIError, - waitOnFailure, -) +from gam.util.api import _getAdminEmail, waitOnFailure +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( OrderBy, checkArgumentPresent, diff --git a/src/gam/cmd/drive/revisions.py b/src/gam/cmd/drive/revisions.py index f3ae2a34..11df4e80 100644 --- a/src/gam/cmd/drive/revisions.py +++ b/src/gam/cmd/drive/revisions.py @@ -13,7 +13,7 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import callGAPI, callGAPIpages +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, StartEndTime, diff --git a/src/gam/cmd/drive/shareddrives.py b/src/gam/cmd/drive/shareddrives.py index d068d8bf..c20b347c 100644 --- a/src/gam/cmd/drive/shareddrives.py +++ b/src/gam/cmd/drive/shareddrives.py @@ -24,14 +24,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import ( - _getAdminEmail, - _getValueFromOAuth, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, -) +from gam.util.api import _getAdminEmail, _getValueFromOAuth, buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkArgumentPresent, checkForExtraneousArguments, diff --git a/src/gam/cmd/drive/transfer/fileops.py b/src/gam/cmd/drive/transfer/fileops.py index 616d0402..0587e61e 100644 --- a/src/gam/cmd/drive/transfer/fileops.py +++ b/src/gam/cmd/drive/transfer/fileops.py @@ -23,7 +23,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIitems, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( OrderBy, UTF8, diff --git a/src/gam/cmd/drive/transfer/ownership.py b/src/gam/cmd/drive/transfer/ownership.py index abca77b2..7bb766b1 100644 --- a/src/gam/cmd/drive/transfer/ownership.py +++ b/src/gam/cmd/drive/transfer/ownership.py @@ -16,7 +16,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( OrderBy, formatFileSize, diff --git a/src/gam/cmd/gmail/cse.py b/src/gam/cmd/gmail/cse.py index 21841361..d6ff651b 100644 --- a/src/gam/cmd/gmail/cse.py +++ b/src/gam/cmd/gmail/cse.py @@ -14,7 +14,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( UTF8, getArgument, diff --git a/src/gam/cmd/gmail/delegates.py b/src/gam/cmd/gmail/delegates.py index 41e77dc5..34579ef6 100644 --- a/src/gam/cmd/gmail/delegates.py +++ b/src/gam/cmd/gmail/delegates.py @@ -11,7 +11,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI from gam.util.args import checkArgumentPresent, checkForExtraneousArguments, getArgument from gam.util.csv_pf import CSVPrintFile from gam.util.display import ( diff --git a/src/gam/cmd/gmail/filters.py b/src/gam/cmd/gmail/filters.py index f4bbdbf2..3fa29a83 100644 --- a/src/gam/cmd/gmail/filters.py +++ b/src/gam/cmd/gmail/filters.py @@ -15,7 +15,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIitems +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import ( ONE_KILO_10_BYTES, ONE_MEGA_10_BYTES, diff --git a/src/gam/cmd/gmail/forms.py b/src/gam/cmd/gmail/forms.py index 1bfb6568..13504e55 100644 --- a/src/gam/cmd/gmail/forms.py +++ b/src/gam/cmd/gmail/forms.py @@ -12,7 +12,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( getAddCSVData, getArgument, diff --git a/src/gam/cmd/gmail/labels.py b/src/gam/cmd/gmail/labels.py index 6bb45790..c607d6a1 100644 --- a/src/gam/cmd/gmail/labels.py +++ b/src/gam/cmd/gmail/labels.py @@ -13,7 +13,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages, checkGAPIError +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( LABEL_BACKGROUND_COLORS, LABEL_TEXT_COLORS, diff --git a/src/gam/cmd/gmail/messages.py b/src/gam/cmd/gmail/messages.py index 21701960..440c7109 100644 --- a/src/gam/cmd/gmail/messages.py +++ b/src/gam/cmd/gmail/messages.py @@ -23,14 +23,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import ( - _getAdminEmail, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, - checkGAPIError, -) +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( SORF_MSG_FILE_ARGUMENTS, UTF8, diff --git a/src/gam/cmd/gmail/profile.py b/src/gam/cmd/gmail/profile.py index b888893b..c07cbb06 100644 --- a/src/gam/cmd/gmail/profile.py +++ b/src/gam/cmd/gmail/profile.py @@ -15,7 +15,9 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import getArgument, getInteger from gam.util.csv_pf import CSVPrintFile, getTodriveOnly from gam.util.display import ( diff --git a/src/gam/cmd/gmail/settings.py b/src/gam/cmd/gmail/settings.py index df6408a0..16d7cfe6 100644 --- a/src/gam/cmd/gmail/settings.py +++ b/src/gam/cmd/gmail/settings.py @@ -13,7 +13,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIitems +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import ( LANGUAGE_CODES_MAP, SORF_SIG_FILE_ARGUMENTS, diff --git a/src/gam/cmd/gmail/signature.py b/src/gam/cmd/gmail/signature.py index 772ace77..a8e549c4 100644 --- a/src/gam/cmd/gmail/signature.py +++ b/src/gam/cmd/gmail/signature.py @@ -13,7 +13,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI from gam.util.args import ( FALSE_VALUES, SORF_MSG_FILE_ARGUMENTS, diff --git a/src/gam/cmd/gmail/smime.py b/src/gam/cmd/gmail/smime.py index 7aa58744..60535fb8 100644 --- a/src/gam/cmd/gmail/smime.py +++ b/src/gam/cmd/gmail/smime.py @@ -14,7 +14,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIitems +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import UTF8, getArgument, getEmailAddress, getString from gam.util.csv_pf import CSVPrintFile, flattenJSON from gam.util.display import ( diff --git a/src/gam/cmd/groups/groups.py b/src/gam/cmd/groups/groups.py index 456955c2..3e86f994 100644 --- a/src/gam/cmd/groups/groups.py +++ b/src/gam/cmd/groups/groups.py @@ -17,7 +17,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages, checkGAPIError +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( ARCHIVED_ARGUMENTS, DELIVERY_SETTINGS_UNDEFINED, diff --git a/src/gam/cmd/groups/members.py b/src/gam/cmd/groups/members.py index 67823157..c236f352 100644 --- a/src/gam/cmd/groups/members.py +++ b/src/gam/cmd/groups/members.py @@ -26,16 +26,10 @@ from gam.var import Act, Cmd, Ent, Ind from gam.cmd.groups.groups import ALL_GROUP_MEMBER_TYPES, MEMBEROPTION_DISPLAYMATCH, MEMBEROPTION_GETDELIVERYSETTINGS, MEMBEROPTION_INCLUDEDERIVEDMEMBERSHIP, MEMBEROPTION_ISARCHIVED, MEMBEROPTION_ISSUSPENDED, MEMBEROPTION_MATCHPATTERN, MEMBEROPTION_MEMBERNAMES, MEMBEROPTION_NODUPLICATES, MEMBEROPTION_RECURSIVE, getGroupMemberTypes, GroupIsAbuseOrPostmaster, mapGroupEmailForSettings from gam.util.access import entityUnknownWarning -from gam.util.api import ( - _finalizeGAPIpagesResult, - _processGAPIpagesResult, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, - checkGAPIError, - waitOnFailure, -) +from gam.util.api import buildGAPIObject, waitOnFailure +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import _finalizeGAPIpagesResult, _processGAPIpagesResult +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( ARCHIVED_ARGUMENTS, SUSPENDED_ARGUMENTS, diff --git a/src/gam/cmd/licenses.py b/src/gam/cmd/licenses.py index c2c9d538..9e65c2a8 100644 --- a/src/gam/cmd/licenses.py +++ b/src/gam/cmd/licenses.py @@ -10,7 +10,8 @@ from gamlib import msgs as Msg from gamlib import skus as SKU from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPIpages from gam.util.args import getArgument, getGoogleProductList, getGoogleSKUList, getInteger from gam.util.csv_pf import CSVPrintFile, getItemFieldsFromFieldsList from gam.util.display import entityActionNotPerformedWarning, getPageMessageForWhom, printEntityKVList diff --git a/src/gam/cmd/meet.py b/src/gam/cmd/meet.py index e07256e8..4111f416 100644 --- a/src/gam/cmd/meet.py +++ b/src/gam/cmd/meet.py @@ -8,7 +8,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( getArgument, getBoolean, diff --git a/src/gam/cmd/mobile.py b/src/gam/cmd/mobile.py index c7a475df..76ed449e 100644 --- a/src/gam/cmd/mobile.py +++ b/src/gam/cmd/mobile.py @@ -8,7 +8,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages, yieldGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages, yieldGAPIpages from gam.util.args import ( UTF8, checkArgumentPresent, diff --git a/src/gam/cmd/notes.py b/src/gam/cmd/notes.py index 20e59761..74585e07 100644 --- a/src/gam/cmd/notes.py +++ b/src/gam/cmd/notes.py @@ -12,7 +12,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( SORF_TEXT_ARGUMENTS, checkForExtraneousArguments, diff --git a/src/gam/cmd/oauth.py b/src/gam/cmd/oauth.py index 2855e32d..2a58ae15 100644 --- a/src/gam/cmd/oauth.py +++ b/src/gam/cmd/oauth.py @@ -31,13 +31,13 @@ from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.api import ( buildGAPIObject, - callGAPI, getClientCredentials, getHttpObj, getOauth2TxtCredentials, shortenURL, writeClientCredentials, ) +from gam.util.api_call import callGAPI from gam.util.args import ( UTF8, YYYYMMDDTHHMMSSZ_FORMAT, diff --git a/src/gam/cmd/orgunits.py b/src/gam/cmd/orgunits.py index 16cadcdf..8c71da87 100644 --- a/src/gam/cmd/orgunits.py +++ b/src/gam/cmd/orgunits.py @@ -12,16 +12,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit -from gam.util.api import ( - _finalizeGAPIpagesResult, - _processGAPIpagesResult, - buildGAPIObject, - callGAPI, - callGAPIpages, - checkGAPIError, - waitOnFailure, - yieldGAPIpages, -) +from gam.util.api import buildGAPIObject, waitOnFailure +from gam.util.api_call import _finalizeGAPIpagesResult, _processGAPIpagesResult +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError, yieldGAPIpages from gam.util.args import ( ARCHIVED_ARGUMENTS, SUSPENDED_ARGUMENTS, diff --git a/src/gam/cmd/people.py b/src/gam/cmd/people.py index 97aa8690..884e5543 100644 --- a/src/gam/cmd/people.py +++ b/src/gam/cmd/people.py @@ -72,14 +72,10 @@ from gam.cmd.contacts import ( PEOPLE_UPDATE_TIME, PEOPLE_URLS, PEOPLE_USER_DEFINED, ) from gam.util.access import entityUnknownWarning -from gam.util.api import ( - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, - getHttpObj, - writeGotMessage, -) +from gam.util.api import buildGAPIObject, getHttpObj +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import writeGotMessage +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( SORTORDER_CHOICE_MAP, UID_PATTERN, diff --git a/src/gam/cmd/printers.py b/src/gam/cmd/printers.py index a5aee38f..2135d9b1 100644 --- a/src/gam/cmd/printers.py +++ b/src/gam/cmd/printers.py @@ -10,7 +10,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( getArgument, getBoolean, diff --git a/src/gam/cmd/project.py b/src/gam/cmd/project.py index aba4223d..68ee820b 100644 --- a/src/gam/cmd/project.py +++ b/src/gam/cmd/project.py @@ -33,10 +33,6 @@ from gam.util.api import ( _getSvcAcctData, buildGAPIObject, buildGAPIObjectNoAuthentication, - buildGAPIServiceObject, - callGAPI, - callGAPIitems, - callGAPIpages, getAPIService, getHttpObj, getSvcAcctCredentials, @@ -46,6 +42,8 @@ from gam.util.api import ( transportAuthorizedHttp, transportCreateRequest, ) +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( UTF8, checkArgumentPresent, diff --git a/src/gam/cmd/reports.py b/src/gam/cmd/reports.py index dc3211a3..b4c418d2 100644 --- a/src/gam/cmd/reports.py +++ b/src/gam/cmd/reports.py @@ -12,7 +12,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import accessErrorExit, entityUnknownWarning -from gam.util.api import _getAdminEmail, buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( DELTA_DATE_FORMAT_REQUIRED, DELTA_DATE_PATTERN, diff --git a/src/gam/cmd/reseller.py b/src/gam/cmd/reseller.py index fd0eeeea..78e6f832 100644 --- a/src/gam/cmd/reseller.py +++ b/src/gam/cmd/reseller.py @@ -12,7 +12,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gamlib import skus as SKU from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( LANGUAGE_CODES_MAP, checkForExtraneousArguments, diff --git a/src/gam/cmd/resources.py b/src/gam/cmd/resources.py index 3f2c4c7f..16479c5e 100644 --- a/src/gam/cmd/resources.py +++ b/src/gam/cmd/resources.py @@ -10,7 +10,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit, entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( LANGUAGE_CODES_MAP, UID_PATTERN, diff --git a/src/gam/cmd/schemas.py b/src/gam/cmd/schemas.py index 63454e94..139954c8 100644 --- a/src/gam/cmd/schemas.py +++ b/src/gam/cmd/schemas.py @@ -8,7 +8,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit -from gam.util.api import buildGAPIObject, callGAPI +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI from gam.util.args import ( checkForExtraneousArguments, getArgument, diff --git a/src/gam/cmd/send_email.py b/src/gam/cmd/send_email.py index 058266d7..34f0ec11 100644 --- a/src/gam/cmd/send_email.py +++ b/src/gam/cmd/send_email.py @@ -11,13 +11,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import entityUnknownWarning -from gam.util.api import ( - _getAdminEmail, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, -) +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( FALSE, SORF_MSG_FILE_ARGUMENTS, diff --git a/src/gam/cmd/sites.py b/src/gam/cmd/sites.py index 0ba07400..a7044eba 100644 --- a/src/gam/cmd/sites.py +++ b/src/gam/cmd/sites.py @@ -9,14 +9,9 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import ( - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIitems, - callGAPIpages, - getHttpObj, -) +from gam.util.api import buildGAPIObject, getHttpObj +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( UTF8, checkForExtraneousArguments, diff --git a/src/gam/cmd/sso.py b/src/gam/cmd/sso.py index 25247ff5..57978e92 100644 --- a/src/gam/cmd/sso.py +++ b/src/gam/cmd/sso.py @@ -9,7 +9,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkForExtraneousArguments, getArgument, diff --git a/src/gam/cmd/tasks.py b/src/gam/cmd/tasks.py index a79d1497..7450a4ef 100644 --- a/src/gam/cmd/tasks.py +++ b/src/gam/cmd/tasks.py @@ -9,7 +9,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIpages +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkArgumentPresent, checkForExtraneousArguments, diff --git a/src/gam/cmd/userop/licenses.py b/src/gam/cmd/userop/licenses.py index ff1cf321..0626f233 100644 --- a/src/gam/cmd/userop/licenses.py +++ b/src/gam/cmd/userop/licenses.py @@ -18,7 +18,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gamlib import skus as SKU from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI from gam.util.args import ( checkArgumentPresent, getArgument, diff --git a/src/gam/cmd/userop/photos.py b/src/gam/cmd/userop/photos.py index da6643ff..298d35e7 100644 --- a/src/gam/cmd/userop/photos.py +++ b/src/gam/cmd/userop/photos.py @@ -19,7 +19,9 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI, getHttpObj +from gam.util.api import buildGAPIObject, getHttpObj +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI from gam.util.args import ( UTF8, checkForExtraneousArguments, diff --git a/src/gam/cmd/userop/sheets.py b/src/gam/cmd/userop/sheets.py index 6768e348..f5fc1d36 100644 --- a/src/gam/cmd/userop/sheets.py +++ b/src/gam/cmd/userop/sheets.py @@ -12,7 +12,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIServiceObject, callGAPI, callGAPIitems +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import getArgument, getBoolean, getJSON, getString from gam.util.csv_pf import ( CSVPrintFile, diff --git a/src/gam/cmd/userop/tokens.py b/src/gam/cmd/userop/tokens.py index 26f87934..401bf702 100644 --- a/src/gam/cmd/userop/tokens.py +++ b/src/gam/cmd/userop/tokens.py @@ -14,7 +14,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import ClientAPIAccessDeniedExit, entityUnknownWarning -from gam.util.api import _getAdminEmail, buildGAPIObject, callGAPI, callGAPIitems +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIitems from gam.util.args import ( checkArgumentPresent, checkForExtraneousArguments, diff --git a/src/gam/cmd/userop/usergroups.py b/src/gam/cmd/userop/usergroups.py index 7b7332a4..e5377bec 100644 --- a/src/gam/cmd/userop/usergroups.py +++ b/src/gam/cmd/userop/usergroups.py @@ -24,7 +24,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkArgumentPresent, checkForExtraneousArguments, diff --git a/src/gam/cmd/users/display.py b/src/gam/cmd/users/display.py index fc8d1197..5a23d247 100644 --- a/src/gam/cmd/users/display.py +++ b/src/gam/cmd/users/display.py @@ -65,15 +65,8 @@ from gam.cmd.users.manage import ( # cross-module refs _initSchemaParms, ) from gam.util.access import accessErrorExit, checkEntityDNEorAccessErrorExit, entityUnknownWarning -from gam.util.api import ( - _getAdminEmail, - buildGAPIObject, - callGAPI, - callGAPIpages, - checkGAPIError, - waitOnFailure, - yieldGAPIpages, -) +from gam.util.api import _getAdminEmail, buildGAPIObject, waitOnFailure +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError, yieldGAPIpages from gam.util.args import ( formatLocalTime, getAddCSVData, diff --git a/src/gam/cmd/users/manage.py b/src/gam/cmd/users/manage.py index 0fa5aafa..e22460c2 100644 --- a/src/gam/cmd/users/manage.py +++ b/src/gam/cmd/users/manage.py @@ -22,7 +22,8 @@ from gamlib import state as GM from gamlib import msgs as Msg from gamlib import skus as SKU from gam.util.access import accessErrorExit, duplicateAliasGroupUserWarning, entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages, checkGAPIError +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages, checkGAPIError from gam.util.args import ( LANGUAGE_CODES_MAP, SORF_MSG_FILE_ARGUMENTS, diff --git a/src/gam/cmd/userservices.py b/src/gam/cmd/userservices.py index c5ee20e4..d954dc28 100644 --- a/src/gam/cmd/userservices.py +++ b/src/gam/cmd/userservices.py @@ -26,13 +26,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Act, Cmd, Ent, Ind from gam.util.access import checkEntityAFDNEorAccessErrorExit, entityUnknownWarning -from gam.util.api import ( - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIitems, - callGAPIpages, -) +from gam.util.api import buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIitems, callGAPIpages from gam.util.args import ( BCP47_LANGUAGE_CODES_MAP, CALENDAR_COLOR_MAP, diff --git a/src/gam/cmd/vault/holds.py b/src/gam/cmd/vault/holds.py index a390759d..f1aaf19b 100644 --- a/src/gam/cmd/vault/holds.py +++ b/src/gam/cmd/vault/holds.py @@ -42,13 +42,9 @@ from gam.cmd.vault.matters import ( getMatterItem, warnMatterNotOpen, ) -from gam.util.api import ( - _getAdminEmail, - buildGAPIObject, - buildGAPIServiceObject, - callGAPI, - callGAPIpages, -) +from gam.util.api import _getAdminEmail, buildGAPIObject +from gam.util.svcacct import buildGAPIServiceObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( checkForExtraneousArguments, getArgument, diff --git a/src/gam/cmd/vault/matters.py b/src/gam/cmd/vault/matters.py index 518503a9..6b97cfbf 100644 --- a/src/gam/cmd/vault/matters.py +++ b/src/gam/cmd/vault/matters.py @@ -16,7 +16,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg -from gam.util.api import buildGAPIObject, callGAPI, callGAPIpages +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI, callGAPIpages from gam.util.args import ( UID_PATTERN, getArgument, diff --git a/src/gam/util/access.py b/src/gam/util/access.py index 31a4cfa2..f2c9f661 100644 --- a/src/gam/util/access.py +++ b/src/gam/util/access.py @@ -15,7 +15,8 @@ from gamlib import state as GM from gamlib import indent from gamlib import msgs as Msg from gam.constants import API_ACCESS_DENIED_RC, INVALID_DOMAIN_RC -from util.api import _getAdminEmail, _getSvcAcctData, buildGAPIObject, callGAPI, APIAccessDeniedExit, ClientAPIAccessDeniedExit, SvcAcctAPIAccessDeniedExit, SvcAcctAPIDisabledExit +from util.api import _getAdminEmail, _getSvcAcctData, buildGAPIObject, APIAccessDeniedExit, ClientAPIAccessDeniedExit, SvcAcctAPIAccessDeniedExit, SvcAcctAPIDisabledExit +from util.api_call import callGAPI from util.args import getEmailAddressDomain, getPhraseDNEorSNA from util.display import ENTITY_DOES_NOT_EXIST_RC, ENTITY_DUPLICATE_RC, entityActionFailedWarning, entityDoesNotExistWarning, entityServiceNotApplicableWarning from util.errors import OAUTH2SERVICE_JSON_REQUIRED_RC diff --git a/src/gam/util/api.py b/src/gam/util/api.py index 7c1956d8..9b44aaae 100644 --- a/src/gam/util/api.py +++ b/src/gam/util/api.py @@ -55,6 +55,7 @@ from util.errors import INVALID_JSON_RC, OAUTH2SERVICE_JSON_REQUIRED_RC, OAUTH2_ from util.fileio import FILE_ERROR_RC, UNKNOWN, checkAPICallsRate, incrAPICallsRetryData, readFile, writeFile from util.output import ERROR_PREFIX, flushStderr, setSysExitRC, stderrErrorMsg, systemErrorExit, writeStderr, writeStdout + HTML_TITLE_PATTERN = re.compile(r'.*(.+)') from gam.constants import GAM_LATEST_RELEASE, GAM_USER_AGENT, __author__, __version__ @@ -729,649 +730,6 @@ def getGDataOAuthToken(gdataObj, credentials=None): gdataObj.source = GAM_USER_AGENT return True -def checkGDataError(e, service): - error = e.args - reason = error[0].get('reason', '') - body = error[0].get('body', '').decode(UTF8) - # First check for errors that need special handling - if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']: - keep_domain = service.domain - getGDataOAuthToken(service) - service.domain = keep_domain - return (GDATA.TOKEN_EXPIRED, reason) - error_code = getattr(e, 'error_code', 600) - if GC.Values[GC.DEBUG_LEVEL] > 0: - writeStdout(f'{ERROR_PREFIX} {error_code}: {reason}, {body}\n') - if error_code == 600: - if (body.startswith('Quota exceeded for the current request') or - body.startswith('Quota exceeded for quota metric') or - body.startswith('Request rate higher than configured')): - return (GDATA.QUOTA_EXCEEDED, body) - if (body.startswith('Photo delete failed') or - body.startswith('Upload photo failed') or - body.startswith('Photo query failed')): - return (GDATA.NOT_FOUND, body) - if body.startswith(GDATA.API_DEPRECATED_MSG): - return (GDATA.API_DEPRECATED, body) - if reason == 'Too Many Requests': - return (GDATA.QUOTA_EXCEEDED, reason) - if reason == 'Bad Gateway': - return (GDATA.BAD_GATEWAY, reason) - if reason == 'Gateway Timeout': - return (GDATA.GATEWAY_TIMEOUT, reason) - if reason == 'Service Unavailable': - return (GDATA.SERVICE_UNAVAILABLE, reason) - if reason == 'Service disabled by G Suite admin.': - return (GDATA.FORBIDDEN, reason) - if reason == 'Internal Server Error': - return (GDATA.INTERNAL_SERVER_ERROR, reason) - if reason == 'Token invalid - Invalid token: Token disabled, revoked, or expired.': - return (GDATA.TOKEN_INVALID, 'Token disabled, revoked, or expired. Please delete and re-create oauth.txt') - if reason == 'Token invalid - AuthSub token has wrong scope': - return (GDATA.INSUFFICIENT_PERMISSIONS, reason) - if reason.startswith('Only administrators can request entries belonging to'): - return (GDATA.INSUFFICIENT_PERMISSIONS, reason) - if reason == 'You are not authorized to access this API': - return (GDATA.INSUFFICIENT_PERMISSIONS, reason) - if reason == 'Invalid domain.': - return (GDATA.INVALID_DOMAIN, reason) - if reason.startswith('You are not authorized to perform operations on the domain'): - return (GDATA.INVALID_DOMAIN, reason) - if reason == 'Bad Request': - if 'already exists' in body: - return (GDATA.ENTITY_EXISTS, Msg.DUPLICATE) - return (GDATA.BAD_REQUEST, body) - if reason == 'Forbidden': - return (GDATA.FORBIDDEN, body) - if reason == 'Not Found': - return (GDATA.NOT_FOUND, Msg.DOES_NOT_EXIST) - if reason == 'Not Implemented': - return (GDATA.NOT_IMPLEMENTED, body) - if reason == 'Precondition Failed': - return (GDATA.PRECONDITION_FAILED, reason) - elif error_code == 602: - if body.startswith(GDATA.API_DEPRECATED_MSG): - return (GDATA.API_DEPRECATED, body) - if reason == 'Bad Request': - return (GDATA.BAD_REQUEST, body) - elif error_code == 610: - if reason == 'Service disabled by G Suite admin.': - return (GDATA.FORBIDDEN, reason) - - # We got a "normal" error, define the mapping below - error_code_map = { - 1000: reason, - 1001: reason, - 1002: 'Unauthorized and forbidden', - 1100: 'User deleted recently', - 1200: 'Domain user limit exceeded', - 1201: 'Domain alias limit exceeded', - 1202: 'Domain suspended', - 1203: 'Domain feature unavailable', - 1300: f'Entity {getattr(e, "invalidInput", "")} exists', - 1301: f'Entity {getattr(e, "invalidInput", "")} Does Not Exist', - 1302: 'Entity Name Is Reserved', - 1303: f'Entity {getattr(e, "invalidInput", "")} name not valid', - 1306: f'{getattr(e, "invalidInput", "")} has members. Cannot delete.', - 1317: f'Invalid input {getattr(e, "invalidInput", "")}, reason {getattr(e, "reason", "")}', - 1400: 'Invalid Given Name', - 1401: 'Invalid Family Name', - 1402: 'Invalid Password', - 1403: 'Invalid Username', - 1404: 'Invalid Hash Function Name', - 1405: 'Invalid Hash Digest Length', - 1406: 'Invalid Email Address', - 1407: 'Invalid Query Parameter Value', - 1408: 'Invalid SSO Signing Key', - 1409: 'Invalid Encryption Public Key', - 1410: 'Feature Unavailable For User', - 1411: 'Invalid Encryption Public Key Format', - 1500: 'Too Many Recipients On Email List', - 1501: 'Too Many Aliases For User', - 1502: 'Too Many Delegates For User', - 1601: 'Duplicate Destinations', - 1602: 'Too Many Destinations', - 1603: 'Invalid Route Address', - 1700: 'Group Cannot Contain Cycle', - 1800: 'Group Cannot Contain Cycle', - 1801: f'Invalid value {getattr(e, "invalidInput", "")}', - } - return (error_code, error_code_map.get(error_code, f'Unknown Error: {str(e)}')) - -def callGData(service, function, - bailOnInternalServerError=False, softErrors=False, - throwErrors=None, retryErrors=None, triesLimit=0, - **kwargs): - if throwErrors is None: - throwErrors = [] - if retryErrors is None: - retryErrors = [] - if triesLimit == 0: - triesLimit = GC.Values[GC.API_CALLS_TRIES_LIMIT] - allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors - method = getattr(service, function) - if GC.Values[GC.API_CALLS_RATE_CHECK]: - checkAPICallsRate() - for n in range(1, triesLimit+1): - try: - return method(**kwargs) - except (gdata.service.RequestError, gdata.apps.service.AppsForYourDomainException) as e: - error_code, error_message = checkGDataError(e, service) - if (n != triesLimit) and (error_code in allRetryErrors): - if (error_code == GDATA.INTERNAL_SERVER_ERROR and - bailOnInternalServerError and n == GC.Values[GC.BAIL_ON_INTERNAL_ERROR_TRIES]): - raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message) - waitOnFailure(n, triesLimit, error_code, error_message) - continue - if error_code in throwErrors: - if error_code in GDATA.ERROR_CODE_EXCEPTION_MAP: - raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message) - raise - if softErrors: - stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}') - return None - if error_code == GDATA.INSUFFICIENT_PERMISSIONS: - APIAccessDeniedExit() - systemErrorExit(GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}') - except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: - if n != triesLimit: - waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e)) - continue - handleServerError(e) - except google.auth.exceptions.RefreshError as e: - if isinstance(e.args, tuple): - e = e.args[0] - handleOAuthTokenError(e, GDATA.SERVICE_NOT_APPLICABLE in throwErrors) - raise GDATA.ERROR_CODE_EXCEPTION_MAP[GDATA.SERVICE_NOT_APPLICABLE](str(e)) - except (http.client.ResponseNotReady, OSError) as e: - errMsg = f'Connection error: {str(e) or repr(e)}' - if n != triesLimit: - waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg) - continue - if softErrors: - writeStderr(f'\n{ERROR_PREFIX}{errMsg} - Giving up.\n') - return None - systemErrorExit(SOCKET_ERROR_RC, errMsg) - -def writeGotMessage(msg): - if GC.Values[GC.SHOW_GETTINGS_GOT_NL]: - writeStderr(msg) - else: - writeStderr('\r') - msgLen = len(msg) - if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]: - writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen)) - else: - writeStderr(msg) - GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen - flushStderr() - -def callGDataPages(service, function, - pageMessage=None, - softErrors=False, throwErrors=None, retryErrors=None, - uri=None, - **kwargs): - if throwErrors is None: - throwErrors = [] - if retryErrors is None: - retryErrors = [] - nextLink = None - allResults = [] - totalItems = 0 - while True: - this_page = callGData(service, function, - softErrors=softErrors, throwErrors=throwErrors, retryErrors=retryErrors, - uri=uri, - **kwargs) - if this_page: - nextLink = this_page.GetNextLink() - pageItems = len(this_page.entry) - if pageItems == 0: - nextLink = None - totalItems += pageItems - allResults.extend(this_page.entry) - else: - nextLink = None - pageItems = 0 - if pageMessage: - show_message = pageMessage.replace(TOTAL_ITEMS_MARKER, str(totalItems)) - writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems))) - if nextLink is None: - if pageMessage and (pageMessage[-1] != '\n'): - writeStderr('\r\n') - flushStderr() - return allResults - uri = nextLink.href - if 'url_params' in kwargs: - kwargs['url_params'].pop('start-index', None) - -def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True): - def makeErrorDict(code, reason, message): - return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}} - - try: - error = json.loads(e.content.decode(UTF8)) - if GC.Values[GC.DEBUG_LEVEL] > 0: - writeStdout(f'{ERROR_PREFIX} JSON: {str(error)}\n') - except (IndexError, KeyError, SyntaxError, TypeError, ValueError): - eContent = e.content.decode(UTF8) if isinstance(e.content, bytes) else e.content - lContent = eContent.lower() - if GC.Values[GC.DEBUG_LEVEL] > 0: - writeStdout(f'{ERROR_PREFIX} HTTP: {str(eContent)}\n') - if eContent[0:15] != '': - if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')): - return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) - if (e.resp['status'] == '429') and (lContent.startswith('quota exceeded for quota metric')): - return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) - if (e.resp['status'] == '502') and ('bad gateway' in lContent): - return (e.resp['status'], GAPI.BAD_GATEWAY, eContent) - if (e.resp['status'] == '503') and (lContent.startswith('quota exceeded for the current request')): - return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) - if (e.resp['status'] == '504') and ('gateway timeout' in lContent): - return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent) - else: - tg = HTML_TITLE_PATTERN.match(lContent) - lContent = tg.group(1) if tg else 'bad request' - if (e.resp['status'] == '403') and ('invalid domain.' in lContent): - error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found') - elif (e.resp['status'] == '403') and ('domain cannot use apis.' in lContent): - error = makeErrorDict(403, GAPI.DOMAIN_CANNOT_USE_APIS, 'Domain cannot use apis') - elif (e.resp['status'] == '400') and ('invalidssosigningkey' in lContent): - error = makeErrorDict(400, GAPI.INVALID, 'InvalidSsoSigningKey') - elif (e.resp['status'] == '400') and ('unknownerror' in lContent): - error = makeErrorDict(400, GAPI.INVALID, 'UnknownError') - elif (e.resp['status'] == '400') and ('featureunavailableforuser' in lContent): - error = makeErrorDict(400, GAPI.SERVICE_NOT_AVAILABLE, 'Feature Unavailable For User') - elif (e.resp['status'] == '400') and ('entitydoesnotexist' in lContent): - error = makeErrorDict(400, GAPI.NOT_FOUND, 'Entity Does Not Exist') - elif (e.resp['status'] == '400') and ('entitynamenotvalid' in lContent): - error = makeErrorDict(400, GAPI.INVALID_INPUT, 'Entity Name Not Valid') - elif (e.resp['status'] == '400') and ('failed to parse Content-Range header' in lContent): - error = makeErrorDict(400, GAPI.BAD_REQUEST, 'Failed to parse Content-Range header') - elif (e.resp['status'] == '400') and ('request contains an invalid argument' in lContent): - error = makeErrorDict(400, GAPI.INVALID_ARGUMENT, 'Request contains an invalid argument') - elif (e.resp['status'] == '404') and ('not found' in lContent): - error = makeErrorDict(404, GAPI.NOT_FOUND, lContent) - elif (e.resp['status'] == '404') and ('bad request' in lContent): - error = makeErrorDict(404, GAPI.BAD_REQUEST, lContent) - elif retryOnHttpError: - return (-1, None, eContent) - elif softErrors: - stderrErrorMsg(eContent) - return (0, None, None) - else: - systemErrorExit(HTTP_ERROR_RC, eContent) - requiredScopes = '' - wwwAuthenticate = e.resp.get('www-authenticate', '') - if 'insufficient_scope' in wwwAuthenticate: - mg = re.match(r'.+scope="(.+)"', wwwAuthenticate) - if mg: - requiredScopes = mg.group(1).split(' ') - if 'error' in error: - http_status = error['error']['code'] - reason = '' - if 'errors' in error['error'] and 'message' in error['error']['errors'][0]: - message = error['error']['errors'][0]['message'] - if 'reason' in error['error']['errors'][0]: - reason = error['error']['errors'][0]['reason'] - elif 'errors' in error['error'] and 'Unknown Error' in error['error']['message'] and 'reason' in error['error']['errors'][0]: - message = error['error']['errors'][0]['reason'] - else: - message = error['error']['message'] - status = error['error'].get('status', '') - lmessage = message.lower() if message is not None else '' - if http_status == 500: - if not lmessage or status == 'UNKNOWN': - if not lmessage: - message = Msg.UNKNOWN - error = makeErrorDict(http_status, GAPI.UNKNOWN_ERROR, message) - elif 'backend error' in lmessage: - error = makeErrorDict(http_status, GAPI.BACKEND_ERROR, message) - elif 'internal error encountered' in lmessage: - error = makeErrorDict(http_status, GAPI.INTERNAL_ERROR, message) - elif 'role assignment exists: roleassignment' in lmessage: - error = makeErrorDict(http_status, GAPI.DUPLICATE, message) - elif 'role assignment exists: roleid' in lmessage: - error = makeErrorDict(http_status, GAPI.DUPLICATE, message) - elif 'operation not supported' in lmessage: - error = makeErrorDict(http_status, GAPI.OPERATION_NOT_SUPPORTED, message) - elif 'failed status in update settings response' in lmessage: - error = makeErrorDict(http_status, GAPI.INVALID_INPUT, message) - elif 'cannot delete a field in use.resource.fields' in lmessage: - error = makeErrorDict(http_status, GAPI.FIELD_IN_USE, message) - elif status == 'INTERNAL': - error = makeErrorDict(http_status, GAPI.INTERNAL_ERROR, message) - elif http_status == 501: - if status == 'UNIMPLEMENTED': - error = makeErrorDict(http_status, GAPI.UNIMPLEMENTED_ERROR, message) - elif http_status == 502: - if 'bad gateway' in lmessage: - error = makeErrorDict(http_status, GAPI.BAD_GATEWAY, message) - elif http_status == 503: - if message.startswith('quota exceeded for the current request'): - error = makeErrorDict(http_status, GAPI.QUOTA_EXCEEDED, message) - elif status == 'UNAVAILABLE' or 'the service is currently unavailable' in lmessage: - error = makeErrorDict(http_status, GAPI.SERVICE_NOT_AVAILABLE, message) - elif http_status == 504: - if 'gateway timeout' in lmessage: - error = makeErrorDict(http_status, GAPI.GATEWAY_TIMEOUT, message) - elif http_status == 400: - if '@attachmentnotvisible' in lmessage: - error = makeErrorDict(http_status, GAPI.BAD_REQUEST, message) - elif status == 'INVALID_ARGUMENT': - error = makeErrorDict(http_status, GAPI.INVALID_ARGUMENT, message) - elif status == 'FAILED_PRECONDITION' or 'precondition check failed' in lmessage: - error = makeErrorDict(http_status, GAPI.FAILED_PRECONDITION, message) - elif 'does not match' in lmessage or 'invalid' in lmessage: - error = makeErrorDict(http_status, GAPI.INVALID, message) - elif http_status == 401: - if 'active session is invalid' in lmessage and reason == 'authError': -# message += ' Drive SDK API access disabled' -# message = Msg.SERVICE_NOT_ENABLED.format('Drive') - error = makeErrorDict(http_status, GAPI.AUTH_ERROR, message) - elif status == 'PERMISSION_DENIED': - error = makeErrorDict(http_status, GAPI.PERMISSION_DENIED, message) - elif status == 'UNAUTHENTICATED': - error = makeErrorDict(http_status, GAPI.AUTH_ERROR, message) - elif http_status == 403: - if 'quota exceeded for quota metric' in lmessage: - error = makeErrorDict(http_status, GAPI.QUOTA_EXCEEDED, message) - elif 'the authenticated user cannot access this service' in lmessage: - error = makeErrorDict(http_status, GAPI.SERVICE_NOT_AVAILABLE, message) - elif status == 'PERMISSION_DENIED' or 'the caller does not have permission' in lmessage or 'permission iam.serviceaccountkeys' in lmessage: - if requiredScopes: - message += f', {Msg.NO_SCOPES_FOR_API.format(API.findAPIforScope(requiredScopes))}' - error = makeErrorDict(http_status, GAPI.PERMISSION_DENIED, message) - elif http_status == 404: - if status == 'NOT_FOUND' or 'requested entity was not found' in lmessage or 'does not exist' in lmessage: - error = makeErrorDict(http_status, GAPI.NOT_FOUND, message) - elif http_status == 409: - if status == 'ALREADY_EXISTS' or 'requested entity already exists' in lmessage: - error = makeErrorDict(http_status, GAPI.ALREADY_EXISTS, message) - elif status == 'ABORTED' or 'the operation was aborted' in lmessage: - error = makeErrorDict(http_status, GAPI.ABORTED, message) - elif http_status == 412: - if 'insufficient archived user licenses' in lmessage: - error = makeErrorDict(http_status, GAPI.INSUFFICIENT_ARCHIVED_USER_LICENSES, message) - elif http_status == 413: - if 'request too large' in lmessage: - error = makeErrorDict(http_status, GAPI.UPLOAD_TOO_LARGE, message) - elif http_status == 429: - if status == 'RESOURCE_EXHAUSTED' or 'quota exceeded' in lmessage or 'insufficient quota' in lmessage: - error = makeErrorDict(http_status, GAPI.QUOTA_EXCEEDED, message) - else: - if 'error_description' in error: - if error['error_description'] == 'Invalid Value': - message = error['error_description'] - http_status = 400 - error = makeErrorDict(http_status, GAPI.INVALID, message) - else: - systemErrorExit(GOOGLE_API_ERROR_RC, str(error)) - else: - systemErrorExit(GOOGLE_API_ERROR_RC, str(error)) - try: - reason = error['error']['errors'][0]['reason'] - for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []): - if messageItem[0] in message: - if reason in [GAPI.NOT_FOUND, GAPI.RESOURCE_NOT_FOUND] and mapNotFound: - message = Msg.DOES_NOT_EXIST - reason = messageItem[1] - break - if reason == GAPI.INVALID_SHARING_REQUEST: - loc = message.find('User message: ') - if loc != -1: - message = message[loc+15:] - else: - loc = message.find('User message: ""') - if loc != -1: - message = message[:loc+14]+f'"{reason}"' - except KeyError: - reason = f'{http_status}' - return (http_status, reason, message) - -def callGAPI(service, function, - bailOnInternalError=False, bailOnTransientError=False, bailOnInvalidError=False, - softErrors=False, mapNotFound=True, - throwReasons=None, retryReasons=None, triesLimit=0, - **kwargs): - if throwReasons is None: - throwReasons = [] - if retryReasons is None: - retryReasons = [] - if triesLimit == 0: - triesLimit = GC.Values[GC.API_CALLS_TRIES_LIMIT] - allRetryReasons = GAPI.DEFAULT_RETRY_REASONS+retryReasons - method = getattr(service, function) - svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST]) - if GC.Values[GC.API_CALLS_RATE_CHECK]: - checkAPICallsRate() - for n in range(1, triesLimit+1): - try: - return method(**svcparms).execute() - except googleapiclient.errors.HttpError as e: - http_status, reason, message = checkGAPIError(e, softErrors=softErrors, retryOnHttpError=n < 3, mapNotFound=mapNotFound) - if http_status == -1: - # The error detail indicated that we should retry this request - # We'll refresh credentials and make another pass - try: -# service._http.credentials.refresh(getHttpObj()) - service._http.credentials.refresh(transportCreateRequest()) - except TypeError: - systemErrorExit(HTTP_ERROR_RC, message) - continue - if http_status == 0: - return None - if (n != triesLimit) and ((reason in allRetryReasons) or - (GC.Values[GC.RETRY_API_SERVICE_NOT_AVAILABLE] and (reason == GAPI.SERVICE_NOT_AVAILABLE))): - if (reason in [GAPI.INTERNAL_ERROR, GAPI.BACKEND_ERROR] and - bailOnInternalError and n == GC.Values[GC.BAIL_ON_INTERNAL_ERROR_TRIES]): - raise GAPI.REASON_EXCEPTION_MAP[reason](message) - if (reason in [GAPI.INVALID] and - bailOnInvalidError and n == GC.Values[GC.BAIL_ON_INTERNAL_ERROR_TRIES]): - raise GAPI.REASON_EXCEPTION_MAP[reason](message) - waitOnFailure(n, triesLimit, reason, message) - if reason == GAPI.TRANSIENT_ERROR and bailOnTransientError: - raise GAPI.REASON_EXCEPTION_MAP[reason](message) - continue - if reason in throwReasons: - if reason in GAPI.REASON_EXCEPTION_MAP: - raise GAPI.REASON_EXCEPTION_MAP[reason](message) - raise e - if softErrors: - stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}') - return None - if reason == GAPI.INSUFFICIENT_PERMISSIONS: - APIAccessDeniedExit() - systemErrorExit(HTTP_ERROR_RC, formatHTTPError(http_status, reason, message)) - except googleapiclient.errors.MediaUploadSizeError as e: - raise e - except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: - if n != triesLimit: - service._http.connections = {} - waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e)) - continue - handleServerError(e) - except google.auth.exceptions.RefreshError as e: - if isinstance(e.args, tuple): - e = e.args[0] - handleOAuthTokenError(e, GAPI.SERVICE_NOT_AVAILABLE in throwReasons) - raise GAPI.REASON_EXCEPTION_MAP[GAPI.SERVICE_NOT_AVAILABLE](str(e)) - except (http.client.ResponseNotReady, OSError) as e: - errMsg = f'Connection error: {str(e) or repr(e)}' - if n != triesLimit: - waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg) - continue - if softErrors: - writeStderr(f'\n{ERROR_PREFIX}{errMsg} - Giving up.\n') - return None - systemErrorExit(SOCKET_ERROR_RC, errMsg) - except ValueError as e: - if clearServiceCache(service): - continue - systemErrorExit(GOOGLE_API_ERROR_RC, str(e)) - except TypeError as e: - systemErrorExit(GOOGLE_API_ERROR_RC, str(e)) - -def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType): - showMessage = pageMessage.replace(TOTAL_ITEMS_MARKER, str(totalItems)) - if pageItems: - if messageAttribute: - firstItem = results[0] if pageItems > 0 else {} - lastItem = results[-1] if pageItems > 1 else firstItem - if isinstance(messageAttribute, str): - firstItem = str(firstItem.get(messageAttribute, '')) - lastItem = str(lastItem.get(messageAttribute, '')) - else: - for attr in messageAttribute: - firstItem = firstItem.get(attr, {}) - lastItem = lastItem.get(attr, {}) - firstItem = str(firstItem) - lastItem = str(lastItem) - showMessage = showMessage.replace(FIRST_ITEM_MARKER, firstItem) - showMessage = showMessage.replace(LAST_ITEM_MARKER, lastItem) - else: - showMessage = showMessage.replace(FIRST_ITEM_MARKER, '') - showMessage = showMessage.replace(LAST_ITEM_MARKER, '') - writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems)))) - -def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType): - if results: - pageToken = results.get('nextPageToken') - if items in results: - pageItems = len(results[items]) - totalItems += pageItems - if allResults is not None: - allResults.extend(results[items]) - else: - results = {items: []} - pageItems = 0 - else: - pageToken = None - results = {items: []} - pageItems = 0 - if pageMessage: - _showGAPIpagesResult(results[items], pageItems, totalItems, pageMessage, messageAttribute, entityType) - return (pageToken, totalItems) - -def _finalizeGAPIpagesResult(pageMessage): - if pageMessage and (pageMessage[-1] != '\n'): - writeStderr('\r\n') - flushStderr() - -def _setMaxArgResults(maxItems, pageArgsInBody, kwargs): - if pageArgsInBody: - kwargs.setdefault('body', {}) - maxArg = '' - maxResults = 0 - if maxItems: - if not pageArgsInBody: - maxResults = kwargs.get('maxResults', 0) - if maxResults: - maxArg = 'maxResults' - else: - maxResults = kwargs.get('pageSize', 0) - if maxResults: - maxArg = 'pageSize' - else: - maxResults = kwargs['body'].get('maxResults', 0) - if maxResults: - maxArg = 'maxResults' - else: - maxResults = kwargs['body'].get('pageSize', 0) - if maxResults: - maxArg = 'pageSize' - return (maxArg, maxResults) - -def callGAPIpages(service, function, items, - pageMessage=None, messageAttribute=None, maxItems=0, noFinalize=False, - throwReasons=None, retryReasons=None, - pageArgsInBody=False, - **kwargs): - if throwReasons is None: - throwReasons = [] - if retryReasons is None: - retryReasons = [] - allResults = [] - totalItems = 0 - maxArg, maxResults = _setMaxArgResults(maxItems, pageArgsInBody, kwargs) - entityType = Ent.Getting() if pageMessage else None - while True: - if maxArg and maxItems-totalItems < maxResults: - if not pageArgsInBody: - kwargs[maxArg] = maxItems-totalItems - else: - kwargs['body'][maxArg] = maxItems-totalItems - results = callGAPI(service, function, - throwReasons=throwReasons, retryReasons=retryReasons, - **kwargs) - pageToken, totalItems = _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType) - if not pageToken or (maxItems and totalItems >= maxItems): - if not noFinalize: - _finalizeGAPIpagesResult(pageMessage) - return allResults - if not pageArgsInBody: - kwargs['pageToken'] = pageToken - else: - kwargs['body']['pageToken'] = pageToken - -def yieldGAPIpages(service, function, items, - pageMessage=None, messageAttribute=None, maxItems=0, noFinalize=False, - throwReasons=None, retryReasons=None, - pageArgsInBody=False, - **kwargs): - if throwReasons is None: - throwReasons = [] - if retryReasons is None: - retryReasons = [] - totalItems = 0 - maxArg, maxResults = _setMaxArgResults(maxItems, pageArgsInBody, kwargs) - entityType = Ent.Getting() if pageMessage else None - while True: - if maxArg and maxItems-totalItems < maxResults: - if not pageArgsInBody: - kwargs[maxArg] = maxItems-totalItems - else: - kwargs['body'][maxArg] = maxItems-totalItems - results = callGAPI(service, function, - throwReasons=throwReasons, retryReasons=retryReasons, - **kwargs) - if results: - pageToken = results.get('nextPageToken') - if items in results: - pageItems = len(results[items]) - totalItems += pageItems - else: - results = {items: []} - pageItems = 0 - else: - pageToken = None - results = {items: []} - pageItems = 0 - if pageMessage: - _showGAPIpagesResult(results[items], pageItems, totalItems, pageMessage, messageAttribute, entityType) - yield results[items] - if not pageToken or (maxItems and totalItems >= maxItems): - if not noFinalize: - _finalizeGAPIpagesResult(pageMessage) - return - if not pageArgsInBody: - kwargs['pageToken'] = pageToken - else: - kwargs['body']['pageToken'] = pageToken - -def callGAPIitems(service, function, items, - throwReasons=None, retryReasons=None, - **kwargs): - if throwReasons is None: - throwReasons = [] - if retryReasons is None: - retryReasons = [] - results = callGAPI(service, function, - throwReasons=throwReasons, retryReasons=retryReasons, - **kwargs) - if results: - return results.get(items, []) - return [] def readDiscoveryFile(api_version): disc_filename = f'{api_version}.json' @@ -1411,61 +769,6 @@ def buildGAPIObject(api, credentials=None): GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id return service -def getSaUser(user): - currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API] - currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] - from util.uid import convertUIDtoEmailAddress # local to avoid api↔uid cycle - userEmail = convertUIDtoEmailAddress(user) if user else None - GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI - GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes - return userEmail - -def chooseSaAPI(api1, api2): - _getSvcAcctData() - if api1 in GM.Globals[GM.SVCACCT_SCOPES]: - return api1 - return api2 - -def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True): - userEmail = getSaUser(user) - if GM.Globals[GM.HTTP_OBJECT] is None: - GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR]) - httpObj = GM.Globals[GM.HTTP_OBJECT] - service = getService(api, httpObj) - credentials = getSvcAcctCredentials(api, userEmail) - request = transportCreateRequest(httpObj) - triesLimit = 3 - for n in range(1, triesLimit+1): - try: - credentials.refresh(request) - service._http = transportAuthorizedHttp(credentials, http=httpObj) - return (userEmail, service) - except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: - if n != triesLimit: - httpObj.connections = {} - waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e)) - continue - handleServerError(e) - except google.auth.exceptions.RefreshError as e: - if isinstance(e.args, tuple): - e = e.args[0] - if n < triesLimit: - if isinstance(e, str): - eContent = e - else: - eContent = e.content.decode(UTF8) if isinstance(e.content, bytes) else e.content - if eContent[0:15] == '': - if GC.Values[GC.DEBUG_LEVEL] > 0: - writeStdout(f'{ERROR_PREFIX} HTTP: {str(eContent)}\n') - lContent = eContent.lower() - tg = HTML_TITLE_PATTERN.match(lContent) - lContent = tg.group(1) if tg else '' - if lContent.startswith('Error 502 (Server Error)'): - time.sleep(30) - continue - handleOAuthTokenError(e, True, displayError, i, count) - return (userEmail, None) - def buildGAPIObjectNoAuthentication(api): httpObj = getHttpObj(cache=GM.Globals[GM.CACHE_DIR]) service = getService(api, httpObj) @@ -1482,21 +785,6 @@ def initGDataObject(gdataObj, api): gdataObj.debug = True return gdataObj -def getGDataUserCredentials(api, user, i, count): - userEmail = getSaUser(user) - credentials = getSvcAcctCredentials(api, userEmail) - request = transportCreateRequest() - try: - credentials.refresh(request) - return (userEmail, credentials) - except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: - handleServerError(e) - except google.auth.exceptions.RefreshError as e: - if isinstance(e.args, tuple): - e = e.args[0] - handleOAuthTokenError(e, True, True, i, count) - return (userEmail, None) - def getContactsObject(): contactsObject = initGDataObject(gdata.apps.contacts.service.ContactsService(contactFeed=True), API.CONTACTS) diff --git a/src/gam/util/api_call.py b/src/gam/util/api_call.py new file mode 100644 index 00000000..c639e1dd --- /dev/null +++ b/src/gam/util/api_call.py @@ -0,0 +1,677 @@ +"""Low-level Google API call wrappers with retry logic. + +Contains callGAPI/callGAPIpages/callGData and their error-checking +helpers. Separated from api.py (which handles auth/credentials/service +construction) to break the api<->uid circular dependency. +""" + +import http.client +import json +import re +import ssl +import sys +import time + +import googleapiclient.errors +import httplib2 + +from gamlib import api as API +from gamlib import settings as GC +from gamlib import gapi as GAPI +from gamlib import gdata as GDATA +from gamlib import state as GM +from gamlib import msgs as Msg +from gam.var import Ent +from gam.constants import GOOGLE_API_ERROR_RC, HTTP_ERROR_RC, NETWORK_ERROR_RC, SOCKET_ERROR_RC, SYSTEM_ERROR_RC +from util.api import APIAccessDeniedExit, clearServiceCache, getGDataOAuthToken, getHttpObj, handleOAuthTokenError, handleServerError, transportCreateRequest, waitOnFailure +from util.args import UTF8, formatHTTPError +from util.display import FIRST_ITEM_MARKER, LAST_ITEM_MARKER, SERVICE_NOT_APPLICABLE_RC, TOTAL_ITEMS_MARKER, entityActionFailedWarning, printKeyValueList, userServiceNotEnabledWarning +from util.errors import INVALID_JSON_RC +from util.fileio import checkAPICallsRate, incrAPICallsRetryData +from util.output import ERROR_PREFIX, flushStderr, setSysExitRC, stderrErrorMsg, systemErrorExit, writeStderr, writeStdout + +HTML_TITLE_PATTERN = re.compile(r'.*(.+)') + +def checkGDataError(e, service): + error = e.args + reason = error[0].get('reason', '') + body = error[0].get('body', '').decode(UTF8) + # First check for errors that need special handling + if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']: + keep_domain = service.domain + getGDataOAuthToken(service) + service.domain = keep_domain + return (GDATA.TOKEN_EXPIRED, reason) + error_code = getattr(e, 'error_code', 600) + if GC.Values[GC.DEBUG_LEVEL] > 0: + writeStdout(f'{ERROR_PREFIX} {error_code}: {reason}, {body}\n') + if error_code == 600: + if (body.startswith('Quota exceeded for the current request') or + body.startswith('Quota exceeded for quota metric') or + body.startswith('Request rate higher than configured')): + return (GDATA.QUOTA_EXCEEDED, body) + if (body.startswith('Photo delete failed') or + body.startswith('Upload photo failed') or + body.startswith('Photo query failed')): + return (GDATA.NOT_FOUND, body) + if body.startswith(GDATA.API_DEPRECATED_MSG): + return (GDATA.API_DEPRECATED, body) + if reason == 'Too Many Requests': + return (GDATA.QUOTA_EXCEEDED, reason) + if reason == 'Bad Gateway': + return (GDATA.BAD_GATEWAY, reason) + if reason == 'Gateway Timeout': + return (GDATA.GATEWAY_TIMEOUT, reason) + if reason == 'Service Unavailable': + return (GDATA.SERVICE_UNAVAILABLE, reason) + if reason == 'Service disabled by G Suite admin.': + return (GDATA.FORBIDDEN, reason) + if reason == 'Internal Server Error': + return (GDATA.INTERNAL_SERVER_ERROR, reason) + if reason == 'Token invalid - Invalid token: Token disabled, revoked, or expired.': + return (GDATA.TOKEN_INVALID, 'Token disabled, revoked, or expired. Please delete and re-create oauth.txt') + if reason == 'Token invalid - AuthSub token has wrong scope': + return (GDATA.INSUFFICIENT_PERMISSIONS, reason) + if reason.startswith('Only administrators can request entries belonging to'): + return (GDATA.INSUFFICIENT_PERMISSIONS, reason) + if reason == 'You are not authorized to access this API': + return (GDATA.INSUFFICIENT_PERMISSIONS, reason) + if reason == 'Invalid domain.': + return (GDATA.INVALID_DOMAIN, reason) + if reason.startswith('You are not authorized to perform operations on the domain'): + return (GDATA.INVALID_DOMAIN, reason) + if reason == 'Bad Request': + if 'already exists' in body: + return (GDATA.ENTITY_EXISTS, Msg.DUPLICATE) + return (GDATA.BAD_REQUEST, body) + if reason == 'Forbidden': + return (GDATA.FORBIDDEN, body) + if reason == 'Not Found': + return (GDATA.NOT_FOUND, Msg.DOES_NOT_EXIST) + if reason == 'Not Implemented': + return (GDATA.NOT_IMPLEMENTED, body) + if reason == 'Precondition Failed': + return (GDATA.PRECONDITION_FAILED, reason) + elif error_code == 602: + if body.startswith(GDATA.API_DEPRECATED_MSG): + return (GDATA.API_DEPRECATED, body) + if reason == 'Bad Request': + return (GDATA.BAD_REQUEST, body) + elif error_code == 610: + if reason == 'Service disabled by G Suite admin.': + return (GDATA.FORBIDDEN, reason) + + # We got a "normal" error, define the mapping below + error_code_map = { + 1000: reason, + 1001: reason, + 1002: 'Unauthorized and forbidden', + 1100: 'User deleted recently', + 1200: 'Domain user limit exceeded', + 1201: 'Domain alias limit exceeded', + 1202: 'Domain suspended', + 1203: 'Domain feature unavailable', + 1300: f'Entity {getattr(e, "invalidInput", "")} exists', + 1301: f'Entity {getattr(e, "invalidInput", "")} Does Not Exist', + 1302: 'Entity Name Is Reserved', + 1303: f'Entity {getattr(e, "invalidInput", "")} name not valid', + 1306: f'{getattr(e, "invalidInput", "")} has members. Cannot delete.', + 1317: f'Invalid input {getattr(e, "invalidInput", "")}, reason {getattr(e, "reason", "")}', + 1400: 'Invalid Given Name', + 1401: 'Invalid Family Name', + 1402: 'Invalid Password', + 1403: 'Invalid Username', + 1404: 'Invalid Hash Function Name', + 1405: 'Invalid Hash Digest Length', + 1406: 'Invalid Email Address', + 1407: 'Invalid Query Parameter Value', + 1408: 'Invalid SSO Signing Key', + 1409: 'Invalid Encryption Public Key', + 1410: 'Feature Unavailable For User', + 1411: 'Invalid Encryption Public Key Format', + 1500: 'Too Many Recipients On Email List', + 1501: 'Too Many Aliases For User', + 1502: 'Too Many Delegates For User', + 1601: 'Duplicate Destinations', + 1602: 'Too Many Destinations', + 1603: 'Invalid Route Address', + 1700: 'Group Cannot Contain Cycle', + 1800: 'Group Cannot Contain Cycle', + 1801: f'Invalid value {getattr(e, "invalidInput", "")}', + } + return (error_code, error_code_map.get(error_code, f'Unknown Error: {str(e)}')) + +def callGData(service, function, + bailOnInternalServerError=False, softErrors=False, + throwErrors=None, retryErrors=None, triesLimit=0, + **kwargs): + if throwErrors is None: + throwErrors = [] + if retryErrors is None: + retryErrors = [] + if triesLimit == 0: + triesLimit = GC.Values[GC.API_CALLS_TRIES_LIMIT] + allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors + method = getattr(service, function) + if GC.Values[GC.API_CALLS_RATE_CHECK]: + checkAPICallsRate() + for n in range(1, triesLimit+1): + try: + return method(**kwargs) + except (gdata.service.RequestError, gdata.apps.service.AppsForYourDomainException) as e: + error_code, error_message = checkGDataError(e, service) + if (n != triesLimit) and (error_code in allRetryErrors): + if (error_code == GDATA.INTERNAL_SERVER_ERROR and + bailOnInternalServerError and n == GC.Values[GC.BAIL_ON_INTERNAL_ERROR_TRIES]): + raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message) + waitOnFailure(n, triesLimit, error_code, error_message) + continue + if error_code in throwErrors: + if error_code in GDATA.ERROR_CODE_EXCEPTION_MAP: + raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message) + raise + if softErrors: + stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}') + return None + if error_code == GDATA.INSUFFICIENT_PERMISSIONS: + APIAccessDeniedExit() + systemErrorExit(GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}') + except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: + if n != triesLimit: + waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e)) + continue + handleServerError(e) + except google.auth.exceptions.RefreshError as e: + if isinstance(e.args, tuple): + e = e.args[0] + handleOAuthTokenError(e, GDATA.SERVICE_NOT_APPLICABLE in throwErrors) + raise GDATA.ERROR_CODE_EXCEPTION_MAP[GDATA.SERVICE_NOT_APPLICABLE](str(e)) + except (http.client.ResponseNotReady, OSError) as e: + errMsg = f'Connection error: {str(e) or repr(e)}' + if n != triesLimit: + waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg) + continue + if softErrors: + writeStderr(f'\n{ERROR_PREFIX}{errMsg} - Giving up.\n') + return None + systemErrorExit(SOCKET_ERROR_RC, errMsg) + +def writeGotMessage(msg): + if GC.Values[GC.SHOW_GETTINGS_GOT_NL]: + writeStderr(msg) + else: + writeStderr('\r') + msgLen = len(msg) + if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]: + writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen)) + else: + writeStderr(msg) + GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen + flushStderr() + +def callGDataPages(service, function, + pageMessage=None, + softErrors=False, throwErrors=None, retryErrors=None, + uri=None, + **kwargs): + if throwErrors is None: + throwErrors = [] + if retryErrors is None: + retryErrors = [] + nextLink = None + allResults = [] + totalItems = 0 + while True: + this_page = callGData(service, function, + softErrors=softErrors, throwErrors=throwErrors, retryErrors=retryErrors, + uri=uri, + **kwargs) + if this_page: + nextLink = this_page.GetNextLink() + pageItems = len(this_page.entry) + if pageItems == 0: + nextLink = None + totalItems += pageItems + allResults.extend(this_page.entry) + else: + nextLink = None + pageItems = 0 + if pageMessage: + show_message = pageMessage.replace(TOTAL_ITEMS_MARKER, str(totalItems)) + writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems))) + if nextLink is None: + if pageMessage and (pageMessage[-1] != '\n'): + writeStderr('\r\n') + flushStderr() + return allResults + uri = nextLink.href + if 'url_params' in kwargs: + kwargs['url_params'].pop('start-index', None) + +def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True): + def makeErrorDict(code, reason, message): + return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}} + + try: + error = json.loads(e.content.decode(UTF8)) + if GC.Values[GC.DEBUG_LEVEL] > 0: + writeStdout(f'{ERROR_PREFIX} JSON: {str(error)}\n') + except (IndexError, KeyError, SyntaxError, TypeError, ValueError): + eContent = e.content.decode(UTF8) if isinstance(e.content, bytes) else e.content + lContent = eContent.lower() + if GC.Values[GC.DEBUG_LEVEL] > 0: + writeStdout(f'{ERROR_PREFIX} HTTP: {str(eContent)}\n') + if eContent[0:15] != '': + if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')): + return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) + if (e.resp['status'] == '429') and (lContent.startswith('quota exceeded for quota metric')): + return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) + if (e.resp['status'] == '502') and ('bad gateway' in lContent): + return (e.resp['status'], GAPI.BAD_GATEWAY, eContent) + if (e.resp['status'] == '503') and (lContent.startswith('quota exceeded for the current request')): + return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) + if (e.resp['status'] == '504') and ('gateway timeout' in lContent): + return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent) + else: + tg = HTML_TITLE_PATTERN.match(lContent) + lContent = tg.group(1) if tg else 'bad request' + if (e.resp['status'] == '403') and ('invalid domain.' in lContent): + error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found') + elif (e.resp['status'] == '403') and ('domain cannot use apis.' in lContent): + error = makeErrorDict(403, GAPI.DOMAIN_CANNOT_USE_APIS, 'Domain cannot use apis') + elif (e.resp['status'] == '400') and ('invalidssosigningkey' in lContent): + error = makeErrorDict(400, GAPI.INVALID, 'InvalidSsoSigningKey') + elif (e.resp['status'] == '400') and ('unknownerror' in lContent): + error = makeErrorDict(400, GAPI.INVALID, 'UnknownError') + elif (e.resp['status'] == '400') and ('featureunavailableforuser' in lContent): + error = makeErrorDict(400, GAPI.SERVICE_NOT_AVAILABLE, 'Feature Unavailable For User') + elif (e.resp['status'] == '400') and ('entitydoesnotexist' in lContent): + error = makeErrorDict(400, GAPI.NOT_FOUND, 'Entity Does Not Exist') + elif (e.resp['status'] == '400') and ('entitynamenotvalid' in lContent): + error = makeErrorDict(400, GAPI.INVALID_INPUT, 'Entity Name Not Valid') + elif (e.resp['status'] == '400') and ('failed to parse Content-Range header' in lContent): + error = makeErrorDict(400, GAPI.BAD_REQUEST, 'Failed to parse Content-Range header') + elif (e.resp['status'] == '400') and ('request contains an invalid argument' in lContent): + error = makeErrorDict(400, GAPI.INVALID_ARGUMENT, 'Request contains an invalid argument') + elif (e.resp['status'] == '404') and ('not found' in lContent): + error = makeErrorDict(404, GAPI.NOT_FOUND, lContent) + elif (e.resp['status'] == '404') and ('bad request' in lContent): + error = makeErrorDict(404, GAPI.BAD_REQUEST, lContent) + elif retryOnHttpError: + return (-1, None, eContent) + elif softErrors: + stderrErrorMsg(eContent) + return (0, None, None) + else: + systemErrorExit(HTTP_ERROR_RC, eContent) + requiredScopes = '' + wwwAuthenticate = e.resp.get('www-authenticate', '') + if 'insufficient_scope' in wwwAuthenticate: + mg = re.match(r'.+scope="(.+)"', wwwAuthenticate) + if mg: + requiredScopes = mg.group(1).split(' ') + if 'error' in error: + http_status = error['error']['code'] + reason = '' + if 'errors' in error['error'] and 'message' in error['error']['errors'][0]: + message = error['error']['errors'][0]['message'] + if 'reason' in error['error']['errors'][0]: + reason = error['error']['errors'][0]['reason'] + elif 'errors' in error['error'] and 'Unknown Error' in error['error']['message'] and 'reason' in error['error']['errors'][0]: + message = error['error']['errors'][0]['reason'] + else: + message = error['error']['message'] + status = error['error'].get('status', '') + lmessage = message.lower() if message is not None else '' + if http_status == 500: + if not lmessage or status == 'UNKNOWN': + if not lmessage: + message = Msg.UNKNOWN + error = makeErrorDict(http_status, GAPI.UNKNOWN_ERROR, message) + elif 'backend error' in lmessage: + error = makeErrorDict(http_status, GAPI.BACKEND_ERROR, message) + elif 'internal error encountered' in lmessage: + error = makeErrorDict(http_status, GAPI.INTERNAL_ERROR, message) + elif 'role assignment exists: roleassignment' in lmessage: + error = makeErrorDict(http_status, GAPI.DUPLICATE, message) + elif 'role assignment exists: roleid' in lmessage: + error = makeErrorDict(http_status, GAPI.DUPLICATE, message) + elif 'operation not supported' in lmessage: + error = makeErrorDict(http_status, GAPI.OPERATION_NOT_SUPPORTED, message) + elif 'failed status in update settings response' in lmessage: + error = makeErrorDict(http_status, GAPI.INVALID_INPUT, message) + elif 'cannot delete a field in use.resource.fields' in lmessage: + error = makeErrorDict(http_status, GAPI.FIELD_IN_USE, message) + elif status == 'INTERNAL': + error = makeErrorDict(http_status, GAPI.INTERNAL_ERROR, message) + elif http_status == 501: + if status == 'UNIMPLEMENTED': + error = makeErrorDict(http_status, GAPI.UNIMPLEMENTED_ERROR, message) + elif http_status == 502: + if 'bad gateway' in lmessage: + error = makeErrorDict(http_status, GAPI.BAD_GATEWAY, message) + elif http_status == 503: + if message.startswith('quota exceeded for the current request'): + error = makeErrorDict(http_status, GAPI.QUOTA_EXCEEDED, message) + elif status == 'UNAVAILABLE' or 'the service is currently unavailable' in lmessage: + error = makeErrorDict(http_status, GAPI.SERVICE_NOT_AVAILABLE, message) + elif http_status == 504: + if 'gateway timeout' in lmessage: + error = makeErrorDict(http_status, GAPI.GATEWAY_TIMEOUT, message) + elif http_status == 400: + if '@attachmentnotvisible' in lmessage: + error = makeErrorDict(http_status, GAPI.BAD_REQUEST, message) + elif status == 'INVALID_ARGUMENT': + error = makeErrorDict(http_status, GAPI.INVALID_ARGUMENT, message) + elif status == 'FAILED_PRECONDITION' or 'precondition check failed' in lmessage: + error = makeErrorDict(http_status, GAPI.FAILED_PRECONDITION, message) + elif 'does not match' in lmessage or 'invalid' in lmessage: + error = makeErrorDict(http_status, GAPI.INVALID, message) + elif http_status == 401: + if 'active session is invalid' in lmessage and reason == 'authError': +# message += ' Drive SDK API access disabled' +# message = Msg.SERVICE_NOT_ENABLED.format('Drive') + error = makeErrorDict(http_status, GAPI.AUTH_ERROR, message) + elif status == 'PERMISSION_DENIED': + error = makeErrorDict(http_status, GAPI.PERMISSION_DENIED, message) + elif status == 'UNAUTHENTICATED': + error = makeErrorDict(http_status, GAPI.AUTH_ERROR, message) + elif http_status == 403: + if 'quota exceeded for quota metric' in lmessage: + error = makeErrorDict(http_status, GAPI.QUOTA_EXCEEDED, message) + elif 'the authenticated user cannot access this service' in lmessage: + error = makeErrorDict(http_status, GAPI.SERVICE_NOT_AVAILABLE, message) + elif status == 'PERMISSION_DENIED' or 'the caller does not have permission' in lmessage or 'permission iam.serviceaccountkeys' in lmessage: + if requiredScopes: + message += f', {Msg.NO_SCOPES_FOR_API.format(API.findAPIforScope(requiredScopes))}' + error = makeErrorDict(http_status, GAPI.PERMISSION_DENIED, message) + elif http_status == 404: + if status == 'NOT_FOUND' or 'requested entity was not found' in lmessage or 'does not exist' in lmessage: + error = makeErrorDict(http_status, GAPI.NOT_FOUND, message) + elif http_status == 409: + if status == 'ALREADY_EXISTS' or 'requested entity already exists' in lmessage: + error = makeErrorDict(http_status, GAPI.ALREADY_EXISTS, message) + elif status == 'ABORTED' or 'the operation was aborted' in lmessage: + error = makeErrorDict(http_status, GAPI.ABORTED, message) + elif http_status == 412: + if 'insufficient archived user licenses' in lmessage: + error = makeErrorDict(http_status, GAPI.INSUFFICIENT_ARCHIVED_USER_LICENSES, message) + elif http_status == 413: + if 'request too large' in lmessage: + error = makeErrorDict(http_status, GAPI.UPLOAD_TOO_LARGE, message) + elif http_status == 429: + if status == 'RESOURCE_EXHAUSTED' or 'quota exceeded' in lmessage or 'insufficient quota' in lmessage: + error = makeErrorDict(http_status, GAPI.QUOTA_EXCEEDED, message) + else: + if 'error_description' in error: + if error['error_description'] == 'Invalid Value': + message = error['error_description'] + http_status = 400 + error = makeErrorDict(http_status, GAPI.INVALID, message) + else: + systemErrorExit(GOOGLE_API_ERROR_RC, str(error)) + else: + systemErrorExit(GOOGLE_API_ERROR_RC, str(error)) + try: + reason = error['error']['errors'][0]['reason'] + for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []): + if messageItem[0] in message: + if reason in [GAPI.NOT_FOUND, GAPI.RESOURCE_NOT_FOUND] and mapNotFound: + message = Msg.DOES_NOT_EXIST + reason = messageItem[1] + break + if reason == GAPI.INVALID_SHARING_REQUEST: + loc = message.find('User message: ') + if loc != -1: + message = message[loc+15:] + else: + loc = message.find('User message: ""') + if loc != -1: + message = message[:loc+14]+f'"{reason}"' + except KeyError: + reason = f'{http_status}' + return (http_status, reason, message) + +def callGAPI(service, function, + bailOnInternalError=False, bailOnTransientError=False, bailOnInvalidError=False, + softErrors=False, mapNotFound=True, + throwReasons=None, retryReasons=None, triesLimit=0, + **kwargs): + if throwReasons is None: + throwReasons = [] + if retryReasons is None: + retryReasons = [] + if triesLimit == 0: + triesLimit = GC.Values[GC.API_CALLS_TRIES_LIMIT] + allRetryReasons = GAPI.DEFAULT_RETRY_REASONS+retryReasons + method = getattr(service, function) + svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST]) + if GC.Values[GC.API_CALLS_RATE_CHECK]: + checkAPICallsRate() + for n in range(1, triesLimit+1): + try: + return method(**svcparms).execute() + except googleapiclient.errors.HttpError as e: + http_status, reason, message = checkGAPIError(e, softErrors=softErrors, retryOnHttpError=n < 3, mapNotFound=mapNotFound) + if http_status == -1: + # The error detail indicated that we should retry this request + # We'll refresh credentials and make another pass + try: +# service._http.credentials.refresh(getHttpObj()) + service._http.credentials.refresh(transportCreateRequest()) + except TypeError: + systemErrorExit(HTTP_ERROR_RC, message) + continue + if http_status == 0: + return None + if (n != triesLimit) and ((reason in allRetryReasons) or + (GC.Values[GC.RETRY_API_SERVICE_NOT_AVAILABLE] and (reason == GAPI.SERVICE_NOT_AVAILABLE))): + if (reason in [GAPI.INTERNAL_ERROR, GAPI.BACKEND_ERROR] and + bailOnInternalError and n == GC.Values[GC.BAIL_ON_INTERNAL_ERROR_TRIES]): + raise GAPI.REASON_EXCEPTION_MAP[reason](message) + if (reason in [GAPI.INVALID] and + bailOnInvalidError and n == GC.Values[GC.BAIL_ON_INTERNAL_ERROR_TRIES]): + raise GAPI.REASON_EXCEPTION_MAP[reason](message) + waitOnFailure(n, triesLimit, reason, message) + if reason == GAPI.TRANSIENT_ERROR and bailOnTransientError: + raise GAPI.REASON_EXCEPTION_MAP[reason](message) + continue + if reason in throwReasons: + if reason in GAPI.REASON_EXCEPTION_MAP: + raise GAPI.REASON_EXCEPTION_MAP[reason](message) + raise e + if softErrors: + stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}') + return None + if reason == GAPI.INSUFFICIENT_PERMISSIONS: + APIAccessDeniedExit() + systemErrorExit(HTTP_ERROR_RC, formatHTTPError(http_status, reason, message)) + except googleapiclient.errors.MediaUploadSizeError as e: + raise e + except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: + if n != triesLimit: + service._http.connections = {} + waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e)) + continue + handleServerError(e) + except google.auth.exceptions.RefreshError as e: + if isinstance(e.args, tuple): + e = e.args[0] + handleOAuthTokenError(e, GAPI.SERVICE_NOT_AVAILABLE in throwReasons) + raise GAPI.REASON_EXCEPTION_MAP[GAPI.SERVICE_NOT_AVAILABLE](str(e)) + except (http.client.ResponseNotReady, OSError) as e: + errMsg = f'Connection error: {str(e) or repr(e)}' + if n != triesLimit: + waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg) + continue + if softErrors: + writeStderr(f'\n{ERROR_PREFIX}{errMsg} - Giving up.\n') + return None + systemErrorExit(SOCKET_ERROR_RC, errMsg) + except ValueError as e: + if clearServiceCache(service): + continue + systemErrorExit(GOOGLE_API_ERROR_RC, str(e)) + except TypeError as e: + systemErrorExit(GOOGLE_API_ERROR_RC, str(e)) + +def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType): + showMessage = pageMessage.replace(TOTAL_ITEMS_MARKER, str(totalItems)) + if pageItems: + if messageAttribute: + firstItem = results[0] if pageItems > 0 else {} + lastItem = results[-1] if pageItems > 1 else firstItem + if isinstance(messageAttribute, str): + firstItem = str(firstItem.get(messageAttribute, '')) + lastItem = str(lastItem.get(messageAttribute, '')) + else: + for attr in messageAttribute: + firstItem = firstItem.get(attr, {}) + lastItem = lastItem.get(attr, {}) + firstItem = str(firstItem) + lastItem = str(lastItem) + showMessage = showMessage.replace(FIRST_ITEM_MARKER, firstItem) + showMessage = showMessage.replace(LAST_ITEM_MARKER, lastItem) + else: + showMessage = showMessage.replace(FIRST_ITEM_MARKER, '') + showMessage = showMessage.replace(LAST_ITEM_MARKER, '') + writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems)))) + +def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType): + if results: + pageToken = results.get('nextPageToken') + if items in results: + pageItems = len(results[items]) + totalItems += pageItems + if allResults is not None: + allResults.extend(results[items]) + else: + results = {items: []} + pageItems = 0 + else: + pageToken = None + results = {items: []} + pageItems = 0 + if pageMessage: + _showGAPIpagesResult(results[items], pageItems, totalItems, pageMessage, messageAttribute, entityType) + return (pageToken, totalItems) + +def _finalizeGAPIpagesResult(pageMessage): + if pageMessage and (pageMessage[-1] != '\n'): + writeStderr('\r\n') + flushStderr() + +def _setMaxArgResults(maxItems, pageArgsInBody, kwargs): + if pageArgsInBody: + kwargs.setdefault('body', {}) + maxArg = '' + maxResults = 0 + if maxItems: + if not pageArgsInBody: + maxResults = kwargs.get('maxResults', 0) + if maxResults: + maxArg = 'maxResults' + else: + maxResults = kwargs.get('pageSize', 0) + if maxResults: + maxArg = 'pageSize' + else: + maxResults = kwargs['body'].get('maxResults', 0) + if maxResults: + maxArg = 'maxResults' + else: + maxResults = kwargs['body'].get('pageSize', 0) + if maxResults: + maxArg = 'pageSize' + return (maxArg, maxResults) + +def callGAPIpages(service, function, items, + pageMessage=None, messageAttribute=None, maxItems=0, noFinalize=False, + throwReasons=None, retryReasons=None, + pageArgsInBody=False, + **kwargs): + if throwReasons is None: + throwReasons = [] + if retryReasons is None: + retryReasons = [] + allResults = [] + totalItems = 0 + maxArg, maxResults = _setMaxArgResults(maxItems, pageArgsInBody, kwargs) + entityType = Ent.Getting() if pageMessage else None + while True: + if maxArg and maxItems-totalItems < maxResults: + if not pageArgsInBody: + kwargs[maxArg] = maxItems-totalItems + else: + kwargs['body'][maxArg] = maxItems-totalItems + results = callGAPI(service, function, + throwReasons=throwReasons, retryReasons=retryReasons, + **kwargs) + pageToken, totalItems = _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType) + if not pageToken or (maxItems and totalItems >= maxItems): + if not noFinalize: + _finalizeGAPIpagesResult(pageMessage) + return allResults + if not pageArgsInBody: + kwargs['pageToken'] = pageToken + else: + kwargs['body']['pageToken'] = pageToken + +def yieldGAPIpages(service, function, items, + pageMessage=None, messageAttribute=None, maxItems=0, noFinalize=False, + throwReasons=None, retryReasons=None, + pageArgsInBody=False, + **kwargs): + if throwReasons is None: + throwReasons = [] + if retryReasons is None: + retryReasons = [] + totalItems = 0 + maxArg, maxResults = _setMaxArgResults(maxItems, pageArgsInBody, kwargs) + entityType = Ent.Getting() if pageMessage else None + while True: + if maxArg and maxItems-totalItems < maxResults: + if not pageArgsInBody: + kwargs[maxArg] = maxItems-totalItems + else: + kwargs['body'][maxArg] = maxItems-totalItems + results = callGAPI(service, function, + throwReasons=throwReasons, retryReasons=retryReasons, + **kwargs) + if results: + pageToken = results.get('nextPageToken') + if items in results: + pageItems = len(results[items]) + totalItems += pageItems + else: + results = {items: []} + pageItems = 0 + else: + pageToken = None + results = {items: []} + pageItems = 0 + if pageMessage: + _showGAPIpagesResult(results[items], pageItems, totalItems, pageMessage, messageAttribute, entityType) + yield results[items] + if not pageToken or (maxItems and totalItems >= maxItems): + if not noFinalize: + _finalizeGAPIpagesResult(pageMessage) + return + if not pageArgsInBody: + kwargs['pageToken'] = pageToken + else: + kwargs['body']['pageToken'] = pageToken + +def callGAPIitems(service, function, items, + throwReasons=None, retryReasons=None, + **kwargs): + if throwReasons is None: + throwReasons = [] + if retryReasons is None: + retryReasons = [] + results = callGAPI(service, function, + throwReasons=throwReasons, retryReasons=retryReasons, + **kwargs) + if results: + return results.get(items, []) + return [] diff --git a/src/gam/util/csv_pf.py b/src/gam/util/csv_pf.py index 2e6c2179..3cb6e30e 100644 --- a/src/gam/util/csv_pf.py +++ b/src/gam/util/csv_pf.py @@ -20,7 +20,9 @@ from gamlib import state as GM from gamlib import msgs as Msg from gam.constants import DEFAULT_FILE_APPEND_MODE, INSUFFICIENT_PERMISSIONS_RC, MAX_GOOGLE_SHEET_CELLS, NO_CSV_DATA_TO_UPLOAD_RC, ROOT from tempfile import TemporaryFile -from util.api import _getAdminEmail, buildGAPIObject, buildGAPIServiceObject, callGAPI, callGAPIpages, chooseSaAPI +from util.api import _getAdminEmail, buildGAPIObject +from util.svcacct import buildGAPIServiceObject, chooseSaAPI +from util.api_call import callGAPI, callGAPIpages from util.args import LOCALE_CODES_MAP, NEVER_TIME, TRUE_FALSE, UTF8, YYYYMMDD_FORMAT, YYYYMMDD_PATTERN, escapeCRsNLs, formatLocalTime, formatLocalTimestamp, getArgument, getBoolean, getCharacter, getChoice, getInteger, getLanguageCode, getSheetEntity, getSheetIdFromSheetEntity, getString, normalizeEmailAddressOrUID, protectedSheetId, todaysTime from util.display import ACTION_FAILED_RC, entityActionFailedWarning, entityActionPerformed, printBlankLine, printJSONKey, printJSONValue, printKeyValueList, userDriveServiceNotEnabledWarning from util.email import send_email diff --git a/src/gam/util/email.py b/src/gam/util/email.py index 72d5e458..cb9a8042 100644 --- a/src/gam/util/email.py +++ b/src/gam/util/email.py @@ -21,7 +21,9 @@ from gam.var import Act, Ent from gamlib import api as API from gamlib import settings as GC from gamlib import gapi as GAPI -from util.api import _getAdminEmail, buildGAPIObject, buildGAPIServiceObject, callGAPI +from util.api import _getAdminEmail, buildGAPIObject +from util.svcacct import buildGAPIServiceObject +from util.api_call import callGAPI from util.args import NAME_EMAIL_ADDRESS_PATTERN, UTF8, normalizeEmailAddressOrUID from util.display import entityActionFailedWarning, entityActionPerformed, entityActionPerformedMessage from util.errors import usageErrorExit diff --git a/src/gam/util/entity.py b/src/gam/util/entity.py index 0a3b23ea..2b9a520a 100644 --- a/src/gam/util/entity.py +++ b/src/gam/util/entity.py @@ -136,7 +136,9 @@ from util.gdoc import getGDocData, getStorageFileData, openCSVFileReader from util.output import formatKeyValueList, printErrorMessage, setSysExitRC, stderrErrorMsg, systemErrorExit, writeStderr from gam.util.access import ClientAPIAccessDeniedExit, accessErrorExit from util.access import accessErrorExit, checkEntityDNEorAccessErrorExit, entityUnknownWarning -from util.api import _getAdminEmail, buildGAPIObject, buildGAPIServiceObject, callGAPI, callGAPIitems, callGAPIpages, yieldGAPIpages +from util.api import _getAdminEmail, buildGAPIObject +from util.svcacct import buildGAPIServiceObject +from util.api_call import callGAPI, callGAPIitems, callGAPIpages, yieldGAPIpages from gam.var import Act, Cmd, Ent def getQueries(myarg): diff --git a/src/gam/util/gdoc.py b/src/gam/util/gdoc.py index 94497401..6cbe9737 100644 --- a/src/gam/util/gdoc.py +++ b/src/gam/util/gdoc.py @@ -29,7 +29,9 @@ from util.display import ACTION_NOT_PERFORMED_RC, userDriveServiceNotEnabledWarn from util.errors import entityActionFailedExit, entityDoesNotExistExit from util.fileio import FILE_ERROR_RC, UTF8_SIG, fileErrorMessage, getGDocSheetDataFailedExit, getGDocSheetDataRetryWarning, openFile, setFilePath from util.output import stderrWarningMsg, systemErrorExit -from util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI +from util.api import buildGAPIObject +from util.svcacct import buildGAPIServiceObject +from util.api_call import callGAPI diff --git a/src/gam/util/group_parents.py b/src/gam/util/group_parents.py index c4ae5b6e..0a820296 100644 --- a/src/gam/util/group_parents.py +++ b/src/gam/util/group_parents.py @@ -8,7 +8,7 @@ from gamlib import gapi as GAPI from gamlib import indent from gam.util.access import accessErrorExit -from gam.util.api import callGAPIpages +from gam.util.api_call import callGAPIpages from gam.util.csv_pf import flattenJSON from gam.util.display import badRequestWarning, printKeyValueListWithCount from gam.util.domain_filters import _setUserGroupArgs diff --git a/src/gam/util/orgunits.py b/src/gam/util/orgunits.py index d4ec5c2e..34d96132 100644 --- a/src/gam/util/orgunits.py +++ b/src/gam/util/orgunits.py @@ -10,7 +10,8 @@ from gamlib import settings as GC from gamlib import gapi as GAPI from gamlib import state as GM from util.access import accessErrorExit, checkEntityAFDNEorAccessErrorExit -from util.api import buildGAPIObject, callGAPI +from util.api import buildGAPIObject +from util.api_call import callGAPI from util.args import encodeOrgUnitPath, makeOrgUnitPathAbsolute, makeOrgUnitPathRelative from util.errors import entityDoesNotExistExit, invalidArgumentExit, missingArgumentExit from gam.var import Cmd, Ent diff --git a/src/gam/util/svcacct.py b/src/gam/util/svcacct.py new file mode 100644 index 00000000..f5a6b8e0 --- /dev/null +++ b/src/gam/util/svcacct.py @@ -0,0 +1,99 @@ +"""Service account API object construction. + +Builds authenticated Google API service objects for domain-wide delegation +(service account) access. Separated from api.py to break the api↔uid +circular dependency — these functions need both api.py (for credentials/service +construction) and uid.py (for UID-to-email resolution). +""" + +import time + +import google.auth.exceptions +import httplib2 + +from gamlib import api as API +from gamlib import settings as GC +from gamlib import state as GM +from gam.constants import NETWORK_ERROR_RC +from util.api import ( + _getSvcAcctData, + getHttpObj, + getService, + getSvcAcctCredentials, + handleOAuthTokenError, + handleServerError, + transportAuthorizedHttp, + transportCreateRequest, + waitOnFailure, +) +from util.args import UTF8 +from util.output import ERROR_PREFIX, writeStdout +from util.uid import convertUIDtoEmailAddress + +HTML_TITLE_PATTERN = __import__('re').compile(r'.*(.+)') + + +def getSaUser(user): + """Resolve a user UID to email, preserving current client API state. + + convertUIDtoEmailAddress may call buildGAPIObject as a side effect, + which modifies GM.Globals[CURRENT_CLIENT_API/SCOPES]. This wrapper + saves and restores those globals so service account authentication + isn't disrupted. + """ + currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API] + currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] + userEmail = convertUIDtoEmailAddress(user) if user else None + GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI + GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes + return userEmail + + +def chooseSaAPI(api1, api2): + """Choose between two API versions based on service account scope availability.""" + _getSvcAcctData() + if api1 in GM.Globals[GM.SVCACCT_SCOPES]: + return api1 + return api2 + + +def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True): + """Build an authenticated Google API service object for a service account user.""" + userEmail = getSaUser(user) + if GM.Globals[GM.HTTP_OBJECT] is None: + GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR]) + httpObj = GM.Globals[GM.HTTP_OBJECT] + service = getService(api, httpObj) + credentials = getSvcAcctCredentials(api, userEmail) + request = transportCreateRequest(httpObj) + triesLimit = 3 + for n in range(1, triesLimit+1): + try: + credentials.refresh(request) + service._http = transportAuthorizedHttp(credentials, http=httpObj) + return (userEmail, service) + except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: + if n != triesLimit: + httpObj.connections = {} + waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e)) + continue + handleServerError(e) + except google.auth.exceptions.RefreshError as e: + if isinstance(e.args, tuple): + e = e.args[0] + if n < triesLimit: + if isinstance(e, str): + eContent = e + else: + eContent = e.content.decode(UTF8) if isinstance(e.content, bytes) else e.content + if eContent[0:15] == '': + if GC.Values[GC.DEBUG_LEVEL] > 0: + writeStdout(f'{ERROR_PREFIX} HTTP: {str(eContent)}\n') + lContent = eContent.lower() + tg = HTML_TITLE_PATTERN.match(lContent) + lContent = tg.group(1) if tg else '' + if lContent.startswith('Error 502 (Server Error)'): + time.sleep(30) + continue + handleOAuthTokenError(e, True, displayError, i, count) + return (userEmail, None) diff --git a/src/gam/util/tags.py b/src/gam/util/tags.py index 7de31649..4949fd43 100644 --- a/src/gam/util/tags.py +++ b/src/gam/util/tags.py @@ -9,7 +9,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.util.access import entityUnknownWarning -from gam.util.api import buildGAPIObject, callGAPI +from gam.util.api import buildGAPIObject +from gam.util.api_call import callGAPI from gam.util.args import ( FALSE, TRUE, diff --git a/src/gam/util/uid.py b/src/gam/util/uid.py index 40e9279b..e4cb8beb 100644 --- a/src/gam/util/uid.py +++ b/src/gam/util/uid.py @@ -10,7 +10,8 @@ from gamlib import gapi as GAPI from gamlib import state as GM from gamlib import msgs as Msg from gam.var import Ent -from util.api import buildGAPIObject, callGAPI +from util.api import buildGAPIObject +from util.api_call import callGAPI from util.args import normalizeEmailAddressOrUID from util.display import entityDoesNotExistWarning, printGettingAllEntityItemsForWhom from util.errors import usageErrorExit @@ -117,5 +118,3 @@ def convertEmailAddressToUID(emailAddressOrUID, cd=None, emailType='user', saved if savedLocation is not None: Cmd.SetLocation(savedLocation) entityDoesNotExistExit([Ent.USER, Ent.GROUP][emailType == 'group'], normalizedEmailAddressOrUID, errMsg=getPhraseDNEorSNA(normalizedEmailAddressOrUID)) - - diff --git a/tests/test_imports.py b/tests/test_imports.py index 65d104a6..a206f1c9 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -33,17 +33,12 @@ class TestAllModulesImport: assert not failures, "Failed to import:\n" + "\n".join(failures) def test_no_local_imports_in_util(self): - """util/ modules should not have function-scope imports from other util/ modules. - - The one known exception is api.py:getSaUser (documented api<->uid cycle). - """ + """util/ modules should not have function-scope imports from other util/ modules.""" import re util_dir = os.path.join(os.path.dirname(__file__), '..', 'src', 'gam', 'util') util_dir = os.path.normpath(util_dir) - KNOWN_EXCEPTIONS = { - ('api.py', 'from util.uid import convertUIDtoEmailAddress'), - } + KNOWN_EXCEPTIONS = set() violations = [] for f in sorted(os.listdir(util_dir)):