From 5037a9bbfd0b555a7595225dd3bc932c09987cb6 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 23 Dec 2015 07:35:36 -0800 Subject: [PATCH 001/100] Debugging only incompatible with gam batch - and gam csv - --- src/gam.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index 5d9388e6..2cd2d904 100755 --- a/src/gam.py +++ b/src/gam.py @@ -249,6 +249,7 @@ GC_VAR_INFO = { } MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' +MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV headers of "{1}".' @@ -1708,7 +1709,7 @@ def buildUserIdToNameMap(): def user_from_userid(userid): if not GM_Globals[GM_MAP_USER_ID_TO_NAME]: buildUserIdToNameMap() - return GM_Globals[GM_MAP_USER_ID_TO_NAME].get(userid, '') + return GM_Globals[GM_MAP_USER_ID_TO_NAME].get(userid, u'') SERVICE_NAME_TO_ID_MAP = { u'Drive': u'55656082996', @@ -6891,14 +6892,14 @@ def doGetBackupCodes(users): codes = callGAPI(service=cd.verificationCodes(), function=u'list', throw_reasons=[u'invalidArgument', u'invalid'], userKey=user) except googleapiclient.errors.HttpError: codes = None - printBackupCodes(user, codes) + printBackupCodes(user, codes) def doGenBackupCodes(users): cd = buildGAPIObject(u'directory') for user in users: callGAPI(service=cd.verificationCodes(), function=u'generate', userKey=user) codes = callGAPI(service=cd.verificationCodes(), function=u'list', userKey=user) - printBackupCodes(user, codes) + printBackupCodes(user, codes) def doDelBackupCodes(users): cd = buildGAPIObject(u'directory') @@ -8997,7 +8998,10 @@ try: SetGlobalVariables() if sys.argv[1].lower() == u'batch': import shlex - f = openFile(sys.argv[2]) + filename = sys.argv[2] + if (filename == u'-') and (GC_Values[GC_DEBUG_LEVEL] > 0): + systemErrorExit(2, MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE.format(u'batch')) + f = openFile(filename) items = list() for line in f: argv = shlex.split(line) @@ -9015,11 +9019,10 @@ try: run_batch(items) sys.exit(0) elif sys.argv[1].lower() == u'csv': - if httplib2.debuglevel > 0: - print u'Sorry, CSV commands are not compatible with debug. Delete debug.gam and try again.' - sys.exit(1) - csv_filename = sys.argv[2] - f = openFile(csv_filename) + filename = sys.argv[2] + if (filename == u'-') and (GC_Values[GC_DEBUG_LEVEL] > 0): + systemErrorExit(2, MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE.format(u'csv')) + f = openFile(filename) input_file = csv.DictReader(f) if sys.argv[3].lower() != 'gam': print 'ERROR: "gam csv " should be followed by a full GAM command...' From d79c28d2d306851d3517f170dfc2c6447653f79d Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 23 Dec 2015 11:07:08 -0800 Subject: [PATCH 002/100] Eliminate unneeded service=, function=, items= in calls to API functions Saves 5.5kb! --- src/gam.py | 657 ++++++++++++++++++++++++++--------------------------- 1 file changed, 328 insertions(+), 329 deletions(-) diff --git a/src/gam.py b/src/gam.py index 79ac0562..941f9809 100755 --- a/src/gam.py +++ b/src/gam.py @@ -708,14 +708,14 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re except TypeError, e: systemErrorExit(4, e) -def callGAPIpages(service, function, items=u'items', nextPageToken=u'nextPageToken', page_message=None, message_attribute=None, **kwargs): +def callGAPIpages(service, function, items, page_message=None, message_attribute=None, **kwargs): pageToken = None all_pages = list() total_items = 0 while True: - this_page = callGAPI(service=service, function=function, pageToken=pageToken, **kwargs) + this_page = callGAPI(service, function, pageToken=pageToken, **kwargs) if this_page: - pageToken = this_page.get(nextPageToken) + pageToken = this_page.get(u'nextPageToken') if items in this_page: page_items = len(this_page[items]) total_items += page_items @@ -1001,7 +1001,7 @@ def showReport(): while True: try: page_message = u'Got %%num_items%% users\n' - usage = callGAPIpages(service=rep.userUsageReport(), function=u'get', items=u'usageReports', page_message=page_message, throw_reasons=[u'invalid'], + usage = callGAPIpages(rep.userUsageReport(), u'get', u'usageReports', page_message=page_message, throw_reasons=[u'invalid'], date=str(try_date), userKey=userKey, customerId=customerId, filters=filters, parameters=parameters) break except googleapiclient.errors.HttpError, e: @@ -1039,7 +1039,7 @@ def showReport(): elif report in [u'customer', u'customers', u'domain']: while True: try: - usage = callGAPIpages(service=rep.customerUsageReports(), function=u'get', items=u'usageReports', throw_reasons=[u'invalid'], + usage = callGAPIpages(rep.customerUsageReports(), u'get', u'usageReports', throw_reasons=[u'invalid'], customerId=customerId, date=str(try_date), parameters=parameters) break except googleapiclient.errors.HttpError, e: @@ -1088,7 +1088,7 @@ def showReport(): elif report == u'tokens': report = u'token' page_message = u'Got %%num_items%% items\n' - activities = callGAPIpages(service=rep.activities(), function=u'list', page_message=page_message, applicationName=report, + activities = callGAPIpages(rep.activities(), u'list', u'items', page_message=page_message, applicationName=report, userKey=userKey, customerId=customerId, actorIpAddress=actorIpAddress, startTime=startTime, endTime=endTime, eventName=eventName, filters=filters) if len(activities) > 0: @@ -1148,7 +1148,7 @@ def doDelegates(users): else: # Need to use an alias in delegator domain, first check to see if delegate already has one... cd = buildGAPIObject(u'directory') - aliases = callGAPI(service=cd.users().aliases(), function=u'list', userKey=delegate_email) + aliases = callGAPI(cd.users().aliases(), u'list', userKey=delegate_email) found_alias_in_delegator_domain = False try: for alias in aliases[u'aliases']: @@ -1164,16 +1164,16 @@ def doDelegates(users): delete_alias = True use_delegate_address = u'%s@%s' % (''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz0123456789', 25)), delegator_domain) print u' Giving %s temporary alias %s for delegation' % (delegate_email, use_delegate_address) - callGAPI(service=cd.users().aliases(), function=u'insert', userKey=delegate_email, body={u'alias': use_delegate_address}) + callGAPI(cd.users().aliases(), u'insert', userKey=delegate_email, body={u'alias': use_delegate_address}) time.sleep(5) retries = 10 for n in range(1, retries+1): try: - callGData(service=emailsettings, function=u'CreateDelegate', throw_errors=[600, 1000, 1001], delegate=use_delegate_address, delegator=delegator) + callGData(emailsettings, u'CreateDelegate', throw_errors=[600, 1000, 1001], delegate=use_delegate_address, delegator=delegator) break except gdata.apps.service.AppsForYourDomainException, e: # 1st check to see if delegation already exists (causes 1000 error on create when using alias) - get_delegates = callGData(service=emailsettings, function=u'GetDelegates', delegator=delegator) + get_delegates = callGData(emailsettings, u'GetDelegates', delegator=delegator) for get_delegate in get_delegates: if get_delegate[u'address'].lower() == delegate_email: # Delegation is already in place print u'That delegation is already in place...' @@ -1183,8 +1183,8 @@ def doDelegates(users): sys.exit(0) # Emulate functionality of duplicate delegation between users in same domain, returning clean # Now check if either user account is suspended or requires password change cd = buildGAPIObject(u'directory') - delegate_user_details = callGAPI(service=cd.users(), function=u'get', userKey=delegate_email) - delegator_user_details = callGAPI(service=cd.users(), function=u'get', userKey=delegator_email) + delegate_user_details = callGAPI(cd.users(), u'get', userKey=delegate_email) + delegator_user_details = callGAPI(cd.users(), u'get', userKey=delegator_email) if delegate_user_details[u'suspended'] == True: sys.stderr.write(u'ERROR: User %s is suspended. You must unsuspend for delegation.\n' % delegate_email) if delete_alias: @@ -1244,7 +1244,7 @@ def getDelegates(users): else: emailsettings.domain = GC_Values[GC_DOMAIN] sys.stderr.write(u"Getting delegates for %s...\n" % (user + '@' + emailsettings.domain)) - delegates = callGData(service=emailsettings, function=u'GetDelegates', soft_errors=True, delegator=user) + delegates = callGData(emailsettings, u'GetDelegates', soft_errors=True, delegator=user) try: for delegate in delegates: if csv_format: @@ -1273,7 +1273,7 @@ def deleteDelegate(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Deleting %s delegate access to %s (%s of %s)" % (delegate, user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'DeleteDelegate', delegate=delegate, delegator=user) + callGData(emailsettings, u'DeleteDelegate', delegate=delegate, delegator=user) def doAddCourseParticipant(): croom = buildGAPIObject(u'classroom') @@ -1296,7 +1296,7 @@ def doAddCourseParticipant(): print u'ERROR: %s is not a valid argument to "gam course ID add"' % participant_type sys.exit(2) body = {body_attribute: new_id} - callGAPI(service=service, function=u'create', courseId=courseId, body=body) + callGAPI(service, u'create', courseId=courseId, body=body) if courseId[:2] == u'd:': courseId = courseId[2:] if new_id[:2] == u'd:': @@ -1325,7 +1325,7 @@ def doSyncCourseParticipants(): gam_commands.append([u'course', courseId, u'add', participant_type, add_email]) for remove_email in to_remove: gam_commands.append([u'course', courseId, u'remove', participant_type, remove_email]) - run_batch(items=gam_commands) + run_batch(gam_commands) def doDelCourseParticipant(): croom = buildGAPIObject(u'classroom') @@ -1349,7 +1349,7 @@ def doDelCourseParticipant(): else: print u'ERROR: %s is not a valid argument to "gam course ID delete"' % participant_type sys.exit(2) - callGAPI(service=service, function=u'delete', courseId=courseId, **kwargs) + callGAPI(service, u'delete', courseId=courseId, **kwargs) if courseId[:2] == u'd:': courseId = courseId[2:] if remove_id[:2] == u'd:': @@ -1361,7 +1361,7 @@ def doDelCourse(): courseId = sys.argv[3] if not courseId.isdigit(): courseId = u'd:%s' % courseId - callGAPI(service=croom.courses(), function=u'delete', id=courseId) + callGAPI(croom.courses(), u'delete', id=courseId) print u'Deleted Course %s' % courseId def doUpdateCourse(): @@ -1398,14 +1398,14 @@ def doUpdateCourse(): sys.exit(2) updateMask = u','.join(body.keys()) body[u'id'] = courseId - result = callGAPI(service=croom.courses(), function=u'patch', id=courseId, body=body, updateMask=updateMask) + result = callGAPI(croom.courses(), u'patch', id=courseId, body=body, updateMask=updateMask) print u'Updated Course %s' % result[u'id'] def doCreateDomain(): cd = buildGAPIObject(u'directory') domain_name = sys.argv[3] body = {u'domainName': domain_name} - callGAPI(service=cd.domains(), function=u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) + callGAPI(cd.domains(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) print u'Added domain %s' % domain_name def doCreateDomainAlias(): @@ -1413,7 +1413,7 @@ def doCreateDomainAlias(): body = {} body[u'domainAliasName'] = sys.argv[3] body[u'parentDomainName'] = sys.argv[4] - callGAPI(service=cd.domainAliases(), function=u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) + callGAPI(cd.domainAliases(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doUpdateDomain(): cd = buildGAPIObject(u'directory') @@ -1427,7 +1427,7 @@ def doUpdateDomain(): else: print u'ERROR: %s is not a valid argument for "gam update domain"' % sys.argv[i] sys.exit(2) - callGAPI(service=cd.customers(), function=u'update', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) + callGAPI(cd.customers(), u'update', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) print u'%s is now the primary domain.' % domain_name def doGetDomainInfo(): @@ -1436,7 +1436,7 @@ def doGetDomainInfo(): return cd = buildGAPIObject(u'directory') domainName = sys.argv[3] - result = callGAPI(service=cd.domains(), function=u'get', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) + result = callGAPI(cd.domains(), u'get', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) if u'creationTime' in result: result[u'creationTime'] = unicode(datetime.datetime.fromtimestamp(int(result[u'creationTime'])/1000)) if u'domainAliases' in result: @@ -1448,14 +1448,14 @@ def doGetDomainInfo(): def doGetDomainAliasInfo(): cd = buildGAPIObject(u'directory') alias = sys.argv[3] - result = callGAPI(service=cd.domainAliases(), function=u'get', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=alias) + result = callGAPI(cd.domainAliases(), u'get', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=alias) if u'creationTime' in result: result[u'creationTime'] = unicode(datetime.datetime.fromtimestamp(int(result[u'creationTime'])/1000)) print_json(None, result) def doGetCustomerInfo(): cd = buildGAPIObject(u'directory') - customer_info = callGAPI(service=cd.customers(), function=u'get', customerKey=GC_Values[GC_CUSTOMER_ID]) + customer_info = callGAPI(cd.customers(), u'get', customerKey=GC_Values[GC_CUSTOMER_ID]) print_json(None, customer_info) def doUpdateCustomer(): @@ -1495,25 +1495,25 @@ def doUpdateCustomer(): else: print u'ERROR: %s is not a valid argument for "gam update customer"' % myarg sys.exit(2) - callGAPI(service=cd.customers(), function=u'update', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) + callGAPI(cd.customers(), u'update', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) print u'Updated customer' def doDelDomain(): cd = buildGAPIObject(u'directory') domainName = sys.argv[3] - callGAPI(service=cd.domains(), function=u'delete', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) + callGAPI(cd.domains(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) def doDelDomainAlias(): cd = buildGAPIObject(u'directory') domainAliasName = sys.argv[3] - callGAPI(service=cd.domainAliases(), function=u'delete', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=domainAliasName) + callGAPI(cd.domainAliases(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=domainAliasName) def doPrintDomains(): cd = buildGAPIObject(u'directory') titles = [] domains_attributes = [{}] todrive = False - domains = callGAPI(service=cd.domains(), function=u'list', customer=GC_Values[GC_CUSTOMER_ID]) + domains = callGAPI(cd.domains(), u'list', customer=GC_Values[GC_CUSTOMER_ID]) i = 3 while i < len(sys.argv): if sys.argv[i].lower() == u'todrive': @@ -1560,7 +1560,7 @@ def doDelAdmin(): cd = buildGAPIObject(u'directory') roleAssignmentId = sys.argv[3] print u'Deleting Admin Role Assignment %s' % roleAssignmentId - callGAPI(service=cd.roleAssignments(), function=u'delete', + callGAPI(cd.roleAssignments(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], roleAssignmentId=roleAssignmentId) def doCreateAdmin(): @@ -1571,7 +1571,7 @@ def doCreateAdmin(): body[u'assignedTo'] = user[4:] else: print user[:3] - body[u'assignedTo'] = callGAPI(service=cd.users(), function=u'get', + body[u'assignedTo'] = callGAPI(cd.users(), u'get', userKey=user, projection=u'basic', fields=u'id')[u'id'] role = sys.argv[4] if role[:4].lower() == u'uid:': @@ -1594,7 +1594,7 @@ def doCreateAdmin(): else: if orgUnit[0] == u'/': orgUnit = orgUnit[1:] - body[u'orgUnitId'] = callGAPI(service=cd.orgunits(), function=u'get', + body[u'orgUnitId'] = callGAPI(cd.orgunits(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnit, fields=u'orgUnitId')[u'orgUnitId'][3:] if body[u'scopeType'] == u'CUSTOMER': @@ -1602,20 +1602,20 @@ def doCreateAdmin(): else: scope = orgUnit print u'Giving %s admin role %s for %s' % (user, role, scope) - callGAPI(service=cd.roleAssignments(), function=u'insert', + callGAPI(cd.roleAssignments(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doPrintAdminRoles(): cd = buildGAPIObject(u'directory') - roles = callGAPIpages(service=cd.roles(), function=u'list', items=u'items', - customer=GC_Values[GC_CUSTOMER_ID]) + roles = callGAPIpages(cd.roles(), u'list', u'items', + customer=GC_Values[GC_CUSTOMER_ID]) roles_attrib = [{}] for role in roles: role_attrib = {} for key, value in role.items(): if key in [u'kind', u'etag', u'etags']: continue - if not isinstance( value, (str, unicode, bool)): + if not isinstance(value, (str, unicode, bool)): continue if key not in roles_attrib[0]: roles_attrib[0][key] = key @@ -1649,7 +1649,7 @@ def doPrintAdmins(): else: print u'ERROR: %s is not a valid argument for "gam print admins".' % sys.argv[i] sys.exit(2) - admins = callGAPIpages(service=cd.roleAssignments(), function=u'list', + admins = callGAPIpages(cd.roleAssignments(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID], userKey=userKey, roleId=roleId, maxResults=200) admins_attrib = [{}] for admin in admins: @@ -1680,7 +1680,7 @@ def doPrintAdmins(): def buildOrgUnitIdToNameMap(): cd = buildGAPIObject(u'directory') - result = callGAPI(service=cd.orgunits(), function=u'list', + result = callGAPI(cd.orgunits(), u'list', customerId=GC_Values[GC_CUSTOMER_ID], fields=u'organizationUnits(orgUnitPath,orgUnitId)') GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME] = {} @@ -1694,7 +1694,7 @@ def orgunit_from_orgunitid(orgunitid): def buildRoleIdToNameToIdMap(): cd = buildGAPIObject(u'directory') - result = callGAPIpages(service=cd.roles(), function=u'list', items=u'items', + result = callGAPIpages(cd.roles(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,items(roleId,roleName)', maxResults=100) @@ -1716,7 +1716,7 @@ def roleid_from_role(role): def buildUserIdToNameMap(): cd = buildGAPIObject(u'directory') - result = callGAPIpages(service=cd.users(), function=u'list', items=u'users', + result = callGAPIpages(cd.users(), u'list', u'users', customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(id,primaryEmail)', maxResults=GC_Values[GC_USER_MAX_RESULTS]) @@ -1738,7 +1738,7 @@ def appID2app(dt, appID): for serviceName, serviceID in SERVICE_NAME_TO_ID_MAP.items(): if appID == serviceID: return serviceName - online_services = callGAPIpages(service=dt.applications(), function=u'list', items=u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + online_services = callGAPIpages(dt.applications(), u'list', u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for online_service in online_services: if appID == online_service[u'id']: return online_service[u'name'] @@ -1757,7 +1757,7 @@ def app2appID(dt, app): serviceName = app.lower() if serviceName in SERVICE_NAME_CHOICES_MAP: return (SERVICE_NAME_CHOICES_MAP[serviceName], SERVICE_NAME_TO_ID_MAP[SERVICE_NAME_CHOICES_MAP[serviceName]]) - online_services = callGAPIpages(service=dt.applications(), function=u'list', items=u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + online_services = callGAPIpages(dt.applications(), u'list', u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for online_service in online_services: if serviceName == online_service[u'name'].lower(): return online_service[u'id'] @@ -1771,7 +1771,7 @@ def convertToUserID(user): if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) try: - return callGAPI(service=cd.users(), function=u'get', throw_reasons=[u'notFound'], userKey=user, fields=u'id')[u'id'] + return callGAPI(cd.users(), u'get', throw_reasons=[u'notFound'], userKey=user, fields=u'id')[u'id'] except googleapiclient.errors.HttpError: print u'ERROR: no such user %s' % user sys.exit(3) @@ -1779,7 +1779,7 @@ def convertToUserID(user): def convertUserIDtoEmail(uid): cd = buildGAPIObject(u'directory') try: - return callGAPI(service=cd.users(), function=u'get', throw_reasons=[u'notFound'], userKey=uid, fields=u'primaryEmail')[u'primaryEmail'] + return callGAPI(cd.users(), u'get', throw_reasons=[u'notFound'], userKey=uid, fields=u'primaryEmail')[u'primaryEmail'] except googleapiclient.errors.HttpError: print u'ERROR: no such user %s' % id sys.exit(3) @@ -1802,12 +1802,12 @@ def doCreateDataTranfer(): if u'applicationDataTransferParams' not in body[u'applicationDataTransfers'][0]: body[u'applicationDataTransfers'][0][u'applicationTransferParams'] = [] body[u'applicationDataTransfers'][0][u'applicationTransferParams'].append({u'key': key, u'value': parameters[key]}) - result = callGAPI(service=dt.transfers(), function=u'insert', body=body, fields=u'id')[u'id'] + result = callGAPI(dt.transfers(), u'insert', body=body, fields=u'id')[u'id'] print u'Submitted request id %s to transfer %s from %s to %s' % (result, service, old_owner, new_owner) def doPrintTransferApps(): dt = buildGAPIObject(u'datatransfer') - apps = callGAPIpages(service=dt.applications(), function=u'list', items=u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + apps = callGAPIpages(dt.applications(), u'list', u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for app in apps: print_json(None, app) print @@ -1836,8 +1836,8 @@ def doPrintDataTransfers(): print u'ERROR: %s is not a valid argument for "gam print transfers"' % sys.argv[i] sys.exit(2) transfers_attributes = [{}] - transfers = callGAPIpages(service=dt.transfers(), function=u'list', - items=u'dataTransfers', customerId=GC_Values[GC_CUSTOMER_ID], status=status, + transfers = callGAPIpages(dt.transfers(), u'list', u'items', + u'dataTransfers', customerId=GC_Values[GC_CUSTOMER_ID], status=status, newOwnerUserId=newOwnerUserId, oldOwnerUserId=oldOwnerUserId) for transfer in transfers: for i in range(0, len(transfer[u'applicationDataTransfers'])): @@ -1861,7 +1861,7 @@ def doPrintDataTransfers(): def doGetDataTransferInfo(): dt = buildGAPIObject(u'datatransfer') dtId = sys.argv[3] - transfer = callGAPI(service=dt.transfers(), function=u'get', dataTransferId=dtId) + transfer = callGAPI(dt.transfers(), u'get', dataTransferId=dtId) print u'Old Owner: %s' % convertUserIDtoEmail(transfer[u'oldOwnerUserId']) print u'New Owner: %s' % convertUserIDtoEmail(transfer[u'newOwnerUserId']) print u'Request Time: %s' % transfer[u'requestTime'] @@ -1915,7 +1915,7 @@ def doCreateCourse(): body['ownerId'] = u'me' if not u'name' in body: body['name'] = u'Unknown Course' - result = callGAPI(service=croom.courses(), function=u'create', body=body) + result = callGAPI(croom.courses(), u'create', body=body) print u'Created course %s' % result[u'id'] def doGetCourseInfo(): @@ -1923,12 +1923,12 @@ def doGetCourseInfo(): courseId = sys.argv[3] if not courseId.isdigit(): courseId = u'd:%s' % courseId - info = callGAPI(service=croom.courses(), function=u'get', id=courseId) + info = callGAPI(croom.courses(), u'get', id=courseId) print_json(None, info) - teachers = callGAPIpages(service=croom.courses().teachers(), function=u'list', items=u'teachers', courseId=courseId) - students = callGAPIpages(service=croom.courses().students(), function=u'list', items=u'students', courseId=courseId) + teachers = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', courseId=courseId) + students = callGAPIpages(croom.courses().students(), u'list', u'students', courseId=courseId) try: - aliases = callGAPIpages(service=croom.courses().aliases(), function=u'list', items=u'aliases', throw_reasons=[u'notImplemented'], courseId=courseId) + aliases = callGAPIpages(croom.courses().aliases(), u'list', u'aliases', throw_reasons=[u'notImplemented'], courseId=courseId) except googleapiclient.errors.HttpError: aliases = [] if aliases: @@ -1976,7 +1976,7 @@ def doPrintCourses(): sys.exit(2) sys.stderr.write(u'Retrieving courses for organization (may take some time for large accounts)...\n') page_message = u'Got %%num_items%% courses...\n' - all_courses = callGAPIpages(service=croom.courses(), function=u'list', items=u'courses', page_message=page_message, teacherId=teacherId, studentId=studentId) + all_courses = callGAPIpages(croom.courses(), u'list', u'courses', page_message=page_message, teacherId=teacherId, studentId=studentId) for course in all_courses: croom_attributes.append(flatten_json(course)) for item in croom_attributes[-1]: @@ -1990,7 +1990,7 @@ def doPrintCourses(): i = 1 for course in croom_attributes[1:]: sys.stderr.write('Getting aliases for course %s (%s/%s)\n' % (course[u'id'], i, num_courses)) - course_aliases = callGAPIpages(service=croom.courses().aliases(), function=u'list', items=u'aliases', courseId=course[u'id']) + course_aliases = callGAPIpages(croom.courses().aliases(), u'list', u'aliases', courseId=course[u'id']) my_aliases = [] for alias in course_aliases: my_aliases.append(alias[u'alias'][2:]) @@ -2029,21 +2029,21 @@ def doPrintCourseParticipants(): sys.stderr.write(u'Retrieving courses for organization (may take some time for large accounts)...\n') if len(courses) == 0: page_message = u'Got %%num_items%% courses...\n' - all_courses = callGAPIpages(service=croom.courses(), function=u'list', items=u'courses', page_message=page_message, teacherId=teacherId, studentId=studentId) + all_courses = callGAPIpages(croom.courses(), u'list', u'courses', page_message=page_message, teacherId=teacherId, studentId=studentId) for course in all_courses: courses.append(course[u'id']) else: all_courses = [] for course in courses: - all_courses.append(callGAPI(service=croom.courses(), function=u'get', id=course)) + all_courses.append(callGAPI(croom.courses(), u'get', id=course)) y = 1 num_courses = len(all_courses) for course in all_courses: course_id = course[u'id'] teacher_message = u' got %%%%num_items%%%% teachers for course %s (%s/%s)' % (course_id, y, num_courses) student_message = u' got %%%%num_items%%%% students for course %s (%s/%s)' % (course_id, y, num_courses) - teachers = callGAPIpages(service=croom.courses().teachers(), function=u'list', items=u'teachers', page_message=teacher_message, courseId=course_id) - students = callGAPIpages(service=croom.courses().students(), function=u'list', items=u'students', page_message=student_message, courseId=course_id) + teachers = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', page_message=teacher_message, courseId=course_id) + students = callGAPIpages(croom.courses().students(), u'list', u'students', page_message=student_message, courseId=course_id) for teacher in teachers: participant = flatten_json(teacher) participant[u'courseId'] = course_id @@ -2146,7 +2146,7 @@ def doPrintPrintJobs(): sys.exit(2) if sortorder and descending: sortorder = PRINTJOB_DESCENDINGORDER_MAP[sortorder] - jobs = callGAPI(service=cp.jobs(), function=u'list', q=query, status=status, sortorder=sortorder, printerid=printerid, owner=owner) + jobs = callGAPI(cp.jobs(), u'list', q=query, status=status, sortorder=sortorder, printerid=printerid, owner=owner) checkCloudPrintResult(jobs) for job in jobs[u'jobs']: createTime = int(job[u'createTime'])/1000 @@ -2195,7 +2195,7 @@ def doPrintPrinters(): else: print u'ERROR: %s is not a valid argument for "gam print printers"' % sys.argv[i] sys.exit(2) - printers = callGAPI(service=cp.printers(), function=u'list', q=query, type=printer_type, connection_status=connection_status, extra_fields=extra_fields) + printers = callGAPI(cp.printers(), u'list', q=query, type=printer_type, connection_status=connection_status, extra_fields=extra_fields) checkCloudPrintResult(printers) for printer in printers[u'printers']: createTime = int(printer[u'createTime'])/1000 @@ -2248,7 +2248,7 @@ def changeCalendarAttendees(users): cal = buildGAPIServiceObject(u'calendar', user) page_token = None while True: - events_page = callGAPI(service=cal.events(), function=u'list', calendarId=user, pageToken=page_token, timeMin=start_date, timeMax=end_date, showDeleted=False, showHiddenInvitations=False) + events_page = callGAPI(cal.events(), u'list', calendarId=user, pageToken=page_token, timeMin=start_date, timeMax=end_date, showDeleted=False, showHiddenInvitations=False) print u'Got %s items' % len(events_page.get(u'items', [])) for event in events_page.get(u'items', []): if event[u'status'] == u'cancelled': @@ -2284,7 +2284,7 @@ def changeCalendarAttendees(users): body[u'attendees'] = event[u'attendees'] print u'UPDATING %s' % event_summary if do_it: - callGAPI(service=cal.events(), function=u'patch', calendarId=user, eventId=event[u'id'], sendNotifications=False, body=body) + callGAPI(cal.events(), u'patch', calendarId=user, eventId=event[u'id'], sendNotifications=False, body=body) else: print u' not pulling the trigger.' #else: @@ -2303,7 +2303,7 @@ def deleteCalendar(users): if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) cal = buildGAPIServiceObject(u'calendar', user) - callGAPI(service=cal.calendarList(), function=u'delete', calendarId=calendarId) + callGAPI(cal.calendarList(), u'delete', calendarId=calendarId) def addCalendar(users): cal = buildGAPIServiceObject(u'calendar', users[0]) @@ -2371,7 +2371,7 @@ def addCalendar(users): user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) print u"Subscribing %s to %s calendar (%s of %s)" % (user, body['id'], i, count) cal = buildGAPIServiceObject(u'calendar', user) - callGAPI(service=cal.calendarList(), function=u'insert', body=body, colorRgbFormat=colorRgbFormat) + callGAPI(cal.calendarList(), u'insert', body=body, colorRgbFormat=colorRgbFormat) i += 1 def updateCalendar(users): @@ -2436,12 +2436,12 @@ def updateCalendar(users): for user in users: print u"Updating %s's subscription to calendar %s (%s of %s)" % (user, calendarId, i, count) cal = buildGAPIServiceObject(u'calendar', user) - callGAPI(service=cal.calendarList(), function=u'update', calendarId=calendarId, body=body, colorRgbFormat=colorRgbFormat) + callGAPI(cal.calendarList(), u'update', calendarId=calendarId, body=body, colorRgbFormat=colorRgbFormat) def doPrinterShowACL(): cp = buildGAPIObject(u'cloudprint') show_printer = sys.argv[2] - printer_info = callGAPI(service=cp.printers(), function=u'get', printerid=show_printer) + printer_info = callGAPI(cp.printers(), u'get', printerid=show_printer) checkCloudPrintResult(printer_info) for acl in printer_info[u'printers'][0][u'access']: if u'key' in acl: @@ -2463,7 +2463,7 @@ def doPrinterAddACL(): skip_notification = None elif scope.find(u'@') == -1: scope = u'/hd/domain/%s' % scope - result = callGAPI(service=cp.printers(), function=u'share', printerid=printer, role=role, scope=scope, public=public, skip_notification=skip_notification) + result = callGAPI(cp.printers(), u'share', printerid=printer, role=role, scope=scope, public=public, skip_notification=skip_notification) checkCloudPrintResult(result) who = scope if who == None: @@ -2481,7 +2481,7 @@ def doPrinterDelACL(): scope = None elif scope.find(u'@') == -1: scope = u'/hd/domain/%s' % scope - result = callGAPI(service=cp.printers(), function=u'unshare', printerid=printer, scope=scope, public=public) + result = callGAPI(cp.printers(), u'unshare', printerid=printer, scope=scope, public=public) checkCloudPrintResult(result) who = scope if who == None: @@ -2587,7 +2587,7 @@ def doPrintJobFetch(): sys.exit(2) if sortorder and descending: sortorder = PRINTJOB_DESCENDINGORDER_MAP[sortorder] - result = callGAPI(service=cp.jobs(), function=u'list', q=query, status=status, sortorder=sortorder, printerid=printerid, owner=owner) + result = callGAPI(cp.jobs(), u'list', q=query, status=status, sortorder=sortorder, printerid=printerid, owner=owner) if u'errorCode' in result and result[u'errorCode'] == 413: print u'No print jobs.' sys.exit(0) @@ -2610,15 +2610,15 @@ def doPrintJobFetch(): fileName = u'%s-%s' % (fileName, jobid) _, content = cp._http.request(uri=fileUrl, method='GET') if writeFile(fileName, content, continueOnError=True): -# ticket = callGAPI(service=cp.jobs(), function=u'getticket', jobid=jobid, use_cjt=True) - result = callGAPI(service=cp.jobs(), function=u'update', jobid=jobid, semantic_state_diff=ssd) +# ticket = callGAPI(cp.jobs(), u'getticket', jobid=jobid, use_cjt=True) + result = callGAPI(cp.jobs(), u'update', jobid=jobid, semantic_state_diff=ssd) checkCloudPrintResult(result) print u'Printed job %s to %s' % (jobid, fileName) def doDelPrinter(): cp = buildGAPIObject(u'cloudprint') printerid = sys.argv[3] - result = callGAPI(service=cp.printers(), function=u'delete', printerid=printerid) + result = callGAPI(cp.printers(), u'delete', printerid=printerid) checkCloudPrintResult(result) def doGetPrinterInfo(): @@ -2633,7 +2633,7 @@ def doGetPrinterInfo(): else: print u'ERROR: %s is not a valid argument for "gam info printer"' % sys.argv[i] sys.exit(2) - result = callGAPI(service=cp.printers(), function=u'get', printerid=printerid) + result = callGAPI(cp.printers(), u'get', printerid=printerid) checkCloudPrintResult(result) printer_info = result[u'printers'][0] createTime = int(printer_info[u'createTime'])/1000 @@ -2669,7 +2669,7 @@ def doUpdatePrinter(): if not arg_in_item: print u'ERROR: %s is not a valid argument for "gam update printer"' % sys.argv[i] sys.exit(2) - result = callGAPI(service=cp.printers(), function=u'update', printerid=printerid, **kwargs) + result = callGAPI(cp.printers(), u'update', printerid=printerid, **kwargs) checkCloudPrintResult(result) print u'Updated printer %s' % printerid @@ -2705,7 +2705,7 @@ def doPrinterRegister(): form_files = {} body, headers = encode_multipart(form_fields, form_files) #Get the printer first to make sure our OAuth access token is fresh - callGAPI(service=cp.printers(), function=u'list') + callGAPI(cp.printers(), u'list') _, result = cp._http.request(uri='https://www.google.com/cloudprint/register', method='POST', body=body, headers=headers) result = json.loads(result) checkCloudPrintResult(result) @@ -2718,10 +2718,10 @@ def doPrintJobResubmit(): ssd = '''{ "state": {"type": "HELD"} }''' - result = callGAPI(service=cp.jobs(), function=u'update', jobid=jobid, semantic_state_diff=ssd) + result = callGAPI(cp.jobs(), u'update', jobid=jobid, semantic_state_diff=ssd) checkCloudPrintResult(result) - ticket = callGAPI(service=cp.jobs(), function=u'getticket', jobid=jobid, use_cjt=True) - result = callGAPI(service=cp.jobs(), function=u'resubmit', printerid=printerid, jobid=jobid, ticket=ticket) + ticket = callGAPI(cp.jobs(), u'getticket', jobid=jobid, use_cjt=True) + result = callGAPI(cp.jobs(), u'resubmit', printerid=printerid, jobid=jobid, ticket=ticket) checkCloudPrintResult(result) print u'Success resubmitting %s as job %s to printer %s' % (jobid, result[u'job'][u'id'], printerid) @@ -2756,10 +2756,10 @@ def doPrintJobSubmit(): mimetype = u'application/octet-stream' filecontent = readFile(filepath) form_files[u'content'] = {u'filename': content, u'content': filecontent, u'mimetype': mimetype} - #result = callGAPI(service=cp.printers(), function=u'submit', body=body) + #result = callGAPI(cp.printers(), u'submit', body=body) body, headers = encode_multipart(form_fields, form_files) #Get the printer first to make sure our OAuth access token is fresh - callGAPI(service=cp.printers(), function=u'get', printerid=printer) + callGAPI(cp.printers(), u'get', printerid=printer) _, result = cp._http.request(uri='https://www.google.com/cloudprint/submit', method='POST', body=body, headers=headers) checkCloudPrintResult(result) if type(result) is str: @@ -2769,7 +2769,7 @@ def doPrintJobSubmit(): def doDeletePrintJob(): cp = buildGAPIObject(u'cloudprint') job = sys.argv[2] - result = callGAPI(service=cp.jobs(), function=u'delete', jobid=job) + result = callGAPI(cp.jobs(), u'delete', jobid=job) checkCloudPrintResult(result) print u'Print Job %s deleted' % job @@ -2777,7 +2777,7 @@ def doCancelPrintJob(): cp = buildGAPIObject(u'cloudprint') job = sys.argv[2] ssd = '{"state": {"type": "ABORTED", "user_action_cause": {"action_code": "CANCELLED"}}}' - result = callGAPI(service=cp.jobs(), function=u'update', jobid=job, semantic_state_diff=ssd) + result = callGAPI(cp.jobs(), u'update', jobid=job, semantic_state_diff=ssd) checkCloudPrintResult(result) print u'Print Job %s cancelled' % job @@ -2797,7 +2797,7 @@ def doCalendarShowACL(): show_cal = sys.argv[2] if show_cal.find(u'@') == -1: show_cal = u'%s@%s' % (show_cal, GC_Values[GC_DOMAIN]) - acls = callGAPI(service=cal.acl(), function=u'list', calendarId=show_cal) + acls = callGAPI(cal.acl(), u'list', calendarId=show_cal) try: for rule in acls[u'items']: print u' Scope %s - %s' % (rule[u'scope'][u'type'], rule[u'scope'][u'value']) @@ -2852,7 +2852,7 @@ def doCalendarAddACL(calendarId=None, act_as=None, role=None, scope=None, entity body[u'scope'][u'value'] = sys.argv[6].lower() except IndexError: body[u'scope'][u'value'] = GC_Values[GC_DOMAIN] - callGAPI(service=cal.acl(), function=u'insert', calendarId=calendarId, body=body) + callGAPI(cal.acl(), u'insert', calendarId=calendarId, body=body) def doCalendarUpdateACL(): calendarId = sys.argv[2] @@ -2880,7 +2880,7 @@ def doCalendarWipeData(): cal = buildGAPIServiceObject(u'calendar', calendarId) if calendarId.find(u'@') == -1: calendarId = u'%s@%s' % (calendarId, GC_Values[GC_DOMAIN]) - callGAPI(service=cal.calendars(), function=u'clear', calendarId=calendarId) + callGAPI(cal.calendars(), u'clear', calendarId=calendarId) def doCalendarAddEvent(): calendarId = sys.argv[2] @@ -2991,7 +2991,7 @@ def doCalendarAddEvent(): print u'ERROR: %s is not a valid argument for "gam calendar"' % sys.argv[i] sys.exit(2) if not timeZone and u'recurrence' in body: - timeZone = callGAPI(service=cal.calendars(), function=u'get', calendarId=calendarId, fields=u'timeZone')[u'timeZone'] + timeZone = callGAPI(cal.calendars(), u'get', calendarId=calendarId, fields=u'timeZone')[u'timeZone'] if u'recurrence' in body: for a_time in [u'start', u'end']: try: @@ -2999,7 +2999,7 @@ def doCalendarAddEvent(): except KeyError: pass - callGAPI(service=cal.events(), function=u'insert', calendarId=calendarId, sendNotifications=sendNotifications, body=body) + callGAPI(cal.events(), u'insert', calendarId=calendarId, sendNotifications=sendNotifications, body=body) def doProfile(users): @@ -3019,7 +3019,7 @@ def doProfile(users): elif user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) print u'Setting Profile Sharing to %s for %s (%s of %s)' % (body[u'includeInGlobalAddressList'], user, i, count) - callGAPI(service=cd.users(), function=u'patch', soft_errors=True, userKey=user, body=body) + callGAPI(cd.users(), u'patch', soft_errors=True, userKey=user, body=body) i += 1 def showProfile(users): @@ -3031,7 +3031,7 @@ def showProfile(users): user = user[4:] elif user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) - result = callGAPI(service=cd.users(), function=u'get', userKey=user, fields=u'includeInGlobalAddressList') + result = callGAPI(cd.users(), u'get', userKey=user, fields=u'includeInGlobalAddressList') try: print u'User: %s Profile Shared: %s (%s/%s)' % (user, result[u'includeInGlobalAddressList'], i, count) except IndexError: @@ -3069,7 +3069,7 @@ def doPhoto(users): continue image_data = base64.urlsafe_b64encode(image_data) body = {u'photoData': image_data} - callGAPI(service=cd.users().photos(), function=u'update', soft_errors=True, userKey=user, body=body) + callGAPI(cd.users().photos(), u'update', soft_errors=True, userKey=user, body=body) def getPhoto(users): cd = buildGAPIObject(u'directory') @@ -3084,7 +3084,7 @@ def getPhoto(users): print u"Saving photo to %s (%s/%s)" % (filename, i, count) i += 1 try: - photo = callGAPI(service=cd.users().photos(), function=u'get', throw_reasons=[u'notFound'], userKey=user) + photo = callGAPI(cd.users().photos(), u'get', throw_reasons=[u'notFound'], userKey=user) except googleapiclient.errors.HttpError: print u' no photo for %s' % user continue @@ -3107,13 +3107,13 @@ def deletePhoto(users): elif user.find('@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) print u"Deleting photo for %s (%s of %s)" % (user, i, count) - callGAPI(service=cd.users().photos(), function='delete', userKey=user) + callGAPI(cd.users().photos(), 'delete', userKey=user) i += 1 def showCalendars(users): for user in users: cal = buildGAPIServiceObject(u'calendar', user) - feed = callGAPI(service=cal.calendarList(), function=u'list') + feed = callGAPI(cal.calendarList(), u'list') for usercal in feed[u'items']: print u' Name: %s' % usercal['id'] print convertUTF8(u' Summary: %s' % usercal['summary']) @@ -3147,7 +3147,7 @@ def showCalSettings(users): for user in users: for user in users: cal = buildGAPIServiceObject(u'calendar', user) - feed = callGAPI(service=cal.settings(), function='list') + feed = callGAPI(cal.settings(), 'list') for setting in feed[u'items']: print u'%s: %s' % (setting[u'id'], setting[u'value']) @@ -3169,7 +3169,7 @@ def showDriveSettings(users): sys.stderr.write(u'Getting Drive settings for %s (%s of %s)\n' % (user, count, len(users))) count += 1 drive = buildGAPIServiceObject(u'drive', user) - feed = callGAPI(service=drive.about(), function=u'get', soft_errors=True) + feed = callGAPI(drive.about(), u'get', soft_errors=True) if feed == None: continue row = {u'email': user} @@ -3218,7 +3218,7 @@ def doDriveActivity(users): for user in users: activity = buildGAPIServiceObject(u'appsactivity', user) page_message = u'Retrieved %%%%total_items%%%% activities for %s' % user - feed = callGAPIpages(service=activity.activities(), function=u'list', items=u'activities', + feed = callGAPIpages(activity.activities(), u'list', u'activities', page_message=page_message, source=u'drive.google.com', userId=u'me', drive_ancestorId=drive_ancestorId, groupingStrategy=u'none', drive_fileId=drive_fileId, pageSize=GC_Values[GC_ACTIVITY_MAX_RESULTS]) @@ -3233,7 +3233,7 @@ def showDriveFileACL(users): fileId = sys.argv[5] for user in users: drive = buildGAPIServiceObject(u'drive', user) - feed = callGAPI(service=drive.permissions(), function=u'list', fileId=fileId) + feed = callGAPI(drive.permissions(), u'list', fileId=fileId) for permission in feed[u'items']: try: print permission[u'name'] @@ -3255,9 +3255,9 @@ def delDriveFileACL(users): elif permissionId.lower() in [u'anyone']: pass else: - permissionId = callGAPI(service=drive.permissions(), function=u'getIdForEmail', email=permissionId, fields=u'id')[u'id'] + permissionId = callGAPI(drive.permissions(), u'getIdForEmail', email=permissionId, fields=u'id')[u'id'] print u'Removing permission for %s from %s' % (permissionId, fileId) - callGAPI(service=drive.permissions(), function=u'delete', fileId=fileId, permissionId=permissionId) + callGAPI(drive.permissions(), u'delete', fileId=fileId, permissionId=permissionId) def addDriveFileACL(users): fileId = sys.argv[5] @@ -3298,7 +3298,7 @@ def addDriveFileACL(users): sys.exit(2) for user in users: drive = buildGAPIServiceObject(u'drive', user) - result = callGAPI(service=drive.permissions(), function=u'insert', fileId=fileId, sendNotificationEmails=sendNotificationEmails, emailMessage=emailMessage, body=body) + result = callGAPI(drive.permissions(), u'insert', fileId=fileId, sendNotificationEmails=sendNotificationEmails, emailMessage=emailMessage, body=body) print result def updateDriveFileACL(users): @@ -3336,9 +3336,9 @@ def updateDriveFileACL(users): if permissionId[:3].lower() == u'id:': permissionId = permissionId[3:] else: - permissionId = callGAPI(service=drive.permissions(), function=u'getIdForEmail', email=permissionId, fields=u'id')[u'id'] + permissionId = callGAPI(drive.permissions(), u'getIdForEmail', email=permissionId, fields=u'id')[u'id'] print u'updating permissions for %s to file %s' % (permissionId, fileId) - result = callGAPI(service=drive.permissions(), function=u'patch', fileId=fileId, permissionId=permissionId, transferOwnership=transferOwnership, body=body) + result = callGAPI(drive.permissions(), u'patch', fileId=fileId, permissionId=permissionId, transferOwnership=transferOwnership, body=body) print result def showDriveFiles(users): @@ -3431,7 +3431,7 @@ def showDriveFiles(users): sys.exit(2) sys.stderr.write(u'Getting files for %s...\n' % user) page_message = u' got %%%%total_items%%%% files for %s...\n' % user - feed = callGAPIpages(service=drive.files(), function=u'list', + feed = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, soft_errors=True, q=query, fields=fields, maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) for f_file in feed: @@ -3471,7 +3471,7 @@ def showDriveFiles(users): def doDriveSearch(drive, query=None): print u'Searching for files with query: "%s"...' % query page_message = u' got %%total_items%% files...\n' - files = callGAPIpages(service=drive.files(), function=u'list', + files = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, q=query, fields=u'nextPageToken,items(id)', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) ids = list() @@ -3509,7 +3509,7 @@ def deleteDriveFile(users): print u'trashing %s for %s (%s of %s)' % (fileId, user, i, len(file_ids)) else: print u'purging %s for %s (%s of %s)' % (fileId, user, i, len(file_ids)) - callGAPI(service=drive.files(), function=function, fileId=fileId) + callGAPI(drive.files(), function, fileId=fileId) def printDriveFolderContents(feed, folderId, indent): for f_file in feed: @@ -3525,10 +3525,10 @@ def showDriveFileTree(users): if user.find(u'@') == -1: print u'ERROR: got %s, expected a full email address' % user sys.exit(2) - root_folder = callGAPI(service=drive.about(), function=u'get', fields=u'rootFolderId')[u'rootFolderId'] + root_folder = callGAPI(drive.about(), u'get', fields=u'rootFolderId')[u'rootFolderId'] sys.stderr.write(u'Getting all files for %s...\n' % user) page_message = u' got %%%%total_items%%%% files for %s...\n' % user - feed = callGAPIpages(service=drive.files(), function=u'list', page_message=page_message, + feed = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, fields=u'items(id,title,parents(id),mimeType),nextPageToken', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) printDriveFolderContents(feed, root_folder, 0) @@ -3543,15 +3543,15 @@ def deleteEmptyDriveFolders(users): while deleted_empty: sys.stderr.write(u'Getting folders for %s...\n' % user) page_message = u' got %%%%total_items%%%% folders for %s...\n' % user - feed = callGAPIpages(service=drive.files(), function=u'list', page_message=page_message, + feed = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, q=query, fields=u'items(title,id),nextPageToken', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) deleted_empty = False for folder in feed: - children = callGAPI(service=drive.children(), function=u'list', + children = callGAPI(drive.children(), u'list', folderId=folder[u'id'], fields=u'items(id)', maxResults=1) if not u'items' in children or len(children[u'items']) == 0: print convertUTF8(u' deleting empty folder %s...' % folder[u'title']) - callGAPI(service=drive.files(), function=u'delete', fileId=folder[u'id']) + callGAPI(drive.files(), u'delete', fileId=folder[u'id']) deleted_empty = True else: print convertUTF8(u' not deleting folder %s because it contains at least 1 item (%s)' % (folder[u'title'], children[u'items'][0][u'id'])) @@ -3701,15 +3701,15 @@ def doUpdateDriveFile(users): for fileId in fileIds: if operation == u'update': if media_body: - result = callGAPI(service=drive.files(), function=u'update', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields='id') + result = callGAPI(drive.files(), u'update', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields='id') else: - result = callGAPI(service=drive.files(), function=u'patch', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, body=body, fields='id,labels') + result = callGAPI(drive.files(), u'patch', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, body=body, fields='id,labels') try: print u'Successfully updated %s drive file with content from %s' % (result[u'id'], local_filename) except UnboundLocalError: print u'Successfully updated drive file/folder ID %s' % (result[u'id']) else: - result = callGAPI(service=drive.files(), function=u'copy', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, body=body, fields=u'id,labels') + result = callGAPI(drive.files(), u'copy', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, body=body, fields=u'id,labels') print u'Successfully copied %s to %s' % (fileId, result[u'id']) def createDriveFile(users): @@ -3812,7 +3812,7 @@ def createDriveFile(users): body[u'parents'].append({u'id': a_parent}) if local_filepath: media_body = googleapiclient.http.MediaFileUpload(local_filepath, mimetype=mimetype, resumable=True) - result = callGAPI(service=drive.files(), function=u'insert', convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields='id') + result = callGAPI(drive.files(), u'insert', convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields='id') try: print u'Successfully uploaded %s to Drive file ID %s' % (local_filename, result[u'id']) except UnboundLocalError: @@ -3880,7 +3880,7 @@ def downloadDriveFile(users): i = 0 for fileId in fileIds: extension = None - result = callGAPI(service=drive.files(), function=u'get', fileId=fileId, fields=u'fileSize,title,mimeType,downloadUrl,exportLinks') + result = callGAPI(drive.files(), u'get', fileId=fileId, fields=u'fileSize,title,mimeType,downloadUrl,exportLinks') if result[u'mimeType'] == u'application/vnd.google-apps.folder': print convertUTF8(u'Skipping download of folder %s' % result[u'title']) continue @@ -3937,7 +3937,7 @@ def showDriveFileInfo(users): for user in users: fileId = sys.argv[5] drive = buildGAPIServiceObject(u'drive', user) - feed = callGAPI(service=drive.files(), function=u'get', fileId=fileId) + feed = callGAPI(drive.files(), u'get', fileId=fileId) for setting in feed: if setting == u'kind': continue @@ -3976,7 +3976,7 @@ def transferSecCals(users): sys.exit(2) for user in users: source_cal = buildGAPIServiceObject(u'calendar', user) - source_calendars = callGAPIpages(service=source_cal.calendarList(), function=u'list', minAccessRole=u'owner', showHidden=True, fields=u'items(id),nextPageToken') + source_calendars = callGAPIpages(source_cal.calendarList(), u'list', u'items', minAccessRole=u'owner', showHidden=True, fields=u'items(id),nextPageToken') for source_cal in source_calendars: if source_cal[u'id'].find(u'@group.calendar.google.com') != -1: doCalendarAddACL(calendarId=source_cal[u'id'], act_as=user, role=u'owner', scope=u'user', entity=target_user) @@ -3995,12 +3995,12 @@ def transferDriveFiles(users): print u'ERROR: %s is not a valid argument for "gam transfer drive"' % sys.argv[i] sys.exit(2) target_drive = buildGAPIServiceObject(u'drive', target_user) - target_about = callGAPI(service=target_drive.about(), function=u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId') + target_about = callGAPI(target_drive.about(), u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId') target_drive_free = int(target_about[u'quotaBytesTotal']) - int(target_about[u'quotaBytesUsed']) for user in users: counter = 0 source_drive = buildGAPIServiceObject(u'drive', user) - source_about = callGAPI(service=source_drive.about(), function=u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId, permissionId') + source_about = callGAPI(source_drive.about(), u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId, permissionId') source_drive_size = int(source_about[u'quotaBytesUsed']) if target_drive_free < source_drive_size: systemErrorExit(4, MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE.format(source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024)) @@ -4010,7 +4010,7 @@ def transferDriveFiles(users): source_permissionid = source_about[u'permissionId'] print u"Getting file list for source user: %s..." % user page_message = u' got %%total_items%% files\n' - source_drive_files = callGAPIpages(service=source_drive.files(), function=u'list', page_message=page_message, + source_drive_files = callGAPIpages(source_drive.files(), u'list', u'items', page_message=page_message, q=u"'me' in owners and trashed = false", fields=u'items(id,parents,mimeType),nextPageToken') all_source_file_ids = [] for source_drive_file in source_drive_files: @@ -4018,7 +4018,7 @@ def transferDriveFiles(users): total_count = len(source_drive_files) print u"Getting folder list for target user: %s..." % target_user page_message = u' got %%total_items%% folders\n' - target_folders = callGAPIpages(service=target_drive.files(), function=u'list', page_message=page_message, + target_folders = callGAPIpages(target_drive.files(), u'list', u'items', page_message=page_message, q=u"'me' in owners and mimeType = 'application/vnd.google-apps.folder'", fields=u'items(id,title),nextPageToken') got_top_folder = False all_target_folder_ids = [] @@ -4028,7 +4028,7 @@ def transferDriveFiles(users): target_top_folder = target_folder[u'id'] got_top_folder = True if not got_top_folder: - create_folder = callGAPI(service=target_drive.files(), function=u'insert', body={u'title': u'%s old files' % user, u'mimeType': u'application/vnd.google-apps.folder'}, fields=u'id') + create_folder = callGAPI(target_drive.files(), u'insert', body={u'title': u'%s old files' % user, u'mimeType': u'application/vnd.google-apps.folder'}, fields=u'id') target_top_folder = create_folder[u'id'] transferred_files = [] while True: # we loop thru, skipping files until all of their parents are done @@ -4053,7 +4053,7 @@ def transferDriveFiles(users): counter += 1 print u'Changing owner for file %s (%s/%s)' % (drive_file[u'id'], counter, total_count) body = {u'role': u'owner', u'type': u'user', u'value': target_user} - callGAPI(service=source_drive.permissions(), function=u'insert', soft_errors=True, fileId=file_id, sendNotificationEmails=False, body=body) + callGAPI(source_drive.permissions(), u'insert', soft_errors=True, fileId=file_id, sendNotificationEmails=False, body=body) target_parents = [] for parent in source_parents: try: @@ -4063,9 +4063,9 @@ def transferDriveFiles(users): target_parents.append({u'id': parent[u'id']}) except TypeError: pass - callGAPI(service=target_drive.files(), function=u'patch', soft_errors=True, retry_reasons=[u'notFound'], fileId=file_id, body={u'parents': target_parents}) + callGAPI(target_drive.files(), u'patch', soft_errors=True, retry_reasons=[u'notFound'], fileId=file_id, body={u'parents': target_parents}) if remove_source_user: - callGAPI(service=target_drive.permissions(), function=u'delete', soft_errors=True, fileId=file_id, permissionId=source_permissionid) + callGAPI(target_drive.permissions(), u'delete', soft_errors=True, fileId=file_id, permissionId=source_permissionid) if not skipped_files: break @@ -4088,7 +4088,7 @@ def doImap(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting IMAP Access to %s for %s (%s of %s)" % (str(enable), user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateImap', soft_errors=True, username=user, enable=enable) + callGData(emailsettings, u'UpdateImap', soft_errors=True, username=user, enable=enable) def getImap(users): emailsettings = getEmailSettingsObject() @@ -4100,7 +4100,7 @@ def getImap(users): user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - imapsettings = callGData(service=emailsettings, function=u'GetImap', soft_errors=True, username=user) + imapsettings = callGData(emailsettings, u'GetImap', soft_errors=True, username=user) try: print u'User %s IMAP Enabled:%s (%s of %s)' % (user+u'@'+emailsettings.domain, imapsettings[u'enable'], i, count) except TypeError: @@ -4155,9 +4155,9 @@ def doLicense(users, operation): if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) if operation == u'delete': - callGAPI(service=lic.licenseAssignments(), function=operation, soft_errors=True, productId=productId, skuId=skuId, userId=user) + callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=skuId, userId=user) elif operation == u'insert': - callGAPI(service=lic.licenseAssignments(), function=operation, soft_errors=True, productId=productId, skuId=skuId, body={u'userId': user}) + callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=skuId, body={u'userId': user}) elif operation == u'patch': try: old_sku = sys.argv[6] @@ -4167,7 +4167,7 @@ def doLicense(users, operation): print u'ERROR: You need to specify the user\'s old SKU as the last argument' sys.exit(2) _, old_sku = getProductAndSKU(old_sku) - callGAPI(service=lic.licenseAssignments(), function=operation, soft_errors=True, productId=productId, skuId=old_sku, userId=user, body={u'skuId': skuId}) + callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=old_sku, userId=user, body={u'skuId': skuId}) def doPop(users): if sys.argv[4].lower() in true_values: @@ -4220,7 +4220,7 @@ def doPop(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting POP Access to %s for %s (%s of %s)" % (str(enable), user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdatePop', soft_errors=True, username=user, enable=enable, enable_for=enable_for, action=action) + callGData(emailsettings, u'UpdatePop', soft_errors=True, username=user, enable=enable, enable_for=enable_for, action=action) def getPop(users): emailsettings = getEmailSettingsObject() @@ -4230,7 +4230,7 @@ def getPop(users): user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - popsettings = callGData(service=emailsettings, function=u'GetPop', soft_errors=True, username=user) + popsettings = callGData(emailsettings, u'GetPop', soft_errors=True, username=user) try: print u'User %s POP Enabled:%s Action:%s' % (user+u'@'+emailsettings.domain, popsettings[u'enable'], popsettings[u'action']) except TypeError: @@ -4264,7 +4264,7 @@ def doSendAs(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Allowing %s to send as %s (%s of %s)" % (user+u'@'+emailsettings.domain, sendas, i, count) i += 1 - callGData(service=emailsettings, function=u'CreateSendAsAlias', soft_errors=True, username=user, name=sendasName, address=sendas, make_default=make_default, reply_to=reply_to) + callGData(emailsettings, u'CreateSendAsAlias', soft_errors=True, username=user, name=sendasName, address=sendas, make_default=make_default, reply_to=reply_to) def showSendAs(users): emailsettings = getEmailSettingsObject() @@ -4275,7 +4275,7 @@ def showSendAs(users): else: emailsettings.domain = GC_Values[GC_DOMAIN] print u'%s has the following send as aliases:' % (user+u'@'+emailsettings.domain) - sendases = callGData(service=emailsettings, function=u'GetSendAsAlias', soft_errors=True, username=user) + sendases = callGData(emailsettings, u'GetSendAsAlias', soft_errors=True, username=user) try: for sendas in sendases: if sendas[u'isDefault'] == u'true': @@ -4308,7 +4308,7 @@ def doLanguage(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting the language for %s to %s (%s of %s)" % (user+u'@'+emailsettings.domain, language, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateLanguage', soft_errors=True, username=user, language=language) + callGData(emailsettings, u'UpdateLanguage', soft_errors=True, username=user, language=language) def doUTF(users): if sys.argv[4].lower() in true_values: @@ -4329,7 +4329,7 @@ def doUTF(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting UTF-8 to %s for %s (%s of %s)" % (str(SetUTF), user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateGeneral', soft_errors=True, username=user, unicode=SetUTF) + callGData(emailsettings, u'UpdateGeneral', soft_errors=True, username=user, unicode=SetUTF) def doPageSize(users): if sys.argv[4] == u'25' or sys.argv[4] == u'50' or sys.argv[4] == u'100': @@ -4348,7 +4348,7 @@ def doPageSize(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting Page Size to %s for %s (%s of %s)" % (PageSize, user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateGeneral', soft_errors=True, username=user, page_size=PageSize) + callGData(emailsettings, u'UpdateGeneral', soft_errors=True, username=user, page_size=PageSize) def doShortCuts(users): if sys.argv[4].lower() in true_values: @@ -4369,7 +4369,7 @@ def doShortCuts(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting Keyboard Short Cuts to %s for %s (%s of %s)" % (str(SetShortCuts), user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateGeneral', soft_errors=True, username=user, shortcuts=SetShortCuts) + callGData(emailsettings, u'UpdateGeneral', soft_errors=True, username=user, shortcuts=SetShortCuts) def doArrows(users): if sys.argv[4].lower() in true_values: @@ -4390,7 +4390,7 @@ def doArrows(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting Personal Indicator Arrows to %s for %s (%s of %s)" % (str(SetArrows), user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateGeneral', soft_errors=True, username=user, arrows=SetArrows) + callGData(emailsettings, u'UpdateGeneral', soft_errors=True, username=user, arrows=SetArrows) def doSnippets(users): if sys.argv[4].lower() in true_values: @@ -4411,7 +4411,7 @@ def doSnippets(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting Preview Snippets to %s for %s (%s of %s)" % (str(SetSnippets), user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateGeneral', soft_errors=True, username=user, snippets=SetSnippets) + callGData(emailsettings, u'UpdateGeneral', soft_errors=True, username=user, snippets=SetSnippets) def doLabel(users): label = sys.argv[4] @@ -4450,7 +4450,7 @@ def doLabel(users): gmail = buildGAPIServiceObject(u'gmail', user) print u"Creating label %s for %s (%s of %s)" % (label, user, i, count) i += 1 - callGAPI(service=gmail.users().labels(), function=u'create', soft_errors=True, userId=user, body=body) + callGAPI(gmail.users().labels(), u'create', soft_errors=True, userId=user, body=body) def doDeleteMessages(trashOrDelete, users): query = None @@ -4477,8 +4477,7 @@ def doDeleteMessages(trashOrDelete, users): print u'Searching messages for %s' % user gmail = buildGAPIServiceObject(u'gmail', user) page_message = u'Got %%%%total_items%%%% messages for user %s' % user - listResult = callGAPIpages(service=gmail.users().messages(), - function=u'list', items=u'messages', page_message=page_message, + listResult = callGAPIpages(gmail.users().messages(), u'list', u'messages', page_message=page_message, userId=u'me', q=query, includeSpamTrash=True, soft_errors=True) del_count = len(listResult) if not doIt: @@ -4491,7 +4490,7 @@ def doDeleteMessages(trashOrDelete, users): for del_me in listResult: i += 1 print u' %s message %s for user %s (%s/%s)' % (trashOrDelete, del_me[u'id'], user, i, del_count) - callGAPI(service=gmail.users().messages(), function=trashOrDelete, + callGAPI(gmail.users().messages(), trashOrDelete, id=del_me[u'id'], userId=u'me') def doDeleteLabel(users): @@ -4499,7 +4498,7 @@ def doDeleteLabel(users): for user in users: gmail = buildGAPIServiceObject(u'gmail', user) print u'Getting all labels for %s...' % user - labels = callGAPI(service=gmail.users().labels(), function=u'list', userId=user, fields=u'labels(name,id,type)') + labels = callGAPI(gmail.users().labels(), u'list', userId=user, fields=u'labels(name,id,type)') del_labels = [] if label == u'--ALL_LABELS--': for del_label in labels[u'labels']: @@ -4553,7 +4552,7 @@ def showLabels(users): sys.exit(2) for user in users: gmail = buildGAPIServiceObject(u'gmail', user) - labels = callGAPI(service=gmail.users().labels(), function=u'list', userId=user, soft_errors=True) + labels = callGAPI(gmail.users().labels(), u'list', userId=user, soft_errors=True) if labels: for label in labels[u'labels']: if label[u'type'] == u'system' and not show_system: @@ -4581,7 +4580,7 @@ def showGmailProfile(users): gmail = buildGAPIServiceObject(u'gmail', user, soft_errors=True) if not gmail: continue - results = callGAPI(service=gmail.users(), function=u'getProfile', userId=u'me', soft_errors=True) + results = callGAPI(gmail.users(), u'getProfile', userId=u'me', soft_errors=True) if results: for item in results: if item not in profiles[0]: @@ -4619,7 +4618,7 @@ def updateLabels(users): sys.exit(2) for user in users: gmail = buildGAPIServiceObject(u'gmail', user) - labels = callGAPI(service=gmail.users().labels(), function=u'list', userId=user, fields=u'labels(id,name)') + labels = callGAPI(gmail.users().labels(), u'list', userId=user, fields=u'labels(id,name)') label_id = None for label in labels[u'labels']: if label[u'name'].lower() == label_name.lower(): @@ -4627,7 +4626,7 @@ def updateLabels(users): break if not label_id: print 'Error: user does not have a label named %s' % label_name - callGAPI(service=gmail.users().labels(), function=u'patch', soft_errors=True, userId=user, id=label_id, body=body) + callGAPI(gmail.users().labels(), u'patch', soft_errors=True, userId=user, id=label_id, body=body) def renameLabels(users): search = u'^Inbox/(.*)$' @@ -4650,7 +4649,7 @@ def renameLabels(users): pattern = re.compile(search, re.IGNORECASE) for user in users: gmail = buildGAPIServiceObject(u'gmail', user) - labels = callGAPI(service=gmail.users().labels(), function=u'list', userId=user) + labels = callGAPI(gmail.users().labels(), u'list', userId=user) for label in labels[u'labels']: if label[u'type'] == u'system': continue @@ -4659,12 +4658,12 @@ def renameLabels(users): new_label_name = replace % match_result.groups() print u' Renaming "%s" to "%s"' % (label[u'name'], new_label_name) try: - callGAPI(service=gmail.users().labels(), function=u'patch', soft_errors=True, throw_reasons=[u'aborted'], id=label[u'id'], userId=user, body={u'name': new_label_name}) + callGAPI(gmail.users().labels(), u'patch', soft_errors=True, throw_reasons=[u'aborted'], id=label[u'id'], userId=user, body={u'name': new_label_name}) except googleapiclient.errors.HttpError: if merge: print u' Merging %s label to existing %s label' % (label[u'name'], new_label_name) q = u'label:"%s"' % label[u'name'] - messages_to_relabel = callGAPIpages(service=gmail.users().messages(), function=u'list', items=u'messages', userId=user, q=q) + messages_to_relabel = callGAPIpages(gmail.users().messages(), u'list', u'messages', userId=user, q=q) if len(messages_to_relabel) > 0: for new_label in labels[u'labels']: if new_label[u'name'].lower() == new_label_name.lower(): @@ -4674,12 +4673,12 @@ def renameLabels(users): i = 1 for message_to_relabel in messages_to_relabel: print u' relabeling message %s (%s/%s)' % (message_to_relabel[u'id'], i, len(messages_to_relabel)) - callGAPI(service=gmail.users().messages(), function=u'modify', userId=user, id=message_to_relabel[u'id'], body=body) + callGAPI(gmail.users().messages(), u'modify', userId=user, id=message_to_relabel[u'id'], body=body) i += 1 else: print u' no messages with %s label' % label[u'name'] print u' Deleting label %s' % label[u'name'] - callGAPI(service=gmail.users().labels(), function=u'delete', id=label[u'id'], userId=user) + callGAPI(gmail.users().labels(), u'delete', id=label[u'id'], userId=user) else: print u' Error: looks like %s already exists, not renaming. Use the "merge" argument to merge the labels' % new_label_name @@ -4762,7 +4761,7 @@ def doFilter(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Creating filter for %s (%s of %s)" % (user+'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'CreateFilter', soft_errors=True, + callGData(emailsettings, u'CreateFilter', soft_errors=True, username=user, from_=from_, to=to, subject=subject, has_the_word=has_the_word, does_not_have_the_word=does_not_have_the_word, has_attachment=has_attachment, label=label, should_mark_as_read=should_mark_as_read, should_archive=should_archive, should_star=should_star, forward_to=forward_to, should_trash=should_trash, should_not_spam=should_not_spam) @@ -4806,7 +4805,7 @@ def doForward(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Turning forward %s for %s, emails will be %s (%s of %s)" % (sys.argv[4], user+'@'+emailsettings.domain, action, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateForwarding', soft_errors=True, username=user, enable=enable, action=action, forward_to=forward_to) + callGData(emailsettings, u'UpdateForwarding', soft_errors=True, username=user, enable=enable, action=action, forward_to=forward_to) def getForward(users): emailsettings = getEmailSettingsObject() @@ -4816,7 +4815,7 @@ def getForward(users): user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - forward = callGData(service=emailsettings, function=u'GetForward', soft_errors=True, username=user) + forward = callGData(emailsettings, u'GetForward', soft_errors=True, username=user) try: print u"User %s: Forward To:%s Enabled:%s Action:%s" % (user+u'@'+emailsettings.domain, forward[u'forwardTo'], forward[u'enable'], forward[u'action']) except TypeError: @@ -4844,7 +4843,7 @@ def doSignature(users): print u"Setting Signature for %s (%s of %s)" % (user+u'@'+emailsettings.domain, i, count) uri = u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/%s/%s/signature' % (emailsettings.domain, user) i += 1 - callGData(service=emailsettings, function=u'Put', soft_errors=True, data=xmlsig, uri=uri) + callGData(emailsettings, u'Put', soft_errors=True, data=xmlsig, uri=uri) def getSignature(users): emailsettings = getEmailSettingsObject() @@ -4854,7 +4853,7 @@ def getSignature(users): user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - signature = callGData(service=emailsettings, function=u'GetSignature', soft_errors=True, username=user) + signature = callGData(emailsettings, u'GetSignature', soft_errors=True, username=user) try: sys.stderr.write(u"User %s signature:\n " % (user+u'@'+emailsettings.domain)) print convertUTF8(u" %s" % signature[u'signature']) @@ -4880,7 +4879,7 @@ def doWebClips(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Turning Web Clips %s for %s (%s of %s)" % (sys.argv[4], user+u'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateWebClipSettings', soft_errors=True, username=user, enable=enable) + callGData(emailsettings, u'UpdateWebClipSettings', soft_errors=True, username=user, enable=enable) def doVacation(users): subject = message = u'' @@ -4931,7 +4930,7 @@ def doVacation(users): emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting Vacation for %s (%s of %s)" % (user+'@'+emailsettings.domain, i, count) i += 1 - callGData(service=emailsettings, function=u'UpdateVacation', + callGData(emailsettings, u'UpdateVacation', soft_errors=True, username=user, enable=enable, subject=subject, message=message, contacts_only=contacts_only, domain_only=domain_only, start_date=start_date, end_date=end_date) @@ -4944,7 +4943,7 @@ def getVacation(users): user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - vacationsettings = callGData(service=emailsettings, function=u'GetVacation', soft_errors=True, username=user) + vacationsettings = callGData(emailsettings, u'GetVacation', soft_errors=True, username=user) try: print convertUTF8(u'''User %s Enabled: %s @@ -4962,7 +4961,7 @@ def getVacation(users): def doDelSchema(): cd = buildGAPIObject(u'directory') schemaKey = sys.argv[3] - callGAPI(service=cd.schemas(), function=u'delete', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) + callGAPI(cd.schemas(), u'delete', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) print u'Deleted schema %s' % schemaKey def doCreateOrUpdateUserSchema(): @@ -5004,15 +5003,15 @@ def doCreateOrUpdateUserSchema(): print 'ERROR: %s is not a valid argument for "gam create schema"' % sys.argv[i] sys.exit(2) if sys.argv[1].lower() == u'create': - result = callGAPI(service=cd.schemas(), function=u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) + result = callGAPI(cd.schemas(), u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) print 'Created user schema %s' % result[u'schemaName'] elif sys.argv[1].lower() == u'update': - result = callGAPI(service=cd.schemas(), function=u'update', customerId=GC_Values[GC_CUSTOMER_ID], body=body, schemaKey=schemaName) + result = callGAPI(cd.schemas(), u'update', customerId=GC_Values[GC_CUSTOMER_ID], body=body, schemaKey=schemaName) print 'Updated user schema %s' % result[u'schemaName'] def doPrintUserSchemas(): cd = buildGAPIObject(u'directory') - schemas = callGAPI(service=cd.schemas(), function=u'list', customerId=GC_Values[GC_CUSTOMER_ID]) + schemas = callGAPI(cd.schemas(), u'list', customerId=GC_Values[GC_CUSTOMER_ID]) if not schemas or u'schemas' not in schemas: return for schema in schemas[u'schemas']: @@ -5032,7 +5031,7 @@ def doPrintUserSchemas(): def doGetUserSchema(): cd = buildGAPIObject(u'directory') schemaKey = sys.argv[3] - schema = callGAPI(service=cd.schemas(), function=u'get', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) + schema = callGAPI(cd.schemas(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) print u'Schema: %s' % schema[u'schemaName'] for a_key in schema: if a_key not in [u'schemaName', u'fields', u'etag', u'kind']: @@ -5385,10 +5384,10 @@ def doCreateUser(): body[u'password'] = gen_sha512_hash(body[u'password']) body[u'hashFunction'] = u'crypt' print u"Creating account for %s" % body[u'primaryEmail'] - callGAPI(service=cd.users(), function='insert', body=body, fields=u'primaryEmail') + callGAPI(cd.users(), 'insert', body=body, fields=u'primaryEmail') if do_admin: print u' Changing admin status for %s to %s' % (body[u'primaryEmail'], admin_body[u'status']) - callGAPI(service=cd.users(), function=u'makeAdmin', userKey=body[u'primaryEmail'], body=admin_body) + callGAPI(cd.users(), u'makeAdmin', userKey=body[u'primaryEmail'], body=admin_body) def doCreateGroup(): cd = buildGAPIObject(u'directory') @@ -5447,10 +5446,10 @@ def doCreateGroup(): if not got_name: body[u'name'] = body[u'email'] print u"Creating group %s" % body[u'email'] - callGAPI(service=cd.groups(), function=u'insert', body=body, fields=u'email') + callGAPI(cd.groups(), u'insert', body=body, fields=u'email') if use_gs_api: gs = buildGAPIObject(u'groupssettings') - callGAPI(service=gs.groups(), function=u'patch', retry_reasons=[u'serviceLimit'], groupUniqueId=body[u'email'], body=gs_body) + callGAPI(gs.groups(), u'patch', retry_reasons=[u'serviceLimit'], groupUniqueId=body[u'email'], body=gs_body) def doCreateAlias(): cd = buildGAPIObject(u'directory') @@ -5467,14 +5466,14 @@ def doCreateAlias(): targetKey = u'%s@%s' % (targetKey, GC_Values[GC_DOMAIN]) print u'Creating alias %s for %s %s' % (body[u'alias'], target_type, targetKey) if target_type == u'user': - callGAPI(service=cd.users().aliases(), function=u'insert', userKey=targetKey, body=body) + callGAPI(cd.users().aliases(), u'insert', userKey=targetKey, body=body) elif target_type == u'group': - callGAPI(service=cd.groups().aliases(), function=u'insert', groupKey=targetKey, body=body) + callGAPI(cd.groups().aliases(), u'insert', groupKey=targetKey, body=body) elif target_type == u'target': try: - callGAPI(service=cd.users().aliases(), function=u'insert', throw_reasons=[u'invalid'], userKey=targetKey, body=body) + callGAPI(cd.users().aliases(), u'insert', throw_reasons=[u'invalid'], userKey=targetKey, body=body) except googleapiclient.errors.HttpError: - callGAPI(service=cd.groups().aliases(), function=u'insert', groupKey=targetKey, body=body) + callGAPI(cd.groups().aliases(), u'insert', groupKey=targetKey, body=body) def doCreateOrg(): cd = buildGAPIObject(u'directory') @@ -5497,7 +5496,7 @@ def doCreateOrg(): else: print u'ERROR: %s is not a valid argument for "gam create org"' % sys.argv[i] sys.exit(2) - callGAPI(service=cd.orgunits(), function=u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) + callGAPI(cd.orgunits(), u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) def doCreateResourceCalendar(): cd = buildGAPIObject(u'directory') @@ -5515,7 +5514,7 @@ def doCreateResourceCalendar(): print u'ERROR: %s is not a valid argument for "gam create resource"' % sys.argv[i] sys.exit(2) print u'Creating resource %s...' % body[u'resourceId'] - callGAPI(service=cd.resources().calendars(), function=u'insert', + callGAPI(cd.resources().calendars(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doUpdateUser(users): @@ -5894,7 +5893,7 @@ def doUpdateUser(users): elif user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) if u'primaryEmail' in body and body[u'primaryEmail'][:4].lower() == u'vfe@': - user_primary = callGAPI(service=cd.users(), function=u'get', userKey=user, fields=u'primaryEmail,id') + user_primary = callGAPI(cd.users(), u'get', userKey=user, fields=u'primaryEmail,id') user = user_primary[u'id'] user_primary = user_primary[u'primaryEmail'] user_name = user_primary[:user_primary.find(u'@')] @@ -5903,34 +5902,34 @@ def doUpdateUser(users): body[u'emails'] = [{u'type': u'custom', u'customType': u'former_employee', u'primary': False, u'address': user_primary}] sys.stdout.write(u'updating user %s...\n' % user) if do_update_user: - callGAPI(service=cd.users(), function=u'patch', userKey=user, body=body) + callGAPI(cd.users(), u'patch', userKey=user, body=body) if do_admin_user: - callGAPI(service=cd.users(), function=u'makeAdmin', userKey=user, body={u'status': is_admin}) + callGAPI(cd.users(), u'makeAdmin', userKey=user, body={u'status': is_admin}) def doRemoveUsersAliases(users): cd = buildGAPIObject(u'directory') for user in users: - user_aliases = callGAPI(service=cd.users(), function=u'get', userKey=user, fields=u'aliases,id,primaryEmail') + user_aliases = callGAPI(cd.users(), u'get', userKey=user, fields=u'aliases,id,primaryEmail') user_id = user_aliases[u'id'] user_primary = user_aliases[u'primaryEmail'] if u'aliases' in user_aliases: print u'%s has %s aliases' % (user_primary, len(user_aliases[u'aliases'])) for an_alias in user_aliases[u'aliases']: print u' removing alias %s for %s...' % (an_alias, user_primary) - callGAPI(service=cd.users().aliases(), function=u'delete', userKey=user_id, alias=an_alias) + callGAPI(cd.users().aliases(), u'delete', userKey=user_id, alias=an_alias) else: print u'%s has no aliases' % user_primary def doRemoveUsersGroups(users): cd = buildGAPIObject(u'directory') for user in users: - user_groups = callGAPIpages(service=cd.groups(), items=u'groups', function=u'list', userKey=user, fields=u'groups(id,email)') + user_groups = callGAPIpages(cd.groups(), u'list', u'groups', userKey=user, fields=u'groups(id,email)') num_groups = len(user_groups) print u'%s is in %s groups' % (user, num_groups) i = 1 for user_group in user_groups: print u' removing %s from %s (%s/%s)' % (user, user_group[u'email'], i, num_groups) - callGAPI(service=cd.members(), function=u'delete', soft_errors=True, groupKey=user_group[u'id'], memberKey=user) + callGAPI(cd.members(), u'delete', soft_errors=True, groupKey=user_group[u'id'], memberKey=user) i += 1 print u'' @@ -5960,9 +5959,9 @@ def doUpdateGroup(): if sys.argv[4].lower() == u'add': body = {u'role': role} body[u'email'] = user_email - result = callGAPI(service=cd.members(), function=u'insert', soft_errors=True, groupKey=group, body=body) + result = callGAPI(cd.members(), u'insert', soft_errors=True, groupKey=group, body=body) elif sys.argv[4].lower() == u'update': - result = callGAPI(service=cd.members(), function=u'update', soft_errors=True, groupKey=group, memberKey=user_email, body={u'email': user_email, u'role': role}) + result = callGAPI(cd.members(), u'update', soft_errors=True, groupKey=group, memberKey=user_email, body={u'email': user_email, u'role': role}) try: if str(result[u'email']).lower() != user_email.lower(): print u'added %s (primary address) to group' % result[u'email'] @@ -5987,12 +5986,12 @@ def doUpdateGroup(): for user_email in to_add: sys.stderr.write(u' adding %s %s\n' % (role, user_email)) try: - result = callGAPI(service=cd.members(), function=u'insert', soft_errors=True, throw_reasons=[u'duplicate'], groupKey=group, body={u'email': user_email, u'role': role}) + result = callGAPI(cd.members(), u'insert', soft_errors=True, throw_reasons=[u'duplicate'], groupKey=group, body={u'email': user_email, u'role': role}) except googleapiclient.errors.HttpError: - result = callGAPI(service=cd.members(), function=u'update', soft_errors=True, groupKey=group, memberKey=user_email, body={u'email': user_email, u'role': role}) + result = callGAPI(cd.members(), u'update', soft_errors=True, groupKey=group, memberKey=user_email, body={u'email': user_email, u'role': role}) for user_email in to_remove: sys.stderr.write(u' removing %s\n' % user_email) - result = callGAPI(service=cd.members(), function=u'delete', soft_errors=True, groupKey=group, memberKey=user_email) + result = callGAPI(cd.members(), u'delete', soft_errors=True, groupKey=group, memberKey=user_email) elif sys.argv[4].lower() == u'remove': i = 5 if sys.argv[i].lower() in [u'member', u'manager', u'owner']: @@ -6005,7 +6004,7 @@ def doUpdateGroup(): if user_email != u'*' and user_email.find(u'@') == -1: user_email = u'%s@%s' % (user_email, GC_Values[GC_DOMAIN]) sys.stderr.write(u' removing %s\n' % user_email) - result = callGAPI(service=cd.members(), function=u'delete', soft_errors=True, groupKey=group, memberKey=user_email) + result = callGAPI(cd.members(), u'delete', soft_errors=True, groupKey=group, memberKey=user_email) else: i = 4 use_cd_api = False @@ -6071,12 +6070,12 @@ def doUpdateGroup(): cd_body[u'email'] = u'%s@%s' % (cd_body[u'email'], GC_Values[GC_DOMAIN]) except KeyError: pass - cd_result = callGAPI(service=cd.groups(), function=u'patch', groupKey=group, body=cd_body) + cd_result = callGAPI(cd.groups(), u'patch', groupKey=group, body=cd_body) if use_gs_api: gs = buildGAPIObject(u'groupssettings') if use_cd_api: group = cd_result[u'email'] - callGAPI(service=gs.groups(), function=u'patch', retry_reasons=[u'serviceLimit'], groupUniqueId=group, body=gs_body) + callGAPI(gs.groups(), u'patch', retry_reasons=[u'serviceLimit'], groupUniqueId=group, body=gs_body) print u'updated group %s' % group def doUpdateAlias(): @@ -6092,18 +6091,18 @@ def doUpdateAlias(): if target_email.find(u'@') == -1: target_email = u'%s@%s' % (target_email, GC_Values[GC_DOMAIN]) try: - callGAPI(service=cd.users().aliases(), function=u'delete', throw_reasons=[u'invalid'], userKey=alias, alias=alias) + callGAPI(cd.users().aliases(), u'delete', throw_reasons=[u'invalid'], userKey=alias, alias=alias) except googleapiclient.errors.HttpError: - callGAPI(service=cd.groups().aliases(), function=u'delete', groupKey=alias, alias=alias) + callGAPI(cd.groups().aliases(), u'delete', groupKey=alias, alias=alias) if target_type == u'user': - callGAPI(service=cd.users().aliases(), function=u'insert', userKey=target_email, body={u'alias': alias}) + callGAPI(cd.users().aliases(), u'insert', userKey=target_email, body={u'alias': alias}) elif target_type == u'group': - callGAPI(service=cd.groups().aliases(), function=u'insert', groupKey=target_email, body={u'alias': alias}) + callGAPI(cd.groups().aliases(), u'insert', groupKey=target_email, body={u'alias': alias}) elif target_type == u'target': try: - callGAPI(service=cd.users().aliases(), function=u'insert', throw_reasons=[u'invalid'], userKey=target_email, body={u'alias': alias}) + callGAPI(cd.users().aliases(), u'insert', throw_reasons=[u'invalid'], userKey=target_email, body={u'alias': alias}) except googleapiclient.errors.HttpError: - callGAPI(service=cd.groups().aliases(), function=u'insert', groupKey=target_email, body={u'alias': alias}) + callGAPI(cd.groups().aliases(), u'insert', groupKey=target_email, body={u'alias': alias}) print u'updated alias %s' % alias def doUpdateResourceCalendar(): @@ -6126,7 +6125,7 @@ def doUpdateResourceCalendar(): sys.exit(2) # Use patch since it seems to work better. # update requires name to be set. - callGAPI(service=cd.resources().calendars(), function=u'patch', + callGAPI(cd.resources().calendars(), u'patch', customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId, body=body, fields=u'') print u'updated resource %s' % resId @@ -6136,7 +6135,7 @@ def doUpdateCros(): deviceId = sys.argv[3] if deviceId[:6].lower() == u'query:': query = deviceId[6:] - devices_result = callGAPIpages(service=cd.chromeosdevices(), function=u'list', items=u'chromeosdevices', + devices_result = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', query=query, customerId=GC_Values[GC_CUSTOMER_ID], fields=u'chromeosdevices/deviceId,nextPageToken') devices = list() for a_device in devices_result: @@ -6177,7 +6176,7 @@ def doUpdateCros(): i = 1 for this_device in devices: print u' updating %s (%s of %s)' % (this_device, i, device_count) - callGAPI(service=cd.chromeosdevices(), function=u'patch', deviceId=this_device, body=body, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.chromeosdevices(), u'patch', deviceId=this_device, body=body, customerId=GC_Values[GC_CUSTOMER_ID]) i += 1 def doUpdateMobile(): @@ -6214,14 +6213,14 @@ def doUpdateMobile(): print u'ERROR: %s is not a valid argument for "gam update mobile"' % sys.argv[i] sys.exit(2) if doPatch: - callGAPI(service=cd.mobiledevices(), function=u'patch', resourceId=resourceId, body=patch_body, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.mobiledevices(), u'patch', resourceId=resourceId, body=patch_body, customerId=GC_Values[GC_CUSTOMER_ID]) if doAction: - callGAPI(service=cd.mobiledevices(), function=u'action', resourceId=resourceId, body=action_body, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.mobiledevices(), u'action', resourceId=resourceId, body=action_body, customerId=GC_Values[GC_CUSTOMER_ID]) def doDeleteMobile(): cd = buildGAPIObject(u'directory') resourceId = sys.argv[3] - callGAPI(service=cd.mobiledevices(), function='delete', resourceId=resourceId, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.mobiledevices(), 'delete', resourceId=resourceId, customerId=GC_Values[GC_CUSTOMER_ID]) def doUpdateOrg(): cd = buildGAPIObject(u'directory') @@ -6236,7 +6235,7 @@ def doUpdateOrg(): current_cros = 1 for cros in users: sys.stderr.write(u' moving %s to %s (%s/%s)\n' % (cros, orgUnitPath, current_cros, cros_count)) - callGAPI(service=cd.chromeosdevices(), function=u'patch', soft_errors=True, + callGAPI(cd.chromeosdevices(), u'patch', soft_errors=True, customerId=GC_Values[GC_CUSTOMER_ID], deviceId=cros, body={u'orgUnitPath': '//%s' % orgUnitPath}) current_cros += 1 else: @@ -6247,7 +6246,7 @@ def doUpdateOrg(): for user in users: sys.stderr.write(u' moving %s to %s (%s/%s)\n' % (user, orgUnitPath, current_user, user_count)) try: - callGAPI(service=cd.users(), function=u'patch', throw_reasons=[u'conditionNotMet'], userKey=user, body={u'orgUnitPath': orgUnitPath}) + callGAPI(cd.users(), u'patch', throw_reasons=[u'conditionNotMet'], userKey=user, body={u'orgUnitPath': orgUnitPath}) except googleapiclient.errors.HttpError: pass current_user += 1 @@ -6277,7 +6276,7 @@ def doUpdateOrg(): sys.exit(2) if orgUnitPath[0] == u'/': # we don't want a / at the beginning for OU updates orgUnitPath = orgUnitPath[1:] - callGAPI(service=cd.orgunits(), function=u'update', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnitPath, body=body) + callGAPI(cd.orgunits(), u'update', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnitPath, body=body) def doWhatIs(): cd = buildGAPIObject(u'directory') @@ -6285,7 +6284,7 @@ def doWhatIs(): if email.find(u'@') == -1: email = u'%s@%s' % (email, GC_Values[GC_DOMAIN]) try: - user_or_alias = callGAPI(service=cd.users(), function=u'get', throw_reasons=[u'notFound', u'badRequest', u'invalid'], userKey=email, fields=u'primaryEmail') + user_or_alias = callGAPI(cd.users(), u'get', throw_reasons=[u'notFound', u'badRequest', u'invalid'], userKey=email, fields=u'primaryEmail') if user_or_alias[u'primaryEmail'].lower() == email.lower(): sys.stderr.write(u'%s is a user\n\n' % email) doGetUserInfo(user_email=email) @@ -6298,7 +6297,7 @@ def doWhatIs(): sys.stderr.write(u'%s is not a user...\n' % email) sys.stderr.write(u'%s is not a user alias...\n' % email) try: - group = callGAPI(service=cd.groups(), function=u'get', throw_reasons=[u'notFound', u'badRequest'], groupKey=email, fields=u'email') + group = callGAPI(cd.groups(), u'get', throw_reasons=[u'notFound', u'badRequest'], groupKey=email, fields=u'email') except googleapiclient.errors.HttpError: sys.stderr.write(u'%s is not a group either!\n\nDoesn\'t seem to exist!\n\n' % email) sys.exit(1) @@ -6356,7 +6355,7 @@ def doGetUserInfo(user_email=None): else: print u'ERROR: %s is not a valid argument for "gam info user"' % sys.argv[i] sys.exit(2) - user = callGAPI(service=cd.users(), function=u'get', userKey=user_email, projection=projection, customFieldMask=customFieldMask, viewType=viewType) + user = callGAPI(cd.users(), u'get', userKey=user_email, projection=projection, customFieldMask=customFieldMask, viewType=viewType) print u'User: %s' % user[u'primaryEmail'] if u'name' in user and u'givenName' in user[u'name']: print convertUTF8(u'First Name: %s' % user[u'name'][u'givenName']) @@ -6480,7 +6479,7 @@ def doGetUserInfo(user_email=None): for alias in user[u'nonEditableAliases']: print u' %s' % alias if getGroups: - groups = callGAPIpages(service=cd.groups(), function=u'list', items=u'groups', userKey=user_email, fields=u'groups(name,email),nextPageToken') + groups = callGAPIpages(cd.groups(), u'list', u'groups', userKey=user_email, fields=u'groups(name,email),nextPageToken') if len(groups) > 0: print u'Groups: (%s)' % len(groups) for group in groups: @@ -6495,7 +6494,7 @@ def doGetUserInfo(user_email=None): u'Google-Vault-Former-Employee']: productId, skuId = getProductAndSKU(sku) try: - result = callGAPI(service=lic.licenseAssignments(), function=u'get', throw_reasons=['notFound'], userId=user_email, productId=productId, skuId=skuId) + result = callGAPI(lic.licenseAssignments(), u'get', throw_reasons=['notFound'], userId=user_email, productId=productId, skuId=skuId) except googleapiclient.errors.HttpError: continue print u' %s' % result[u'skuId'] @@ -6520,9 +6519,9 @@ def doGetGroupInfo(group_name=None): group_name = group_name[4:] elif group_name.find(u'@') == -1: group_name = group_name+u'@'+GC_Values[GC_DOMAIN] - basic_info = callGAPI(service=cd.groups(), function=u'get', groupKey=group_name) + basic_info = callGAPI(cd.groups(), u'get', groupKey=group_name) try: - settings = callGAPI(service=gs.groups(), function=u'get', retry_reasons=[u'serviceLimit'], throw_reasons=u'authError', + settings = callGAPI(gs.groups(), u'get', retry_reasons=[u'serviceLimit'], throw_reasons=u'authError', groupUniqueId=basic_info[u'email']) # Use email address retrieved from cd since GS API doesn't support uid except googleapiclient.errors.HttpError: pass @@ -6550,7 +6549,7 @@ def doGetGroupInfo(group_name=None): except UnboundLocalError: pass if get_users: - members = callGAPIpages(service=cd.members(), function=u'list', items=u'members', groupKey=group_name) + members = callGAPIpages(cd.members(), u'list', u'members', groupKey=group_name) print u'Members:' for member in members: try: @@ -6569,9 +6568,9 @@ def doGetAliasInfo(alias_email=None): if alias_email.find(u'@') == -1: alias_email = u'%s@%s' % (alias_email, GC_Values[GC_DOMAIN]) try: - result = callGAPI(service=cd.users(), function=u'get', throw_reasons=[u'invalid', u'badRequest'], userKey=alias_email) + result = callGAPI(cd.users(), u'get', throw_reasons=[u'invalid', u'badRequest'], userKey=alias_email) except googleapiclient.errors.HttpError: - result = callGAPI(service=cd.groups(), function=u'get', groupKey=alias_email) + result = callGAPI(cd.groups(), u'get', groupKey=alias_email) print u' Alias Email: %s' % alias_email try: if result[u'primaryEmail'].lower() == alias_email.lower(): @@ -6585,7 +6584,7 @@ def doGetAliasInfo(alias_email=None): def doGetResourceCalendarInfo(): cd = buildGAPIObject(u'directory') resId = sys.argv[3] - resource = callGAPI(service=cd.resources().calendars(), function=u'get', + resource = callGAPI(cd.resources().calendars(), u'get', customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId) for key, value in resource.items(): if key in [u'kind', u'etag', u'etags']: @@ -6595,13 +6594,13 @@ def doGetResourceCalendarInfo(): def doGetCrosInfo(): cd = buildGAPIObject(u'directory') deviceId = sys.argv[3] - info = callGAPI(service=cd.chromeosdevices(), function=u'get', customerId=GC_Values[GC_CUSTOMER_ID], deviceId=deviceId) + info = callGAPI(cd.chromeosdevices(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], deviceId=deviceId) print_json(None, info) def doGetMobileInfo(): cd = buildGAPIObject(u'directory') deviceId = sys.argv[3] - info = callGAPI(service=cd.mobiledevices(), function=u'get', customerId=GC_Values[GC_CUSTOMER_ID], resourceId=deviceId) + info = callGAPI(cd.mobiledevices(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], resourceId=deviceId) print_json(None, info) def print_json(object_name, object_value, spacing=u''): @@ -6653,13 +6652,13 @@ def doUpdateNotification(): print u'ERROR: notifications need to be marked as read or unread.' sys.exit(2) if get_all: - notifications = callGAPIpages(service=cd.notifications(), function=u'list', customer=GC_Values[GC_CUSTOMER_ID], fields=u'items(notificationId,isUnread),nextPageToken') + notifications = callGAPIpages(cd.notifications(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID], fields=u'items(notificationId,isUnread),nextPageToken') for noti in notifications: if noti[u'isUnread'] != isUnread: ids.append(noti[u'notificationId']) print u'Marking %s notification(s) as %s...' % (len(ids), mark_as) for notificationId in ids: - result = callGAPI(service=cd.notifications(), function=u'patch', customer=GC_Values[GC_CUSTOMER_ID], notificationId=notificationId, body={u'isUnread': isUnread}, fields=u'notificationId,isUnread') + result = callGAPI(cd.notifications(), u'patch', customer=GC_Values[GC_CUSTOMER_ID], notificationId=notificationId, body={u'isUnread': isUnread}, fields=u'notificationId,isUnread') if result[u'isUnread']: read_result = u'unread' else: @@ -6682,22 +6681,22 @@ def doDeleteNotification(): print 'ERROR: %s is not a valid argument for "gam delete notification", expected id' % sys.argv[i] sys.exit(2) if get_all: - notifications = callGAPIpages(service=cd.notifications(), function=u'list', customer=GC_Values[GC_CUSTOMER_ID], fields=u'items(notificationId),nextPageToken') + notifications = callGAPIpages(cd.notifications(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID], fields=u'items(notificationId),nextPageToken') for noti in notifications: ids.append(noti[u'notificationId']) print u'Deleting %s notification(s)...' % len(ids) for notificationId in ids: - callGAPI(service=cd.notifications(), function=u'delete', customer=GC_Values[GC_CUSTOMER_ID], notificationId=notificationId) + callGAPI(cd.notifications(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], notificationId=notificationId) print u'deleted %s' % id def doSiteVerifyShow(): verif = buildGAPIObject(u'siteVerification') a_domain = sys.argv[3] - txt_record = callGAPI(service=verif.webResource(), function=u'getToken', body={u'site':{u'type':u'INET_DOMAIN', u'identifier':a_domain}, u'verificationMethod':u'DNS_TXT'}) + txt_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'INET_DOMAIN', u'identifier':a_domain}, u'verificationMethod':u'DNS_TXT'}) print u'TXT Record Name: %s' % a_domain print u'TXT Record Value: %s' % txt_record[u'token'] print - cname_record = callGAPI(service=verif.webResource(), function=u'getToken', body={u'site':{u'type':u'INET_DOMAIN', u'identifier':a_domain}, u'verificationMethod':u'DNS_CNAME'}) + cname_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'INET_DOMAIN', u'identifier':a_domain}, u'verificationMethod':u'DNS_CNAME'}) cname_token = cname_record[u'token'] cname_list = cname_token.split(u' ') cname_subdomain = cname_list[0] @@ -6705,20 +6704,20 @@ def doSiteVerifyShow(): print u'CNAME Record Name: %s.%s' % (cname_subdomain, a_domain) print u'CNAME Record Value: %s' % cname_value print u'' - webserver_file_record = callGAPI(service=verif.webResource(), function=u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'FILE'}) + webserver_file_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'FILE'}) webserver_file_token = webserver_file_record[u'token'] print u'Saving web server verification file to: %s' % webserver_file_token writeFile(webserver_file_token, u'google-site-verification: {0}'.format(webserver_file_token), continueOnError=True) print u'Verification File URL: http://%s/%s' % (a_domain, webserver_file_token) print - webserver_meta_record = callGAPI(service=verif.webResource(), function=u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'META'}) + webserver_meta_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'META'}) print u'Meta URL: http://%s/' % a_domain print u'Meta HTML Header Data: %s' % webserver_meta_record[u'token'] print def doGetSiteVerifications(): verif = buildGAPIObject(u'siteVerification') - sites = callGAPI(service=verif.webResource(), function=u'list') + sites = callGAPI(verif.webResource(), u'list') try: for site in sites[u'items']: print u'Site: %s' % site[u'site'][u'identifier'] @@ -6746,12 +6745,12 @@ def doSiteVerifyAttempt(): identifier = u'http://%s/' % a_domain body = {u'site':{u'type':verify_type, u'identifier':identifier}, u'verificationMethod':verificationMethod} try: - verify_result = callGAPI(service=verif.webResource(), function=u'insert', throw_reasons=[u'badRequest'], verificationMethod=verificationMethod, body=body) + verify_result = callGAPI(verif.webResource(), u'insert', throw_reasons=[u'badRequest'], verificationMethod=verificationMethod, body=body) except googleapiclient.errors.HttpError, e: error = json.loads(e.content) message = error[u'error'][u'errors'][0][u'message'] print u'ERROR: %s' % message - verify_data = callGAPI(service=verif.webResource(), function=u'getToken', body=body) + verify_data = callGAPI(verif.webResource(), u'getToken', body=body) print u'Method: %s' % verify_data[u'method'] print u'Token: %s' % verify_data[u'token'] if verify_data[u'method'] == u'DNS_CNAME': @@ -6808,7 +6807,7 @@ def doGetNotifications(): print 'ERROR: %s is not a valid argument for "gam info notification", expected unreadonly' % sys.argv[i] sys.exit(2) i += 1 - notifications = callGAPIpages(service=cd.notifications(), function=u'list', customer=GC_Values[GC_CUSTOMER_ID]) + notifications = callGAPIpages(cd.notifications(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID]) for notification in notifications: if unread_only and not notification[u'isUnread']: continue @@ -6843,19 +6842,19 @@ def doGetOrgInfo(): print u'ERROR: %s is not a valid argument for "gam info org"' % sys.argv[i] sys.exit(2) if name == u'/': - orgs = callGAPI(service=cd.orgunits(), function=u'list', + orgs = callGAPI(cd.orgunits(), u'list', customerId=GC_Values[GC_CUSTOMER_ID], type=u'children', fields=u'organizationUnits/parentOrgUnitId') name = orgs[u'organizationUnits'][0][u'parentOrgUnitId'] if len(name) > 1 and name[0] == u'/': name = name[1:] - result = callGAPI(service=cd.orgunits(), function=u'get', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=name) + result = callGAPI(cd.orgunits(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=name) print_json(None, result) if get_users: name = result[u'orgUnitPath'] print u'Users: ' page_message = u'Got %%total_items%% users: %%first_item%% - %%last_item%%\n' - users = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, message_attribute=u'primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=u"orgUnitPath='%s'" % name, fields=u'users(primaryEmail,orgUnitPath),nextPageToken', maxResults=GC_Values[GC_USER_MAX_RESULTS]) for user in users: @@ -6869,7 +6868,7 @@ def doGetOrgInfo(): def doGetASPs(users): cd = buildGAPIObject(u'directory') for user in users: - asps = callGAPI(service=cd.asps(), function=u'list', userKey=user) + asps = callGAPI(cd.asps(), u'list', userKey=user) print u'Application-Specific Passwords for %s' % user try: for asp in asps[u'items']: @@ -6889,7 +6888,7 @@ def doDelASP(users): cd = buildGAPIObject(u'directory') codeId = sys.argv[5] for user in users: - callGAPI(service=cd.asps(), function=u'delete', userKey=user, codeId=codeId) + callGAPI(cd.asps(), u'delete', userKey=user, codeId=codeId) print u'deleted ASP %s for %s' % (codeId, user) def printBackupCodes(user, codes): @@ -6907,7 +6906,7 @@ def doGetBackupCodes(users): cd = buildGAPIObject(u'directory') for user in users: try: - codes = callGAPI(service=cd.verificationCodes(), function=u'list', throw_reasons=[u'invalidArgument', u'invalid'], userKey=user) + codes = callGAPI(cd.verificationCodes(), u'list', throw_reasons=[u'invalidArgument', u'invalid'], userKey=user) except googleapiclient.errors.HttpError: codes = None printBackupCodes(user, codes) @@ -6915,15 +6914,15 @@ def doGetBackupCodes(users): def doGenBackupCodes(users): cd = buildGAPIObject(u'directory') for user in users: - callGAPI(service=cd.verificationCodes(), function=u'generate', userKey=user) - codes = callGAPI(service=cd.verificationCodes(), function=u'list', userKey=user) + callGAPI(cd.verificationCodes(), u'generate', userKey=user) + codes = callGAPI(cd.verificationCodes(), u'list', userKey=user) printBackupCodes(user, codes) def doDelBackupCodes(users): cd = buildGAPIObject(u'directory') for user in users: try: - callGAPI(service=cd.verificationCodes(), function=u'invalidate', soft_errors=True, throw_reasons=[u'invalid',], userKey=user) + callGAPI(cd.verificationCodes(), u'invalidate', soft_errors=True, throw_reasons=[u'invalid',], userKey=user) except googleapiclient.errors.HttpError: print u'No 2SV backup codes for %s' % user continue @@ -6949,13 +6948,13 @@ def doGetTokens(users): clientId = commonClientIds(clientId) for user in users: try: - token = callGAPI(service=cd.tokens(), function=u'get', throw_reasons=[u'notFound',], userKey=user, clientId=clientId, fields=u'clientId') + token = callGAPI(cd.tokens(), u'get', throw_reasons=[u'notFound',], userKey=user, clientId=clientId, fields=u'clientId') except googleapiclient.errors.HttpError: continue print u'%s has allowed this token' % user return for user in users: - tokens = callGAPI(service=cd.tokens(), function=u'list', userKey=user) + tokens = callGAPI(cd.tokens(), u'list', userKey=user) print u'Tokens for %s:' % user try: for token in tokens[u'items']: @@ -6982,34 +6981,34 @@ def doDelTokens(users): clientId = sys.argv[6] clientId = commonClientIds(clientId) for user in users: - callGAPI(service=cd.tokens(), function=u'delete', userKey=user, clientId=clientId) + callGAPI(cd.tokens(), u'delete', userKey=user, clientId=clientId) print u'Deleted token for %s' % user def doDeprovUser(users): cd = buildGAPIObject(u'directory') for user in users: print u'Getting Application Specific Passwords for %s' % user - asps = callGAPI(service=cd.asps(), function=u'list', userKey=user, fields=u'items/codeId') + asps = callGAPI(cd.asps(), u'list', userKey=user, fields=u'items/codeId') i = 1 try: for asp in asps[u'items']: print u' deleting ASP %s of %s' % (i, len(asps['items'])) - callGAPI(service=cd.asps(), function=u'delete', userKey=user, codeId=asp[u'codeId']) + callGAPI(cd.asps(), u'delete', userKey=user, codeId=asp[u'codeId']) i += 1 except KeyError: print u'No ASPs' print u'Invalidating 2SV Backup Codes for %s' % user try: - callGAPI(service=cd.verificationCodes(), function=u'invalidate', soft_errors=True, throw_reasons=[u'invalid'], userKey=user) + callGAPI(cd.verificationCodes(), u'invalidate', soft_errors=True, throw_reasons=[u'invalid'], userKey=user) except googleapiclient.errors.HttpError: print u'No 2SV Backup Codes' print u'Getting tokens for %s...' % user - tokens = callGAPI(service=cd.tokens(), function=u'list', userKey=user, fields=u'items/clientId') + tokens = callGAPI(cd.tokens(), u'list', userKey=user, fields=u'items/clientId') i = 1 try: for token in tokens[u'items']: print u' deleting token %s of %s' % (i, len(tokens['items'])) - callGAPI(service=cd.tokens(), function=u'delete', userKey=user, clientId=token[u'clientId']) + callGAPI(cd.tokens(), u'delete', userKey=user, clientId=token[u'clientId']) i += 1 except KeyError: print u'No Tokens' @@ -7020,19 +7019,19 @@ def doUpdateInstance(): command = sys.argv[3].lower() if command == u'language': language = sys.argv[4] - callGData(service=adminObj, function=u'UpdateDefaultLanguage', defaultLanguage=language) + callGData(adminObj, u'UpdateDefaultLanguage', defaultLanguage=language) elif command == u'name': name = sys.argv[4] - callGData(service=adminObj, function=u'UpdateOrganizationName', organizationName=name) + callGData(adminObj, u'UpdateOrganizationName', organizationName=name) elif command == u'admin_secondary_email': admin_secondary_email = sys.argv[4] - callGData(service=adminObj, function=u'UpdateAdminSecondaryEmail', adminSecondaryEmail=admin_secondary_email) + callGData(adminObj, u'UpdateAdminSecondaryEmail', adminSecondaryEmail=admin_secondary_email) elif command == u'logo': logoFile = sys.argv[4] logoImage = readFile(logoFile) - callGData(service=adminObj, function=u'UpdateDomainLogo', logoImage=logoImage) + callGData(adminObj, u'UpdateDomainLogo', logoImage=logoImage) elif command == u'mx_verify': - result = callGData(service=adminObj, function=u'UpdateMXVerificationStatus') + result = callGData(adminObj, u'UpdateMXVerificationStatus') print u'Verification Method: %s' % result[u'verificationMethod'] print u'Verified: %s' % result[u'verified'] elif command == u'sso_settings': @@ -7072,25 +7071,25 @@ def doUpdateInstance(): else: print u'ERROR: unknown option for "gam update domain sso_settings...": %s' % sys.argv[i] sys.exit(2) - callGData(service=adminObj, function=u'UpdateSSOSettings', enableSSO=enableSSO, + callGData(adminObj, u'UpdateSSOSettings', enableSSO=enableSSO, samlSignonUri=samlSignonUri, samlLogoutUri=samlLogoutUri, changePasswordUri=changePasswordUri, ssoWhitelist=ssoWhitelist, useDomainSpecificIssuer=useDomainSpecificIssuer) elif command == u'sso_key': keyFile = sys.argv[4] keyData = readFile(keyFile) - callGData(service=adminObj, function=u'UpdateSSOKey', signingKey=keyData) + callGData(adminObj, u'UpdateSSOKey', signingKey=keyData) elif command == u'user_migrations': value = sys.argv[4].lower() if value not in [u'true', u'false']: print u'ERROR: value for user_migrations must be true or false, got %s' % sys.argv[4] sys.exit(2) - result = callGData(service=adminObj, function=u'UpdateUserMigrationStatus', enableUserMigration=value) + result = callGData(adminObj, u'UpdateUserMigrationStatus', enableUserMigration=value) elif command == u'outbound_gateway': gateway = sys.argv[4] mode = sys.argv[6].upper() try: - result = callGData(service=adminObj, function=u'UpdateOutboundGatewaySettings', smartHost=gateway, smtpMode=mode) + result = callGData(adminObj, u'UpdateOutboundGatewaySettings', smartHost=gateway, smtpMode=mode) except TypeError: pass elif command == u'email_route': @@ -7144,7 +7143,7 @@ def doUpdateInstance(): else: print u'ERROR: %s is not a valid argument for "gam update instance email_route"' % sys.argv[i] sys.exit(2) - callGData(service=adminObj, function=u'AddEmailRoute', routeDestination=destination, routeRewriteTo=rewrite_to, routeEnabled=enabled, + callGData(adminObj, u'AddEmailRoute', routeDestination=destination, routeRewriteTo=rewrite_to, routeEnabled=enabled, bounceNotifications=bounce_notifications, accountHandling=account_handling) else: print u'ERROR: %s is not a valid argument for "gam update instance"' % command @@ -7162,44 +7161,44 @@ def doGetInstanceInfo(): if GC_Values[GC_CUSTOMER_ID] != MY_CUSTOMER: customerId = GC_Values[GC_CUSTOMER_ID] else: - result = callGAPI(service=cd.users(), function=u'list', + result = callGAPI(cd.users(), u'list', fields=u'users(customerId)', customer=GC_Values[GC_CUSTOMER_ID], maxResults=1) try: customerId = result[u'users'][0][u'customerId'] except KeyError: customerId = UNKNOWN print u'Customer ID: %s' % customerId - default_language = callGAPI(service=adm.defaultLanguage(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + default_language = callGAPI(adm.defaultLanguage(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Default Language: %s' % default_language[u'entry'][u'apps$property'][0][u'value'] - org_name = callGAPI(service=adm.organizationName(), function='get', domainName=GC_Values[GC_DOMAIN]) + org_name = callGAPI(adm.organizationName(), 'get', domainName=GC_Values[GC_DOMAIN]) print convertUTF8(u'Organization Name: %s' % org_name[u'entry'][u'apps$property'][0][u'value']) - admin_email = callGAPI(service=adm.adminSecondaryEmail(), function='get', domainName=GC_Values[GC_DOMAIN]) + admin_email = callGAPI(adm.adminSecondaryEmail(), 'get', domainName=GC_Values[GC_DOMAIN]) print u'Admin Secondary Email: %s' % admin_email[u'entry'][u'apps$property'][0][u'value'] - max_users = callGAPI(service=adm.maximumNumberOfUsers(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + max_users = callGAPI(adm.maximumNumberOfUsers(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Maximum Users: %s' % max_users[u'entry'][u'apps$property'][0][u'value'] - current_users = callGAPI(service=adm.currentNumberOfUsers(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + current_users = callGAPI(adm.currentNumberOfUsers(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Current Users: %s' % current_users[u'entry'][u'apps$property'][0][u'value'] - is_dom_verified = callGAPI(service=adm.isVerified(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + is_dom_verified = callGAPI(adm.isVerified(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Domain is Verified: %s' % is_dom_verified[u'entry'][u'apps$property'][0][u'value'] - domain_edition = callGAPI(service=adm.edition(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + domain_edition = callGAPI(adm.edition(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Domain Edition: %s' % domain_edition[u'entry'][u'apps$property'][0][u'value'] - customer_pin = callGAPI(service=adm.customerPIN(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + customer_pin = callGAPI(adm.customerPIN(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Customer PIN: %s' % customer_pin[u'entry'][u'apps$property'][0][u'value'] - creation_time = callGAPI(service=adm.creationTime(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + creation_time = callGAPI(adm.creationTime(), u'get', domainName=GC_Values[GC_DOMAIN]) my_date = creation_time[u'entry'][u'apps$property'][0][u'value'] my_date = my_date[:15] my_offset = creation_time[u'entry'][u'apps$property'][0][u'value'][19:] nice_time = datetime.datetime.strptime(my_date, u"%Y%m%dT%H%M%S") print u'Domain Creation Time: %s %s' % (nice_time, my_offset) - country_code = callGAPI(service=adm.countryCode(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + country_code = callGAPI(adm.countryCode(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Domain Country Code: %s' % country_code[u'entry'][u'apps$property'][0][u'value'] - mxverificationstatus = callGAPI(service=adm.mxVerification(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + mxverificationstatus = callGAPI(adm.mxVerification(), u'get', domainName=GC_Values[GC_DOMAIN]) for entry in mxverificationstatus[u'entry'][u'apps$property']: if entry[u'name'] == u'verified': print u'MX Verification Verified: %s' % entry[u'value'] elif entry[u'name'] == u'verificationMethod': print u'MX Verification Method: %s' % entry[u'value'] - ssosettings = callGAPI(service=adm.ssoGeneral(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + ssosettings = callGAPI(adm.ssoGeneral(), u'get', domainName=GC_Values[GC_DOMAIN]) for entry in ssosettings[u'entry'][u'apps$property']: if entry[u'name'] == u'enableSSO': print u'SSO Enabled: %s' % entry[u'value'] @@ -7213,7 +7212,7 @@ def doGetInstanceInfo(): print u'SSO Whitelist IPs: %s' % entry[u'value'] elif entry[u'name'] == u'useDomainSpecificIssuer': print u'SSO Use Domain Specific Issuer: %s' % entry[u'value'] - ssokey = callGAPI(service=adm.ssoSigningKey(), function=u'get', silent_errors=True, soft_errors=True, domainName=GC_Values[GC_DOMAIN]) + ssokey = callGAPI(adm.ssoSigningKey(), u'get', silent_errors=True, soft_errors=True, domainName=GC_Values[GC_DOMAIN]) try: for entry in ssokey[u'entry'][u'apps$property']: if entry[u'name'] == u'algorithm': @@ -7230,10 +7229,10 @@ def doGetInstanceInfo(): print u'Full SSO Key: %s' % entry[u'value'] except TypeError: pass - migration_status = callGAPI(service=adm.userEmailMigrationEnabled(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + migration_status = callGAPI(adm.userEmailMigrationEnabled(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'User Migration Enabled: %s' % migration_status[u'entry'][u'apps$property'][0][u'value'] outbound_gateway_settings = {u'smartHost': u'', u'smtpMode': u''} # Initialize blank in case we get an 1801 Error - outbound_gateway_settings = callGAPI(service=adm.outboundGateway(), function=u'get', domainName=GC_Values[GC_DOMAIN]) + outbound_gateway_settings = callGAPI(adm.outboundGateway(), u'get', domainName=GC_Values[GC_DOMAIN]) try: for entry in outbound_gateway_settings[u'entry'][u'apps$property']: if entry[u'name'] == u'smartHost': @@ -7252,7 +7251,7 @@ def doDeleteUser(): elif user_email.find(u'@') == -1: user_email = u'%s@%s' % (user_email, GC_Values[GC_DOMAIN]) print u"Deleting account for %s" % (user_email) - callGAPI(service=cd.users(), function=u'delete', userKey=user_email) + callGAPI(cd.users(), u'delete', userKey=user_email) def doUndeleteUser(): cd = buildGAPIObject(u'directory') @@ -7273,8 +7272,8 @@ def doUndeleteUser(): user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) if not user_uid: print u'Looking up UID for %s...' % user - deleted_users = callGAPIpages(service=cd.users(), function=u'list', - items=u'users', customer=GC_Values[GC_CUSTOMER_ID], showDeleted=True, maxResults=GC_Values[GC_USER_MAX_RESULTS]) + deleted_users = callGAPIpages(cd.users(), u'list', u'users', + customer=GC_Values[GC_CUSTOMER_ID], showDeleted=True, maxResults=GC_Values[GC_USER_MAX_RESULTS]) matching_users = list() for deleted_user in deleted_users: if str(deleted_user[u'primaryEmail']).lower() == user: @@ -7299,7 +7298,7 @@ def doUndeleteUser(): else: user_uid = matching_users[0][u'id'] print u"Undeleting account for %s" % user - callGAPI(service=cd.users(), function=u'undelete', userKey=user_uid, body={u'orgUnitPath': orgUnit}) + callGAPI(cd.users(), u'undelete', userKey=user_uid, body={u'orgUnitPath': orgUnit}) def doDeleteGroup(): cd = buildGAPIObject(u'directory') @@ -7309,7 +7308,7 @@ def doDeleteGroup(): elif group.find(u'@') == -1: group = u'%s@%s' % (group, GC_Values[GC_DOMAIN]) print u"Deleting group %s" % group - callGAPI(service=cd.groups(), function=u'delete', groupKey=group) + callGAPI(cd.groups(), u'delete', groupKey=group) def doDeleteAlias(alias_email=None): cd = buildGAPIObject(u'directory') @@ -7327,7 +7326,7 @@ def doDeleteAlias(alias_email=None): print u"Deleting alias %s" % alias_email if is_user or (not is_user and not is_group): try: - callGAPI(service=cd.users().aliases(), function=u'delete', throw_reasons=[u'invalid', u'badRequest', u'notFound'], userKey=alias_email, alias=alias_email) + callGAPI(cd.users().aliases(), u'delete', throw_reasons=[u'invalid', u'badRequest', u'notFound'], userKey=alias_email, alias=alias_email) return except googleapiclient.errors.HttpError, e: error = json.loads(e.content) @@ -7336,13 +7335,13 @@ def doDeleteAlias(alias_email=None): print u'Error: The alias %s does not exist' % alias_email sys.exit(4) if not is_user or (not is_user and not is_group): - callGAPI(service=cd.groups().aliases(), function=u'delete', groupKey=alias_email, alias=alias_email) + callGAPI(cd.groups().aliases(), u'delete', groupKey=alias_email, alias=alias_email) def doDeleteResourceCalendar(): resId = sys.argv[3] cd = buildGAPIObject(u'directory') print u"Deleting resource calendar %s" % resId - callGAPI(service=cd.resources().calendars(), function=u'delete', + callGAPI(cd.resources().calendars(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId) def doDeleteOrg(): @@ -7351,7 +7350,7 @@ def doDeleteOrg(): if name[0] == u'/': name = name[1:] print u"Deleting organization %s" % name - callGAPI(service=cd.orgunits(), function=u'delete', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=name) + callGAPI(cd.orgunits(), u'delete', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=name) def output_csv(csv_list, titles, list_type, todrive): csv.register_dialect(u'nixstdout', lineterminator=u'\n') @@ -7370,7 +7369,7 @@ def output_csv(csv_list, titles, list_type, todrive): print u'{0}{1}'.format(WARNING_PREFIX, MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET) convert = False drive = buildGAPIObject(u'drive') - result = callGAPI(service=drive.files(), function=u'insert', convert=convert, + result = callGAPI(drive.files(), u'insert', convert=convert, body={u'description': u' '.join(sys.argv), u'title': u'%s - %s' % (GC_Values[GC_DOMAIN], list_type), u'mimeType': u'text/csv'}, media_body=googleapiclient.http.MediaInMemoryUpload(string_file.getvalue(), mimetype=u'text/csv')) file_url = result[u'alternateLink'] @@ -7534,7 +7533,7 @@ def doPrintUsers(): fields = u'nextPageToken,users(%s)' % u','.join(user_fields) sys.stderr.write(u"Getting all users in Google Apps account (may take some time on a large account)...\n") page_message = u'Got %%total_items%% users: %%first_item%% - %%last_item%%\n' - all_users = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + all_users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, message_attribute=u'primaryEmail', customer=customer, domain=domain, fields=fields, showDeleted=deleted_only, orderBy=orderBy, sortOrder=sortOrder, viewType=viewType, query=query, projection=projection, customFieldMask=customFieldMask, maxResults=GC_Values[GC_USER_MAX_RESULTS]) @@ -7568,7 +7567,7 @@ def doPrintUsers(): for user in attributes[1:]: user_email = user[u'primaryEmail'] sys.stderr.write(u"Getting Group Membership for %s (%s/%s)\r\n" % (user_email, user_count, total_users)) - groups = callGAPIpages(service=cd.groups(), function=u'list', items=u'groups', userKey=user_email) + groups = callGAPIpages(cd.groups(), u'list', u'groups', userKey=user_email) grouplist = u'' for groupname in groups: grouplist += groupname[u'email']+' ' @@ -7669,7 +7668,7 @@ def doPrintGroups(): sys.exit(2) sys.stderr.write(u"Retrieving All Groups for Google Apps account (may take some time on a large account)...\n") page_message = u'Got %%num_items%% groups: %%first_item%% - %%last_item%%\n' - all_groups = callGAPIpages(service=cd.groups(), function=u'list', items=u'groups', page_message=page_message, + all_groups = callGAPIpages(cd.groups(), u'list', u'groups', page_message=page_message, message_attribute=u'email', customer=customer, domain=usedomain, userKey=usemember, fields=fields) total_groups = len(all_groups) count = 0 @@ -7717,7 +7716,7 @@ def doPrintGroups(): roles = u','.join(roles) sys.stderr.write(u' Getting %s for %s (%s of %s)\n' % (roles, group_vals[u'email'], count, total_groups)) page_message = u'Got %%num_items%% members: %%first_item%% - %%last_item%%\n' - all_group_members = callGAPIpages(service=cd.members(), function=u'list', items=u'members', page_message=page_message, + all_group_members = callGAPIpages(cd.members(), u'list', u'members', page_message=page_message, message_attribute=u'email', groupKey=group_vals[u'email'], roles=roles, fields=u'nextPageToken,members(email,role)') if members: all_true_members = list() @@ -7749,7 +7748,7 @@ def doPrintGroups(): if settings: sys.stderr.write(u" Retrieving Settings for group %s (%s of %s)...\r\n" % (group_vals[u'email'], count, total_groups)) gs = buildGAPIObject(u'groupssettings') - settings = callGAPI(service=gs.groups(), function=u'get', retry_reasons=[u'serviceLimit'], groupUniqueId=group_vals[u'email']) + settings = callGAPI(gs.groups(), u'get', retry_reasons=[u'serviceLimit'], groupUniqueId=group_vals[u'email']) for key in settings: if key in [u'email', u'name', u'description', u'kind', u'etag']: continue @@ -7816,7 +7815,7 @@ def doPrintOrgs(): org_attributes[0][u'Path'] = u'Path' titles.append(u'Path') sys.stderr.write(u"Retrieving All Organizational Units for your account (may take some time on large domain)...") - orgs = callGAPI(service=cd.orgunits(), function=u'list', customerId=GC_Values[GC_CUSTOMER_ID], fields=fields, type=listType, orgUnitPath=orgUnitPath) + orgs = callGAPI(cd.orgunits(), u'list', customerId=GC_Values[GC_CUSTOMER_ID], fields=fields, type=listType, orgUnitPath=orgUnitPath) sys.stderr.write(u"done\n") if not u'organizationUnits' in orgs: print u'0 org units in this Google Apps instance...' @@ -7875,7 +7874,7 @@ def doPrintAliases(): titles = [u'Alias', u'Target', u'TargetType'] sys.stderr.write(u"Retrieving All User Aliases for %s organization (may take some time on large domain)...\n" % GC_Values[GC_DOMAIN]) page_message = u'Got %%num_items%% users %%first_item%% - %%last_item%%\n' - all_users = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + all_users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, message_attribute=u'primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], fields=u'users(primaryEmail,aliases),nextPageToken', maxResults=GC_Values[GC_USER_MAX_RESULTS]) for user in all_users: @@ -7886,7 +7885,7 @@ def doPrintAliases(): continue sys.stderr.write(u"Retrieving All User Aliases for %s organization (may take some time on large domain)...\n" % GC_Values[GC_DOMAIN]) page_message = u'Got %%num_items%% groups %%first_item%% - %%last_item%%\n' - all_groups = callGAPIpages(service=cd.groups(), function=u'list', items=u'groups', page_message=page_message, + all_groups = callGAPIpages(cd.groups(), u'list', u'groups', page_message=page_message, message_attribute=u'email', customer=GC_Values[GC_CUSTOMER_ID], fields=u'groups(email,aliases),nextPageToken') for group in all_groups: @@ -7913,14 +7912,14 @@ def doPrintGroupMembers(): sys.exit(2) member_attributes = [{u'group': u'group'},] if not all_groups: - all_groups = callGAPIpages(service=cd.groups(), function=u'list', items=u'groups', message_attribute=u'email', + all_groups = callGAPIpages(cd.groups(), u'list', u'groups', message_attribute=u'email', customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,groups(email)') total_groups = len(all_groups) i = 1 for group in all_groups: group_email = group[u'email'] sys.stderr.write(u'Getting members for %s (%s/%s)\n' % (group_email, i, total_groups)) - group_members = callGAPIpages(service=cd.members(), function=u'list', items=u'members', message_attribute=u'email', groupKey=group_email) + group_members = callGAPIpages(cd.members(), u'list', u'members', message_attribute=u'email', groupKey=group_email) for member in group_members: member_attr = {u'group': group_email} for title in member: @@ -7969,7 +7968,7 @@ def doPrintMobileDevices(): sys.exit(2) sys.stderr.write(u'Retrieving All Mobile Devices for organization (may take some time for large accounts)...\n') page_message = u'Got %%num_items%% mobile devices...\n' - all_mobile = callGAPIpages(service=cd.mobiledevices(), function=u'list', items=u'mobiledevices', page_message=page_message, + all_mobile = callGAPIpages(cd.mobiledevices(), u'list', u'mobiledevices', page_message=page_message, customerId=GC_Values[GC_CUSTOMER_ID], query=query, orderBy=orderBy, sortOrder=sortOrder, maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) for mobile in all_mobile: @@ -8051,7 +8050,7 @@ def doPrintCrosDevices(): projection = u'FULL' sys.stderr.write(u'Retrieving All Chrome OS Devices for organization (may take some time for large accounts)...\n') page_message = u'Got %%num_items%% Chrome devices...\n' - all_cros = callGAPIpages(service=cd.chromeosdevices(), function=u'list', items=u'chromeosdevices', page_message=page_message, + all_cros = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', page_message=page_message, query=query, customerId=GC_Values[GC_CUSTOMER_ID], projection=projection, orderBy=orderBy, sortOrder=sortOrder, maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) if all_cros: @@ -8118,7 +8117,7 @@ def doPrintLicenses(return_list=False, skus=None): product, sku = getProductAndSKU(sku) page_message = u'Got %%%%total_items%%%% Licenses for %s...\n' % sku try: - licenses += callGAPIpages(service=lic.licenseAssignments(), function=u'listForProductAndSku', throw_reasons=[u'invalid', u'forbidden'], page_message=page_message, + licenses += callGAPIpages(lic.licenseAssignments(), u'listForProductAndSku', u'items', throw_reasons=[u'invalid', u'forbidden'], page_message=page_message, customerId=GC_Values[GC_DOMAIN], productId=product, skuId=sku, fields=u'items(productId,skuId,userId),nextPageToken') except googleapiclient.errors.HttpError: licenses += [] @@ -8126,7 +8125,7 @@ def doPrintLicenses(return_list=False, skus=None): for productId in products: page_message = u'Got %%%%total_items%%%% Licenses for %s...\n' % productId try: - licenses += callGAPIpages(service=lic.licenseAssignments(), function=u'listForProduct', throw_reasons=[u'invalid', u'forbidden'], page_message=page_message, + licenses += callGAPIpages(lic.licenseAssignments(), u'listForProduct', u'items', throw_reasons=[u'invalid', u'forbidden'], page_message=page_message, customerId=GC_Values[GC_DOMAIN], productId=productId, fields=u'items(productId,skuId,userId),nextPageToken') except googleapiclient.errors.HttpError: licenses = +[] @@ -8167,7 +8166,7 @@ def doPrintTokens(): token_attributes[0][title] = title for user in all_users: sys.stderr.write(u' getting tokens for %s\n' % user) - user_tokens = callGAPI(service=cd.tokens(), function='list', userKey=user) + user_tokens = callGAPI(cd.tokens(), 'list', userKey=user) try: for user_token in user_tokens[u'items']: this_token = dict() @@ -8214,7 +8213,7 @@ def doPrintResourceCalendars(): fields = u'nextPageToken,items(%s)' % u','.join(fields) sys.stderr.write(u"Retrieving All Resource Calendars for your account (may take some time on a large domain)\n") page_message = u'Got %%total_items%% resources: %%first_item%% - %%last_item%%\n' - resources = callGAPIpages(service=cd.resources().calendars(), function=u'list', items=u'items', + resources = callGAPIpages(cd.resources().calendars(), u'list', u'items', page_message=page_message, message_attribute=u'resourceId', customer=GC_Values[GC_CUSTOMER_ID], fields=fields, maxResults=500) resources_attrib = [{u'resourceId':u'resourceId', @@ -8272,7 +8271,7 @@ def doCreateMonitor(): if source_user.find('@') > 0: audit.domain = source_user[source_user.find(u'@')+1:] source_user = source_user[:source_user.find(u'@')] - callGData(service=audit, function=u'createEmailMonitor', source_user=source_user, destination_user=destination_user, end_date=end_date, begin_date=begin_date, + callGData(audit, u'createEmailMonitor', source_user=source_user, destination_user=destination_user, end_date=end_date, begin_date=begin_date, incoming_headers_only=incoming_headers_only, outgoing_headers_only=outgoing_headers_only, drafts=drafts, drafts_headers_only=drafts_headers_only, chats=chats, chats_headers_only=chats_headers_only) @@ -8282,7 +8281,7 @@ def doShowMonitors(): if user.find('@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] - results = callGData(service=audit, function=u'getEmailMonitors', user=user) + results = callGData(audit, u'getEmailMonitors', user=user) print sys.argv[4].lower()+u' has the following monitors:' print u'' for monitor in results: @@ -8305,7 +8304,7 @@ def doDeleteMonitor(): if source_user.find(u'@') > 0: audit.domain = source_user[source_user.find(u'@')+1:] source_user = source_user[:source_user.find(u'@')] - callGData(service=audit, function=u'deleteEmailMonitor', source_user=source_user, destination_user=destination_user) + callGData(audit, u'deleteEmailMonitor', source_user=source_user, destination_user=destination_user) def doRequestActivity(): user = sys.argv[4].lower() @@ -8313,7 +8312,7 @@ def doRequestActivity(): if user.find('@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find('@')] - results = callGData(service=audit, function=u'createAccountInformationRequest', user=user) + results = callGData(audit, u'createAccountInformationRequest', user=user) print u'Request successfully submitted:' print u' Request ID: '+results[u'requestId'] print u' User: '+results[u'userEmailAddress'] @@ -8329,7 +8328,7 @@ def doStatusActivityRequests(): audit.domain = user[user.find('@')+1:] user = user[:user.find(u'@')] request_id = sys.argv[5].lower() - results = callGData(service=audit, function=u'getAccountInformationRequestStatus', user=user, request_id=request_id) + results = callGData(audit, u'getAccountInformationRequestStatus', user=user, request_id=request_id) print u'' print u' Request ID: '+results[u'requestId'] print u' User: '+results[u'userEmailAddress'] @@ -8344,7 +8343,7 @@ def doStatusActivityRequests(): pass print u'' except IndexError: - results = callGData(service=audit, function=u'getAllAccountInformationRequestsStatus') + results = callGData(audit, u'getAllAccountInformationRequestsStatus') print u'Current Activity Requests:' print u'' for request in results: @@ -8368,7 +8367,7 @@ def doDownloadActivityRequest(): if user.find(u'@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] - results = callGData(service=audit, function=u'getAccountInformationRequestStatus', user=user, request_id=request_id) + results = callGData(audit, u'getAccountInformationRequestStatus', user=user, request_id=request_id) if results[u'status'] != u'COMPLETED': systemErrorExit(4, MESSAGE_REQUEST_NOT_COMPLETE.format(results[u'status'])) if int(results.get(u'numberOfFiles', u'0')) < 1: @@ -8407,7 +8406,7 @@ def doRequestExport(): if user.find('@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] - results = callGData(service=audit, function=u'createMailboxExportRequest', user=user, begin_date=begin_date, end_date=end_date, include_deleted=include_deleted, + results = callGData(audit, u'createMailboxExportRequest', user=user, begin_date=begin_date, end_date=end_date, include_deleted=include_deleted, search_query=search_query, headers_only=headers_only) print u'Export request successfully submitted:' print u' Request ID: '+results['requestId'] @@ -8433,7 +8432,7 @@ def doDeleteExport(): audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] request_id = sys.argv[5].lower() - callGData(service=audit, function=u'deleteMailboxExportRequest', user=user, request_id=request_id) + callGData(audit, u'deleteMailboxExportRequest', user=user, request_id=request_id) def doDeleteActivityRequest(): audit = getAuditObject() @@ -8442,7 +8441,7 @@ def doDeleteActivityRequest(): audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] request_id = sys.argv[5].lower() - callGData(service=audit, function=u'deleteAccountInformationRequest', user=user, request_id=request_id) + callGData(audit, u'deleteAccountInformationRequest', user=user, request_id=request_id) def doStatusExportRequests(): audit = getAuditObject() @@ -8452,7 +8451,7 @@ def doStatusExportRequests(): audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] request_id = sys.argv[5].lower() - results = callGData(service=audit, function=u'getMailboxExportRequestStatus', user=user, request_id=request_id) + results = callGData(audit, u'getMailboxExportRequestStatus', user=user, request_id=request_id) print u'' print u' Request ID: '+results[u'requestId'] print u' User: '+results[u'userEmailAddress'] @@ -8472,7 +8471,7 @@ def doStatusExportRequests(): except KeyError: pass except IndexError: - results = callGData(service=audit, function=u'getAllMailboxExportRequestsStatus') + results = callGData(audit, u'getAllMailboxExportRequestsStatus') print u'Current Export Requests:' print u'' for request in results: @@ -8501,7 +8500,7 @@ def doWatchExportRequest(): user = user[:user.find(u'@')] request_id = sys.argv[5].lower() while True: - results = callGData(service=audit, function=u'getMailboxExportRequestStatus', user=user, request_id=request_id) + results = callGData(audit, u'getMailboxExportRequestStatus', user=user, request_id=request_id) if results[u'status'] != u'PENDING': print u'status is %s. Sending email.' % results[u'status'] msg_txt = u"\n" @@ -8541,7 +8540,7 @@ def send_email(msg_subj, msg_txt, msg_rcpt=None): msg[u'To'] = msg_rcpt msg_string = msg.as_string() msg_raw = base64.urlsafe_b64encode(msg_string) - callGAPI(service=gmail.users().messages(), function=u'send', userId=sender_email, body={u'raw': msg_raw}) + callGAPI(gmail.users().messages(), u'send', userId=sender_email, body={u'raw': msg_raw}) def doDownloadExportRequest(): user = sys.argv[4].lower() @@ -8550,7 +8549,7 @@ def doDownloadExportRequest(): if user.find(u'@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] - results = callGData(service=audit, function=u'getMailboxExportRequestStatus', user=user, request_id=request_id) + results = callGData(audit, u'getMailboxExportRequestStatus', user=user, request_id=request_id) if results[u'status'] != u'COMPLETED': systemErrorExit(4, MESSAGE_REQUEST_NOT_COMPLETE.format(results[u'status'])) if int(results.get(u'numberOfFiles', u'0')) < 1: @@ -8568,7 +8567,7 @@ def doDownloadExportRequest(): def doUploadAuditKey(): auditkey = sys.stdin.read() audit = getAuditObject() - callGData(service=audit, function=u'updatePGPKey', pgpkey=auditkey) + callGData(audit, u'updatePGPKey', pgpkey=auditkey) def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=False, member_type=None): got_uids = False @@ -8594,7 +8593,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if not silent: sys.stderr.write(u"Getting %s of %s (may take some time for large groups)..." % (member_type_message, group)) page_message = u'Got %%%%total_items%%%% %s...' % member_type_message - members = callGAPIpages(service=cd.members(), function=u'list', items=u'members', page_message=page_message, + members = callGAPIpages(cd.members(), u'list', u'members', page_message=page_message, groupKey=group, roles=member_type, fields=u'nextPageToken,members(email,id)') users = [] for member in members: @@ -8612,7 +8611,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if not silent: sys.stderr.write(u"Getting all users in the Google Apps organization (may take some time on a large domain)...\n") page_message = u'Got %%total_items%% users...' - members = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + members = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail,id,orgUnitPath)', query=u"orgUnitPath='%s'" % ou, maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in members: @@ -8634,7 +8633,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if not silent: sys.stderr.write(u"Getting all users in the Google Apps organization (may take some time on a large domain)...\n") page_message = u'Got %%total_items%% users..' - members = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + members = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail,id)', query=u"orgUnitPath='%s'" % ou, maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in members: @@ -8650,7 +8649,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if not silent: sys.stderr.write(u"Getting all users that match query %s (may take some time on a large domain)...\n" % entity) page_message = u'Got %%total_items%% users...' - members = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + members = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail,id)', query=entity, maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in members: @@ -8684,14 +8683,14 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa entity = u'd:%s' % entity if entity_type in [u'courseparticipants', u'teachers']: page_message = u'Got %%total_items%% teachers...' - teachers = callGAPIpages(service=croom.courses().teachers(), function=u'list', items=u'teachers', page_message=page_message, courseId=entity) + teachers = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', page_message=page_message, courseId=entity) for teacher in teachers: email = teacher[u'profile'].get(u'emailAddress', None) if email: users.append(email) if entity_type in [u'courseparticipants', u'students']: page_message = u'Got %%total_items%% students...' - students = callGAPIpages(service=croom.courses().students(), function=u'list', page_message=page_message, items=u'students', courseId=entity) + students = callGAPIpages(croom.courses().students(), u'list', u'students', page_message=page_message, courseId=entity) for student in students: email = student[u'profile'].get(u'emailAddress', None) if email: @@ -8703,7 +8702,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if not silent: sys.stderr.write(u"Getting all users in Google Apps account (may take some time on a large account)...\n") page_message = u'Got %%total_items%% users...' - all_users = callGAPIpages(service=cd.users(), function=u'list', items=u'users', page_message=page_message, + all_users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail,suspended,id)', maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in all_users: @@ -8717,7 +8716,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa elif entity.lower() == u'cros': if not silent: sys.stderr.write(u"Getting all CrOS devices in Google Apps account (may take some time on a large account)...\n") - all_cros = callGAPIpages(service=cd.chromeosdevices(), function=u'list', items=u'chromeosdevices', + all_cros = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', customerId=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,chromeosdevices(deviceId)', maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) for member in all_cros: @@ -8747,7 +8746,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if return_uids and not got_uids: new_full_users = list() for user in full_users: - user_result = callGAPI(service=cd.users(), function=u'get', userKey=user, fields=u'id') + user_result = callGAPI(cd.users(), u'get', userKey=user, fields=u'id') new_full_users.append(user_result[u'id']) full_users = new_full_users return full_users @@ -8768,7 +8767,7 @@ def OAuthInfo(): access_token = credentials.access_token print u"\nOAuth File: %s" % GC_Values[GC_OAUTH2_TXT] oa2 = buildGAPIObject(u'oauth2') - token_info = callGAPI(service=oa2, function=u'tokeninfo', access_token=access_token) + token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) print u"Client ID: %s" % token_info[u'issued_to'] try: print u"Secret: %s" % credentials.client_secret From cd766d90e412fd91f5f0217547f16f9dde6ebecd Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 24 Dec 2015 23:04:40 -0800 Subject: [PATCH 003/100] Fix doVacation problem --- src/gam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 941f9809..97f8a899 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4884,9 +4884,9 @@ def doWebClips(users): def doVacation(users): subject = message = u'' if sys.argv[4].lower() in true_values: - enable = u'true' + enable = True elif sys.argv[4].lower() in false_values: - enable = u'false' + enable = False else: print u'ERROR: value for "gam vacation" must be true or false, got %s' % sys.argv[4] sys.exit(2) From 2df3aef52dac2a56c6cf2533175f42552819cca3 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Fri, 25 Dec 2015 05:42:47 -0500 Subject: [PATCH 004/100] more fixes to doVacation --- src/gam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index 97f8a899..629fa4ec 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4890,7 +4890,7 @@ def doVacation(users): else: print u'ERROR: value for "gam vacation" must be true or false, got %s' % sys.argv[4] sys.exit(2) - contacts_only = domain_only = u'false' + contacts_only = domain_only = False start_date = end_date = None i = 5 while i < len(sys.argv): @@ -4901,10 +4901,10 @@ def doVacation(users): message = sys.argv[i+1] i += 2 elif sys.argv[i].lower() == u'contactsonly': - contacts_only = u'true' + contacts_only = True i += 1 elif sys.argv[i].lower() == u'domainonly': - domain_only = u'true' + domain_only = True i += 1 elif sys.argv[i].lower() == u'startdate': start_date = sys.argv[i+1] From 4d71b6943cfd0353f7a56204e25c514a372d2077 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 26 Dec 2015 08:46:42 -0800 Subject: [PATCH 005/100] Clean up, gam csv optimization showCalSettings had nested loop over user doLabel cleaned up, has starting point for parsing arguments passed in from main doUpdateUser has starting point for parsing arguments passed in from main gam csv processing is optimized. All '~~xxx~~' and '~xxx' substitutions are found for each argument and loaded into a dictionary once before the loop. During the loop, the substitutions can be performed quickly. --- src/gam.py | 142 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 91 insertions(+), 51 deletions(-) diff --git a/src/gam.py b/src/gam.py index 629fa4ec..8c1d87a2 100755 --- a/src/gam.py +++ b/src/gam.py @@ -365,6 +365,10 @@ def systemErrorExit(sysRC, message): def noPythonSSLExit(): systemErrorExit(8, MESSAGE_NO_PYTHON_SSL) +# Invalid CSV ~Header or ~~Header~~ +def csvFieldErrorExit(fieldName, fieldNames): + systemErrorExit(3, MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(fieldName, u','.join(fieldNames))) + def printLine(message): sys.stdout.write(message+u'\n') # @@ -3145,11 +3149,10 @@ def showCalendars(users): def showCalSettings(users): for user in users: - for user in users: - cal = buildGAPIServiceObject(u'calendar', user) - feed = callGAPI(cal.settings(), 'list') - for setting in feed[u'items']: - print u'%s: %s' % (setting[u'id'], setting[u'value']) + cal = buildGAPIServiceObject(u'calendar', user) + feed = callGAPI(cal.settings(), 'list') + for setting in feed[u'items']: + print u'%s: %s' % (setting[u'id'], setting[u'value']) def showDriveSettings(users): todrive = False @@ -4413,39 +4416,36 @@ def doSnippets(users): i += 1 callGData(emailsettings, u'UpdateGeneral', soft_errors=True, username=user, snippets=SetSnippets) -def doLabel(users): - label = sys.argv[4] - count = len(users) - i = 1 - n = 5 - if sys.argv[3].lower() == u'add': - n = 6 - label = sys.argv[5] +def doLabel(users, i): + label = sys.argv[i] + i += 1 body = {u'name': label} - while n < len(sys.argv): - if sys.argv[n].lower().replace(u'_', u'') == u'labellistvisibility': - if sys.argv[n+1].lower().replace(u'_', u'') == u'hide': + while i < len(sys.argv): + if sys.argv[i].lower().replace(u'_', u'') == u'labellistvisibility': + if sys.argv[i+1].lower().replace(u'_', u'') == u'hide': body[u'labelListVisibility'] = u'labelHide' - elif sys.argv[n+1].lower().replace(u'_', u'') == u'show': + elif sys.argv[i+1].lower().replace(u'_', u'') == u'show': body[u'labelListVisibility'] = u'labelShow' - elif sys.argv[n+1].lower().replace(u'_', u'') == u'showifunread': + elif sys.argv[i+1].lower().replace(u'_', u'') == u'showifunread': body[u'labelListVisibility'] = u'labelShowIfUnread' else: - print u'ERROR: label_list_visibility must be one of hide, show or show_if_unread, got %s' % sys.argv[n+1] + print u'ERROR: label_list_visibility must be one of hide, show or show_if_unread, got %s' % sys.argv[i+1] sys.exit(2) - n += 2 - elif sys.argv[n].lower().replace(u'_', u'') == u'messagelistvisibility': - if sys.argv[n+1].lower().replace(u'_', u'') == u'hide': + i += 2 + elif sys.argv[i].lower().replace(u'_', u'') == u'messagelistvisibility': + if sys.argv[i+1].lower().replace(u'_', u'') == u'hide': body[u'messageListVisibility'] = u'hide' - elif sys.argv[n+1].lower().replace(u'_', u'') == u'show': + elif sys.argv[i+1].lower().replace(u'_', u'') == u'show': body[u'messageListVisibility'] = u'show' else: - print u'ERROR: message_list_visibility must be one of hide or show, got %s' % sys.argv[n+1] + print u'ERROR: message_list_visibility must be one of hide or show, got %s' % sys.argv[i+1] sys.exit(2) - n += 2 + i += 2 else: - print u'ERROR: %s is not a valid argument for this command.' % sys.argv[n] + print u'ERROR: %s is not a valid argument for this command.' % sys.argv[i] sys.exit(2) + count = len(users) + i = 1 for user in users: gmail = buildGAPIServiceObject(u'gmail', user) print u"Creating label %s for %s (%s of %s)" % (label, user, i, count) @@ -5517,15 +5517,11 @@ def doCreateResourceCalendar(): callGAPI(cd.resources().calendars(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) -def doUpdateUser(users): +def doUpdateUser(users, i): cd = buildGAPIObject(u'directory') body = dict() gotPassword = isMD5 = isSHA1 = isCrypt = False is_admin = nohash = None - if sys.argv[1].lower() == u'update': - i = 4 - else: - i = 5 do_update_user = False do_admin_user = False while i < len(sys.argv): @@ -9005,6 +9001,64 @@ def run_batch(items): continue GM_Globals[GM_BATCH_QUEUE].put(python_cmd+item) GM_Globals[GM_BATCH_QUEUE].join() +# +# Process command line arguments, find substitutions +# An argument containing instances of ~~xxx~~ has xxx replaced by the value of field xxx from the CSV file +# An argument containing exactly ~xxx is replaced by the value of field xxx from the CSV file +# Otherwise, the argument is preserved as is +# +# SubFields is a dictionary; the key is the argument number, the value is a list of tuples that mark +# the substition (fieldname, start, end). +# Example: update user '~User' address type work unstructured '~~Street~~, ~~City~~, ~~State~~ ~~ZIP~~' primary +# {2: [('User', 0, 5)], 7: [('Street', 0, 10), ('City', 12, 20), ('State', 22, 31), ('ZIP', 32, 39)]} +# +def getSubFields(i, fieldNames): + subFields = {} + PATTERN = re.compile(r'~~(.+?)~~') + GAM_argv = [] + GAM_argvI = 0 + while i < len(sys.argv): + myarg = sys.argv[i] + if PATTERN.search(myarg): + pos = 0 + while True: + match = PATTERN.search(myarg, pos) + if not match: + break + fieldName = match.group(1) + if fieldName in fieldNames: + subFields.setdefault(GAM_argvI, []) + subFields[GAM_argvI].append((fieldName, match.start(), match.end())) + else: + csvFieldErrorExit(fieldName, fieldNames) + pos = match.end() + GAM_argv.append(myarg) + elif myarg[0] == u'~': + fieldName = myarg[1:] + if fieldName in fieldNames: + subFields[GAM_argvI] = [(fieldName, 0, len(myarg))] + GAM_argv.append(myarg) + else: + csvFieldErrorExit(fieldName, fieldNames) + else: + GAM_argv.append(myarg) + GAM_argvI += 1 + i += 1 + return(GAM_argv, subFields) +# +def processSubFields(GAM_argv, row, subFields): + argv = GAM_argv[:] + for GAM_argvI, fields in subFields.iteritems(): + oargv = argv[GAM_argvI][:] + argv[GAM_argvI] = u'' + pos = 0 + for field in fields: + argv[GAM_argvI] += oargv[pos:field[1]] + if row[field[0]]: + argv[GAM_argvI] += row[field[0]] + pos = field[2] + argv[GAM_argvI] += oargv[pos:] + return argv # Main reload(sys) @@ -9044,24 +9098,10 @@ try: if sys.argv[3].lower() != 'gam': print 'ERROR: "gam csv " should be followed by a full GAM command...' sys.exit(3) - argv_template = sys.argv[4:] - substring_replacements = re.findall(r'~~(.*?)~~', u' '.join(argv_template)) + GAM_argv, subFields = getSubFields(4, input_file.fieldnames) items = list() for row in input_file: - argv = list() - for arg in argv_template: - for substring_replacement in substring_replacements: - try: - arg = arg.replace(u'~~%s~~' % substring_replacement, row[substring_replacement]) - except KeyError: - systemErrorExit(3, u'%s is not in %s' % (substring_replacement, row)) - if arg[0] != '~': - argv.append(arg) - elif arg[1:] in row: - argv.append(row[arg[1:]]) - else: - systemErrorExit(2, MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(arg[1:], ','.join(row.keys()))) - items.append(argv) + items.append(processSubFields(GAM_argv, row, subFields)) closeFile(f) run_batch(items) sys.exit(0) @@ -9099,7 +9139,7 @@ try: sys.exit(0) elif sys.argv[1].lower() == u'update': if sys.argv[2].lower() == u'user': - doUpdateUser([sys.argv[3],]) + doUpdateUser([sys.argv[3],], 4) elif sys.argv[2].lower() == u'group': doUpdateGroup() elif sys.argv[2].lower() in [u'nickname', u'alias']: @@ -9493,7 +9533,7 @@ try: elif addWhat in [u'drivefileacl', u'drivefileacls']: addDriveFileACL(users) elif addWhat in [u'label', u'labels']: - doLabel(users) + doLabel(users, 5) else: print u'ERROR: %s is not a valid argument for "gam add"' % sys.argv[4] sys.exit(2) @@ -9507,7 +9547,7 @@ try: elif sys.argv[4].lower() in [u'license', u'licence']: doLicense(users, u'patch') elif sys.argv[4].lower() == u'user': - doUpdateUser(users) + doUpdateUser(users, 5) elif sys.argv[4].lower() in [u'backupcode', u'backupcodes', u'verificationcodes']: doGenBackupCodes(users) elif sys.argv[4].lower() in [u'drivefile']: @@ -9549,7 +9589,7 @@ try: elif command == u'snippets': doSnippets(users) elif command == u'label': - doLabel(users) + doLabel(users, 4) elif command == u'filter': doFilter(users) elif command == u'forward': From 813d503bb8c80ae689e68f6bb05b1449bda973fa Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 09:15:34 -0500 Subject: [PATCH 006/100] use object already avail instead of buildDiscoveryObject() --- src/gam.py | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/src/gam.py b/src/gam.py index 8c1d87a2..afa154a1 100755 --- a/src/gam.py +++ b/src/gam.py @@ -890,25 +890,6 @@ def buildGAPIServiceObject(api, act_as, soft_errors=False): return False sys.exit(4) -def buildDiscoveryObject(api): - import uritemplate - version = getAPIVer(api) - if api in [u'directory', u'reports']: - api = u'admin' - params = {'api': api, 'apiVersion': version} - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) - requested_url = uritemplate.expand(googleapiclient.discovery.DISCOVERY_URI, params) - resp, content = http.request(requested_url) - if resp.status == 404: - raise googleapiclient.errors.UnknownApiNameOrVersion("name: %s version: %s" % (api, version)) - if resp.status >= 400: - raise googleapiclient.errors.HttpError(resp, content, uri=requested_url) - try: - return json.loads(content) - except ValueError: - sys.stderr.write(u'{0}Failed to parse as JSON: {1}\n'.format(ERROR_PREFIX, content)) - raise googleapiclient.errors.InvalidJsonError() - def commonAppsObjInit(appsObj): if not tryOAuth(appsObj): doRequestOAuth() @@ -5399,6 +5380,7 @@ def doCreateGroup(): got_name = False i = 4 gs_body = dict() + gs = None while i < len(sys.argv): if sys.argv[i].lower() == u'name': body[u'name'] = sys.argv[i+1] @@ -5409,7 +5391,8 @@ def doCreateGroup(): i += 2 else: value = sys.argv[i+1] - gs_object = buildDiscoveryObject(u'groupssettings') + gs = buildGAPIObject(u'groupssettings') + gs_object = gs._rootDesc matches_gs_setting = False for (attrib, params) in gs_object[u'schemas'][u'Groups'][u'properties'].items(): if attrib in [u'kind', u'etag', u'email', u'name', u'description']: @@ -5441,14 +5424,12 @@ def doCreateGroup(): print u'ERROR: %s is not a valid argument for "gam create group"' % sys.argv[i] sys.exit(2) gs_body[attrib] = value - use_gs_api = True i += 2 if not got_name: body[u'name'] = body[u'email'] print u"Creating group %s" % body[u'email'] callGAPI(cd.groups(), u'insert', body=body, fields=u'email') - if use_gs_api: - gs = buildGAPIObject(u'groupssettings') + if gs: callGAPI(gs.groups(), u'patch', retry_reasons=[u'serviceLimit'], groupUniqueId=body[u'email'], body=gs_body) def doCreateAlias(): @@ -6004,7 +5985,7 @@ def doUpdateGroup(): else: i = 4 use_cd_api = False - use_gs_api = False + gs = None gs_body = dict() cd_body = dict() while i < len(sys.argv): @@ -6021,7 +6002,8 @@ def doUpdateGroup(): i += 2 else: value = sys.argv[i+1] - gs_object = buildDiscoveryObject(u'groupssettings') + gs = buildGAPIObject(u'groupssettings') + gs_object = gs._rootDesc matches_gs_setting = False for (attrib, params) in gs_object[u'schemas'][u'Groups'][u'properties'].items(): if attrib in [u'kind', u'etag', u'email']: @@ -6067,8 +6049,7 @@ def doUpdateGroup(): except KeyError: pass cd_result = callGAPI(cd.groups(), u'patch', groupKey=group, body=cd_body) - if use_gs_api: - gs = buildGAPIObject(u'groupssettings') + if gs: if use_cd_api: group = cd_result[u'email'] callGAPI(gs.groups(), u'patch', retry_reasons=[u'serviceLimit'], groupUniqueId=group, body=gs_body) From ef403119d944619d6ad6ae00c7924cccf4cc61a5 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 11:30:03 -0500 Subject: [PATCH 007/100] Intial steps to switch to 100% service accounts Still need to store admin email and granted scopes in a file as well as build these on first run. --- src/gam.py | 402 ++++++++++++----------------------------------------- 1 file changed, 86 insertions(+), 316 deletions(-) diff --git a/src/gam.py b/src/gam.py index afa154a1..5a0bf427 100755 --- a/src/gam.py +++ b/src/gam.py @@ -563,14 +563,23 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) -def tryOAuth(gdataObject): - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - if credentials.access_token_expired: - credentials.refresh(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL])) +def tryOAuth(gdataObject, scope): + credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], + GM_Globals[GM_OAUTH2SERVICE_KEY], + scope=scope, user_agent=GAM_INFO, sub=u'me@u.jaylee.us') + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) + try: + credentials.refresh(http) + except oauth2client.client.AccessTokenRefreshError, e: + if e.message in [u'access_denied', + u'unauthorized_client: Unauthorized client or scope in request.', + u'access_denied: Requested client not authorized.']: + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(scope))) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) + if soft_errors: + return False + sys.exit(4) gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} if not GC_Values[GC_DOMAIN]: GC_Values[GC_DOMAIN] = credentials.id_token.get(u'hd', UNKNOWN).lower() @@ -772,20 +781,10 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -API_SCOPE_MAPPING = { - u'appsactivity': [u'https://www.googleapis.com/auth/activity', - u'https://www.googleapis.com/auth/drive'], - u'calendar': [u'https://www.googleapis.com/auth/calendar',], - u'drive': [u'https://www.googleapis.com/auth/drive',], - u'gmail': [u'https://mail.google.com/',], - u'plus': [u'https://www.googleapis.com/auth/plus.me',], - u'plusDomains': [u'https://www.googleapis.com/auth/plus.me', - u'https://www.googleapis.com/auth/plus.circles.read', - u'https://www.googleapis.com/auth/plus.circles.write'], -} - -def getAPIScope(api): - return API_SCOPE_MAPPING.get(api, []) +def getAPIScope(service): + api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] + granted_scopes = api_scopes # TODO fix to lookup from file + return [val for val in api_scopes if val in granted_scopes] + [u'email'] def getServiceFromDiscoveryDocument(api, version, http): disc_filename = u'%s-%s.json' % (api, version) @@ -802,49 +801,9 @@ def getServiceFromDiscoveryDocument(api, version, http): systemErrorExit(4, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) return googleapiclient.discovery.build_from_document(discovery, base=u'https://www.googleapis.com', http=http) -def buildGAPIObject(api): - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if not credentials or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - credentials.user_agent = GAM_INFO - http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR])) - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' - try: - service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) - except httplib2.CertificateValidationUnsupported: - noPythonSSLExit() - if GC_Values[GC_DOMAIN]: - if not GC_Values[GC_CUSTOMER_ID]: - resp, result = service._http.request(u'https://www.googleapis.com/admin/directory/v1/users?domain={0}&maxResults=1&fields=users(customerId)'.format(GC_Values[GC_DOMAIN])) - try: - resultObj = json.loads(result) - except ValueError: - systemErrorExit(8, u'Unexpected response: {0}'.format(result)) - if resp[u'status'] in [u'403', u'404']: - try: - message = resultObj[u'error'][u'errors'][0][u'message'] - except KeyError: - message = resultObj[u'error'][u'message'] - systemErrorExit(8, u'{0} - {1}'.format(message, GC_Values[GC_DOMAIN])) - try: - GC_Values[GC_CUSTOMER_ID] = resultObj[u'users'][0][u'customerId'] - except KeyError: - GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER - else: - GC_Values[GC_DOMAIN] = credentials.id_token.get(u'hd', UNKNOWN).lower() - if not GC_Values[GC_CUSTOMER_ID]: - GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER - return service - -def buildGAPIServiceObject(api, act_as, soft_errors=False): +def buildGAPIObject(api, act_as=None, soft_errors=False): + if not act_as: + act_as = u'me@u.jaylee.us' if not GM_Globals[GM_OAUTH2SERVICE_KEY]: json_string = readFile(GC_Values[GC_OAUTH2SERVICE_JSON], continueOnError=True, displayError=True) if not json_string: @@ -858,28 +817,27 @@ def buildGAPIServiceObject(api, act_as, soft_errors=False): GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = json_data[u'client_id'] GM_Globals[GM_OAUTH2SERVICE_KEY] = json_data[u'private_key'] except KeyError: - try: - # old format with config in the .json file and key in the .p12 file... - GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = json_data[u'web'][u'client_email'] - GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = json_data[u'web'][u'client_id'] - GM_Globals[GM_OAUTH2SERVICE_KEY] = readFile(GC_Values[GC_OAUTH2SERVICE_JSON].replace(u'.json', u'.p12')) - except KeyError: - printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) - printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) - systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) - scope = getAPIScope(api) + printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) + printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) + systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) + version = getAPIVer(api) + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) + try: + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + except googleapiclient.errors.UnknownApiNameOrVersion: + service = getServiceFromDiscoveryDocument(api, version, http) + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, e) + scope = getAPIScope(service) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=act_as) - http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR])) - version = getAPIVer(api) try: - return googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - return getServiceFromDiscoveryDocument(api, version, http) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) + service._http = credentials.authorize(http) + service._http.request.credentials.refresh(http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', @@ -889,11 +847,12 @@ def buildGAPIServiceObject(api, act_as, soft_errors=False): if soft_errors: return False sys.exit(4) + return service -def commonAppsObjInit(appsObj): - if not tryOAuth(appsObj): +def commonAppsObjInit(appsObj, scope): + if not tryOAuth(appsObj, scope): doRequestOAuth() - tryOAuth(appsObj) + tryOAuth(appsObj, scope) #Identify GAM to Google's Servers appsObj.source = GAM_INFO #Show debugging output if debug.gam exists @@ -903,7 +862,7 @@ def commonAppsObjInit(appsObj): def getAdminSettingsObject(): import gdata.apps.adminsettings.service - return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService()) + return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), scope) def getAuditObject(): import gdata.apps.audit.service @@ -911,7 +870,8 @@ def getAuditObject(): def getEmailSettingsObject(): import gdata.apps.emailsettings.service - return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService()) + scope = [u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/', u'email'] + return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), scope) def geturl(url, dst): import urllib2 @@ -2198,7 +2158,7 @@ def doPrintPrinters(): output_csv(printer_attributes, titles, u'Printers', todrive) def changeCalendarAttendees(users): - cal = buildGAPIServiceObject(u'calendar', users[0]) + cal = buildGAPIObject(u'calendar', users[0]) do_it = True i = 5 allevents = False @@ -2230,7 +2190,7 @@ def changeCalendarAttendees(users): sys.stdout.write(u'Checking user %s\n' % user) if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) - cal = buildGAPIServiceObject(u'calendar', user) + cal = buildGAPIObject(u'calendar', user) page_token = None while True: events_page = callGAPI(cal.events(), u'list', calendarId=user, pageToken=page_token, timeMin=start_date, timeMax=end_date, showDeleted=False, showHiddenInvitations=False) @@ -2280,18 +2240,18 @@ def changeCalendarAttendees(users): break def deleteCalendar(users): - cal = buildGAPIServiceObject(u'calendar', users[0]) + cal = buildGAPIObject(u'calendar', users[0]) calendarId = sys.argv[5] if calendarId.find(u'@') == -1: calendarId = u'%s@%s' % (calendarId, GC_Values[GC_DOMAIN]) for user in users: if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) - cal = buildGAPIServiceObject(u'calendar', user) + cal = buildGAPIObject(u'calendar', user) callGAPI(cal.calendarList(), u'delete', calendarId=calendarId) def addCalendar(users): - cal = buildGAPIServiceObject(u'calendar', users[0]) + cal = buildGAPIObject(u'calendar', users[0]) body = dict() body[u'defaultReminders'] = list() body[u'id'] = sys.argv[5] @@ -2355,7 +2315,7 @@ def addCalendar(users): if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) print u"Subscribing %s to %s calendar (%s of %s)" % (user, body['id'], i, count) - cal = buildGAPIServiceObject(u'calendar', user) + cal = buildGAPIObject(u'calendar', user) callGAPI(cal.calendarList(), u'insert', body=body, colorRgbFormat=colorRgbFormat) i += 1 @@ -2420,7 +2380,7 @@ def updateCalendar(users): count = len(users) for user in users: print u"Updating %s's subscription to calendar %s (%s of %s)" % (user, calendarId, i, count) - cal = buildGAPIServiceObject(u'calendar', user) + cal = buildGAPIObject(u'calendar', user) callGAPI(cal.calendarList(), u'update', calendarId=calendarId, body=body, colorRgbFormat=colorRgbFormat) def doPrinterShowACL(): @@ -2793,7 +2753,7 @@ def doCalendarShowACL(): def doCalendarAddACL(calendarId=None, act_as=None, role=None, scope=None, entity=None): if act_as != None: - cal = buildGAPIServiceObject(u'calendar', act_as) + cal = buildGAPIObject(u'calendar', act_as) else: cal = buildGAPIObject(u'calendar') body = dict() @@ -2862,14 +2822,14 @@ def doCalendarDelACL(): def doCalendarWipeData(): calendarId = sys.argv[2] - cal = buildGAPIServiceObject(u'calendar', calendarId) + cal = buildGAPIObject(u'calendar', calendarId) if calendarId.find(u'@') == -1: calendarId = u'%s@%s' % (calendarId, GC_Values[GC_DOMAIN]) callGAPI(cal.calendars(), u'clear', calendarId=calendarId) def doCalendarAddEvent(): calendarId = sys.argv[2] - cal = buildGAPIServiceObject(u'calendar', calendarId) + cal = buildGAPIObject(u'calendar', calendarId) sendNotifications = timeZone = None i = 4 body = {} @@ -3097,7 +3057,7 @@ def deletePhoto(users): def showCalendars(users): for user in users: - cal = buildGAPIServiceObject(u'calendar', user) + cal = buildGAPIObject(u'calendar', user) feed = callGAPI(cal.calendarList(), u'list') for usercal in feed[u'items']: print u' Name: %s' % usercal['id'] @@ -3130,7 +3090,7 @@ def showCalendars(users): def showCalSettings(users): for user in users: - cal = buildGAPIServiceObject(u'calendar', user) + cal = buildGAPIObject(u'calendar', user) feed = callGAPI(cal.settings(), 'list') for setting in feed[u'items']: print u'%s: %s' % (setting[u'id'], setting[u'value']) @@ -3152,7 +3112,7 @@ def showDriveSettings(users): for user in users: sys.stderr.write(u'Getting Drive settings for %s (%s of %s)\n' % (user, count, len(users))) count += 1 - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) feed = callGAPI(drive.about(), u'get', soft_errors=True) if feed == None: continue @@ -3200,7 +3160,7 @@ def doDriveActivity(users): sys.exit(2) activity_attributes = [{},] for user in users: - activity = buildGAPIServiceObject(u'appsactivity', user) + activity = buildGAPIObject(u'appsactivity', user) page_message = u'Retrieved %%%%total_items%%%% activities for %s' % user feed = callGAPIpages(activity.activities(), u'list', u'activities', page_message=page_message, source=u'drive.google.com', userId=u'me', @@ -3216,7 +3176,7 @@ def doDriveActivity(users): def showDriveFileACL(users): fileId = sys.argv[5] for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) feed = callGAPI(drive.permissions(), u'list', fileId=fileId) for permission in feed[u'items']: try: @@ -3233,7 +3193,7 @@ def delDriveFileACL(users): fileId = sys.argv[5] permissionId = unicode(sys.argv[6]) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if permissionId[:3].lower() == u'id:': permissionId = permissionId[3:] elif permissionId.lower() in [u'anyone']: @@ -3281,7 +3241,7 @@ def addDriveFileACL(users): print u'ERROR: %s is not a valid argument for "gam add drivefileacl"' % sys.argv[i] sys.exit(2) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) result = callGAPI(drive.permissions(), u'insert', fileId=fileId, sendNotificationEmails=sendNotificationEmails, emailMessage=emailMessage, body=body) print result @@ -3316,7 +3276,7 @@ def updateDriveFileACL(users): print u'ERROR: %s is not a valid argument for "gam update drivefileacl"' % sys.argv[i] sys.exit(2) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if permissionId[:3].lower() == u'id:': permissionId = permissionId[3:] else: @@ -3409,7 +3369,7 @@ def showDriveFiles(users): if fields != u'*': fields += ')' for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if user.find(u'@') == -1: print u'ERROR: got %s, expected a full email address' % user sys.exit(2) @@ -3475,7 +3435,7 @@ def deleteDriveFile(users): print u'ERROR: %s is not a valid argument for "gam delete drivefile"' % sys.argv[i] sys.exit(2) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if fileIds[:6].lower() == u'query:': file_ids = doDriveSearch(drive, query=fileIds[6:]) else: @@ -3505,7 +3465,7 @@ def printDriveFolderContents(feed, folderId, indent): def showDriveFileTree(users): for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if user.find(u'@') == -1: print u'ERROR: got %s, expected a full email address' % user sys.exit(2) @@ -3519,7 +3479,7 @@ def showDriveFileTree(users): def deleteEmptyDriveFolders(users): query = u'"me" in owners and mimeType = "application/vnd.google-apps.folder"' for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if user.find(u'@') == -1: print u'ERROR: got %s, expected a full email address' % user sys.exit(2) @@ -3671,7 +3631,7 @@ def doUpdateDriveFile(users): print u'ERROR: you cannot specify both an id and a query.' sys.exit(2) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if parent_query: more_parents = doDriveSearch(drive, query=parent_query) if u'parents' not in body: @@ -3787,7 +3747,7 @@ def createDriveFile(users): print u'ERROR: %s is not a valid argument for "gam create drivefile"' % sys.argv[i] sys.exit(2) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if parent_query: more_parents = doDriveSearch(drive, query=parent_query) if u'parents' not in body: @@ -3851,7 +3811,7 @@ def downloadDriveFile(users): print u'ERROR: you cannot specify both the id and query parameters at the same time.' sys.exit(2) for user in users: - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) if query: fileIds = doDriveSearch(drive, query=query) else: @@ -3920,7 +3880,7 @@ def downloadDriveFile(users): def showDriveFileInfo(users): for user in users: fileId = sys.argv[5] - drive = buildGAPIServiceObject(u'drive', user) + drive = buildGAPIObject(u'drive', user) feed = callGAPI(drive.files(), u'get', fileId=fileId) for setting in feed: if setting == u'kind': @@ -3959,7 +3919,7 @@ def transferSecCals(users): print u'ERROR: %s is not a valid argument for "gam transfer seccals"' % sys.argv[i] sys.exit(2) for user in users: - source_cal = buildGAPIServiceObject(u'calendar', user) + source_cal = buildGAPIObject(u'calendar', user) source_calendars = callGAPIpages(source_cal.calendarList(), u'list', u'items', minAccessRole=u'owner', showHidden=True, fields=u'items(id),nextPageToken') for source_cal in source_calendars: if source_cal[u'id'].find(u'@group.calendar.google.com') != -1: @@ -3978,12 +3938,12 @@ def transferDriveFiles(users): else: print u'ERROR: %s is not a valid argument for "gam transfer drive"' % sys.argv[i] sys.exit(2) - target_drive = buildGAPIServiceObject(u'drive', target_user) + target_drive = buildGAPIObject(u'drive', target_user) target_about = callGAPI(target_drive.about(), u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId') target_drive_free = int(target_about[u'quotaBytesTotal']) - int(target_about[u'quotaBytesUsed']) for user in users: counter = 0 - source_drive = buildGAPIServiceObject(u'drive', user) + source_drive = buildGAPIObject(u'drive', user) source_about = callGAPI(source_drive.about(), u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId, permissionId') source_drive_size = int(source_about[u'quotaBytesUsed']) if target_drive_free < source_drive_size: @@ -4428,7 +4388,7 @@ def doLabel(users, i): count = len(users) i = 1 for user in users: - gmail = buildGAPIServiceObject(u'gmail', user) + gmail = buildGAPIObject(u'gmail', user) print u"Creating label %s for %s (%s of %s)" % (label, user, i, count) i += 1 callGAPI(gmail.users().labels(), u'create', soft_errors=True, userId=user, body=body) @@ -4456,7 +4416,7 @@ def doDeleteMessages(trashOrDelete, users): sys.exit(2) for user in users: print u'Searching messages for %s' % user - gmail = buildGAPIServiceObject(u'gmail', user) + gmail = buildGAPIObject(u'gmail', user) page_message = u'Got %%%%total_items%%%% messages for user %s' % user listResult = callGAPIpages(gmail.users().messages(), u'list', u'messages', page_message=page_message, userId=u'me', q=query, includeSpamTrash=True, soft_errors=True) @@ -4477,7 +4437,7 @@ def doDeleteMessages(trashOrDelete, users): def doDeleteLabel(users): label = sys.argv[5] for user in users: - gmail = buildGAPIServiceObject(u'gmail', user) + gmail = buildGAPIObject(u'gmail', user) print u'Getting all labels for %s...' % user labels = callGAPI(gmail.users().labels(), u'list', userId=user, fields=u'labels(name,id,type)') del_labels = [] @@ -4532,7 +4492,7 @@ def showLabels(users): print u'ERROR: %s is not a valid argument for "gam show labels"' % sys.argv[i] sys.exit(2) for user in users: - gmail = buildGAPIServiceObject(u'gmail', user) + gmail = buildGAPIObject(u'gmail', user) labels = callGAPI(gmail.users().labels(), u'list', userId=user, soft_errors=True) if labels: for label in labels[u'labels']: @@ -4558,7 +4518,7 @@ def showGmailProfile(users): profiles = [{}] for user in users: print 'Getting Gmail profile for %s' % user - gmail = buildGAPIServiceObject(u'gmail', user, soft_errors=True) + gmail = buildGAPIObject(u'gmail', user, soft_errors=True) if not gmail: continue results = callGAPI(gmail.users(), u'getProfile', userId=u'me', soft_errors=True) @@ -4598,7 +4558,7 @@ def updateLabels(users): print u'ERROR: %s is not a valid argument for "gam update labels"' % sys.argv[i] sys.exit(2) for user in users: - gmail = buildGAPIServiceObject(u'gmail', user) + gmail = buildGAPIObject(u'gmail', user) labels = callGAPI(gmail.users().labels(), u'list', userId=user, fields=u'labels(id,name)') label_id = None for label in labels[u'labels']: @@ -4629,7 +4589,7 @@ def renameLabels(users): sys.exit(2) pattern = re.compile(search, re.IGNORECASE) for user in users: - gmail = buildGAPIServiceObject(u'gmail', user) + gmail = buildGAPIObject(u'gmail', user) labels = callGAPI(gmail.users().labels(), u'list', userId=user) for label in labels[u'labels']: if label[u'type'] == u'system': @@ -8758,198 +8718,8 @@ def OAuthInfo(): except KeyError: print u'Google Apps Admin: Unknown' -def doDeleteOAuth(): - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - try: - credentials.revoke_uri = oauth2client.GOOGLE_REVOKE_URI - except AttributeError: - systemErrorExit(1, u'Authorization doesn\'t exist') - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) - sys.stderr.write(u'This OAuth token will self-destruct in 3...') - time.sleep(1) - sys.stderr.write(u'2...') - time.sleep(1) - sys.stderr.write(u'1...') - time.sleep(1) - sys.stderr.write(u'boom!\n') - try: - credentials.revoke(http) - except oauth2client.client.TokenRevokeError, e: - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e.message)) - os.remove(GC_Values[GC_OAUTH2_TXT]) - -class cmd_flags(object): - def __init__(self, noLocalWebserver): - self.short_url = True - self.noauth_local_webserver = noLocalWebserver - self.logging_level = u'ERROR' - self.auth_host_name = u'localhost' - self.auth_host_port = [8080, 9090] - -possible_scopes = [u'https://www.googleapis.com/auth/admin.directory.group', # Groups Directory Scope - u'https://www.googleapis.com/auth/admin.directory.orgunit', # Organization Directory Scope - u'https://www.googleapis.com/auth/admin.directory.user', # Users Directory Scope - u'https://www.googleapis.com/auth/admin.directory.device.chromeos', # Chrome OS Devices Directory Scope - u'https://www.googleapis.com/auth/admin.directory.device.mobile', # Mobile Device Directory Scope - u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/', # Email Settings API - u'https://www.googleapis.com/auth/admin.directory.resource.calendar',# Resource Calendar API - u'https://apps-apis.google.com/a/feeds/compliance/audit/', # Email Audit API - u'https://apps-apis.google.com/a/feeds/domain/', # Admin Settings API - u'https://www.googleapis.com/auth/apps.groups.settings', # Group Settings API - u'https://www.googleapis.com/auth/calendar', # Calendar Data API - u'https://www.googleapis.com/auth/admin.reports.audit.readonly', # Audit Reports - u'https://www.googleapis.com/auth/admin.reports.usage.readonly', # Usage Reports - u'https://www.googleapis.com/auth/drive.file', # Drive API - Admin user access to files created or opened by the app - u'https://www.googleapis.com/auth/apps.licensing', # License Manager API - u'https://www.googleapis.com/auth/admin.directory.user.security', # User Security Directory API - u'https://www.googleapis.com/auth/admin.directory.notifications', # Notifications Directory API - u'https://www.googleapis.com/auth/siteverification', # Site Verification API - u'https://mail.google.com/', # IMAP/SMTP authentication for admin notifications - u'https://www.googleapis.com/auth/admin.directory.userschema', # Customer User Schema - u'https://www.googleapis.com/auth/classroom.rosters https://www.googleapis.com/auth/classroom.courses https://www.googleapis.com/auth/classroom.profile.emails https://www.googleapis.com/auth/classroom.profile.photos', # Classroom API - u'https://www.googleapis.com/auth/cloudprint', # CloudPrint API - u'https://www.googleapis.com/auth/admin.datatransfer', # Data Transfer API - u'https://www.googleapis.com/auth/admin.directory.customer', # Customer API - u'https://www.googleapis.com/auth/admin.directory.domain', # Domain API - u'https://www.googleapis.com/auth/admin.directory.rolemanagement', # Roles API - ] - -def doRequestOAuth(incremental_auth=False): - MISSING_CLIENT_SECRETS_MESSAGE = u""" -WARNING: Please configure OAuth 2.0 - -To make GAM run you will need to populate the client_secrets.json file -found at: - - %s - -with information from the APIs Console . - -See: - -https://github.com/jay0lee/GAM/wiki/CreatingClientSecretsFile - -for instructions. - -""" % GC_Values[GC_CLIENT_SECRETS_JSON] - num_scopes = len(possible_scopes) - menu = u'''Select the authorized scopes for this token. Include a 'r' to grant read-only -access or an 'a' to grant action-only access. - -[%%s] %s) Group Directory API (supports read-only) -[%%s] %s) Organizational Unit Directory API (supports read-only) -[%%s] %s) User Directory API (supports read-only) -[%%s] %s) Chrome OS Device Directory API (supports read-only) -[%%s] %s) Mobile Device Directory API (supports read-only and action) -[%%s] %s) User Email Settings API -[%%s] %s) Resource Calendar API (supports read-only) -[%%s] %s) Audit Monitors, Activity and Mailbox Exports API -[%%s] %s) Admin Settings API -[%%s] %s) Groups Settings API -[%%s] %s) Calendar Data API (supports read-only) -[%%s] %s) Audit Reports API -[%%s] %s) Usage Reports API -[%%s] %s) Drive API (create report documents for admin user only) -[%%s] %s) License Manager API -[%%s] %s) User Security Directory API -[%%s] %s) Notifications Directory API -[%%s] %s) Site Verification API -[%%s] %s) IMAP/SMTP Access (send notifications to admin) -[%%s] %s) User Schemas (supports read-only) -[%%s] %s) Classroom API -[%%s] %s) Cloud Print API -[%%s] %s) Data Transfer API (supports read-only) -[%%s] %s) Customer Directory API (supports read-only) -[%%s] %s) Domains Directory API (supports read-only) -[%%s] %s) Roles API (supports read-only) - - %%s) Select all scopes - %%s) Unselect all scopes - %%s) Continue -''' % tuple(range(0, num_scopes)) - selected_scopes = [u'*'] * num_scopes - selected_scopes[16] = u' ' - select_all_scopes = unicode(str(num_scopes)) - unselect_all_scopes = unicode(str(num_scopes+1)) - authorize_scopes = unicode(str(num_scopes+2)) - scope_choices = (select_all_scopes, unselect_all_scopes, authorize_scopes) - - os.system([u'clear', u'cls'][os.name == u'nt']) - while True: - menu_fill = tuple(selected_scopes) + scope_choices - selection = raw_input(menu % menu_fill) - try: - if selection.lower().find(u'r') != -1: - selection = int(selection.lower().replace(u'r', u'')) - if selection not in [0, 1, 2, 3, 4, 6, 10, 19, 22, 23, 24, 25]: - os.system([u'clear', u'cls'][os.name == u'nt']) - print u'THAT SCOPE DOES NOT SUPPORT READ-ONLY MODE!\n' - continue - selected_scopes[selection] = u'R' - elif selection.lower().find(u'a') != -1: - selection = int(selection.lower().replace(u'a', u'')) - if selection not in [4,]: - os.system([u'clear', u'cls'][os.name == u'nt']) - print u'THAT SCOPE DOES NOT SUPPORT ACTION-ONLY MODE!\n' - continue - selected_scopes[selection] = u'A' - elif int(selection) > -1 and int(selection) < num_scopes: - if selected_scopes[int(selection)] == u' ': - selected_scopes[int(selection)] = u'*' - else: - selected_scopes[int(selection)] = u' ' - elif selection == select_all_scopes: - for i in xrange(0, num_scopes): - selected_scopes[i] = u'*' - elif selection == unselect_all_scopes: - for i in xrange(0, num_scopes): - selected_scopes[i] = u' ' - elif selection == authorize_scopes: - at_least_one = False - for i in range(0, len(selected_scopes)): - if selected_scopes[i] in [u'*', u'R', u'A']: - at_least_one = True - if at_least_one: - break - else: - os.system([u'clear', u'cls'][os.name == u'nt']) - print u"YOU MUST SELECT AT LEAST ONE SCOPE!\n" - continue - else: - os.system([u'clear', u'cls'][os.name == u'nt']) - print u'NOT A VALID SELECTION!' - continue - os.system([u'clear', u'cls'][os.name == u'nt']) - except ValueError: - os.system([u'clear', u'cls'][os.name == u'nt']) - print u'Not a valid selection.' - continue - - if incremental_auth: - scopes = [] - else: - scopes = [u'email',] # Email Display Scope, always included - for i in range(0, len(selected_scopes)): - if selected_scopes[i] == u'*': - scopes.append(possible_scopes[i]) - elif selected_scopes[i] == u'R': - scopes.append(u'%s.readonly' % possible_scopes[i]) - elif selected_scopes[i] == u'A': - scopes.append(u'%s.action' % possible_scopes[i]) - try: - FLOW = oauth2client.client.flow_from_clientsecrets(GC_Values[GC_CLIENT_SECRETS_JSON], scope=scopes) - except oauth2client.client.clientsecrets.InvalidClientSecretsError: - systemErrorExit(14, MISSING_CLIENT_SECRETS_MESSAGE) - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - flags = cmd_flags(noLocalWebserver=GC_Values[GC_NO_BROWSER]) - if credentials is None or credentials.invalid or incremental_auth: - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) - try: - credentials = oauth2client.tools.run_flow(flow=FLOW, storage=storage, flags=flags, http=http) - except httplib2.CertificateValidationUnsupported: - noPythonSSLExit() +def doRequestOAuth(): + pass def batch_worker(): while True: From c629c3424ce16f589712066c7037d484a7dbc35f Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 12:08:37 -0500 Subject: [PATCH 008/100] TODOs for a few ugly hard coded hacks --- src/gam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 5a0bf427..c4ee5ff7 100755 --- a/src/gam.py +++ b/src/gam.py @@ -566,7 +566,7 @@ def doGAMVersion(): def tryOAuth(gdataObject, scope): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=u'me@u.jaylee.us') + scope=scope, user_agent=GAM_INFO, sub=u'me@u.jaylee.us') # TODO lookup admin user from file http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) try: @@ -803,7 +803,7 @@ def getServiceFromDiscoveryDocument(api, version, http): def buildGAPIObject(api, act_as=None, soft_errors=False): if not act_as: - act_as = u'me@u.jaylee.us' + act_as = u'me@u.jaylee.us' # TODO lookup admin user from file if not GM_Globals[GM_OAUTH2SERVICE_KEY]: json_string = readFile(GC_Values[GC_OAUTH2SERVICE_JSON], continueOnError=True, displayError=True) if not json_string: From 08163cc5cd83b0ad11c88a8a15594646628747d3 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 12:13:57 -0500 Subject: [PATCH 009/100] remove some legacy variables --- src/gam.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/gam.py b/src/gam.py index c4ee5ff7..aeefc918 100755 --- a/src/gam.py +++ b/src/gam.py @@ -67,11 +67,9 @@ ERROR = u'ERROR' ERROR_PREFIX = ERROR+u': ' WARNING = u'WARNING' WARNING_PREFIX = WARNING+u': ' -FN_CLIENT_SECRETS_JSON = u'client_secrets.json' FN_EXTRA_ARGS_TXT = u'extra-args.txt' FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' FN_OAUTH2SERVICE_JSON = u'oauth2service.json' -FN_OAUTH2_TXT = u'oauth2.txt' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' # @@ -134,8 +132,6 @@ GC_AUTO_BATCH_MIN = u'auto_batch_min' GC_CACHE_DIR = u'cache_dir' # Character set of batch, csv, data files GC_CHARSET = u'charset' -# Path to client_secrets.json -GC_CLIENT_SECRETS_JSON = u'client_secrets_json' # GAM config directory containing client_secrets.json, oauth2.txt, oauth2service.json, extra_args.txt GC_CONFIG_DIR = u'config_dir' # custmerId from gam.cfg or retrieved from Google @@ -150,9 +146,6 @@ GC_DOMAIN = u'domain' GC_DRIVE_DIR = u'drive_dir' # When retrieving lists of Drive files/folders from API, how many should be retrieved in each chunk GC_DRIVE_MAX_RESULTS = u'drive_max_results' -# If no_browser is False, writeCSVfile won't open a browser when todrive is set -# and doRequestOAuth prints a link and waits for the verification code when oauth2.txt is being created -GC_NO_BROWSER = u'no_browser' # Disable GAM API caching GC_NO_CACHE = u'no_cache' # Disable GAM update check @@ -161,8 +154,6 @@ GC_NO_UPDATE_CHECK = u'no_update_check' GC_NO_VERIFY_SSL = u'no_verify_ssl' # Number of threads for gam batch GC_NUM_THREADS = u'num_threads' -# Path to oauth2.txt -GC_OAUTH2_TXT = u'oauth2_txt' # Path to oauth2service.json GC_OAUTH2SERVICE_JSON = u'oauth2service_json' # Default section to use for processing @@ -183,7 +174,6 @@ GC_Defaults = { GC_AUTO_BATCH_MIN: 0, GC_CACHE_DIR: u'', GC_CHARSET: u'utf-8', - GC_CLIENT_SECRETS_JSON: FN_CLIENT_SECRETS_JSON, GC_CONFIG_DIR: u'', GC_CUSTOMER_ID: u'', GC_DEBUG_LEVEL: 0, @@ -191,12 +181,10 @@ GC_Defaults = { GC_DOMAIN: u'', GC_DRIVE_DIR: u'', GC_DRIVE_MAX_RESULTS: 1000, - GC_NO_BROWSER: FALSE, GC_NO_CACHE: FALSE, GC_NO_UPDATE_CHECK: FALSE, GC_NO_VERIFY_SSL: FALSE, GC_NUM_THREADS: 5, - GC_OAUTH2_TXT: FN_OAUTH2_TXT, GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON, GC_SECTION: u'', GC_SHOW_COUNTS_MIN: 1, @@ -225,7 +213,6 @@ GC_VAR_INFO = { GC_AUTO_BATCH_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, GC_CACHE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_CHARSET: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, - GC_CLIENT_SECRETS_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, GC_CONFIG_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_CUSTOMER_ID: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_DEBUG_LEVEL: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, @@ -233,12 +220,10 @@ GC_VAR_INFO = { GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, - GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_VERIFY_SSL: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NUM_THREADS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, None)}, - GC_OAUTH2_TXT: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, GC_OAUTH2SERVICE_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, GC_SECTION: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_SHOW_COUNTS_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, @@ -465,11 +450,9 @@ def SetGlobalVariables(): _getOldEnvVar(GC_SITE_DIR, u'GAMSITECONFIGDIR') _getOldEnvVar(GC_CACHE_DIR, u'GAMCACHEDIR') _getOldEnvVar(GC_DRIVE_DIR, u'GAMDRIVEDIR') - _getOldEnvVar(GC_OAUTH2_TXT, u'OAUTHFILE') _getOldEnvVar(GC_OAUTH2SERVICE_JSON, u'OAUTHSERVICEFILE') if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' - _getOldEnvVar(GC_CLIENT_SECRETS_JSON, u'CLIENTSECRETS') _getOldEnvVar(GC_DOMAIN, u'GA_DOMAIN') _getOldEnvVar(GC_CUSTOMER_ID, u'CUSTOMER_ID') _getOldEnvVar(GC_CHARSET, u'GAM_CHARSET') @@ -481,7 +464,6 @@ def SetGlobalVariables(): _getOldEnvVar(GC_USER_MAX_RESULTS, u'GAM_USER_MAX_RESULTS') _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', trueValue=4, falseValue=0) _getOldSignalFile(GC_NO_VERIFY_SSL, u'noverifyssl.txt') - _getOldSignalFile(GC_NO_BROWSER, u'nobrowser.txt') _getOldSignalFile(GC_NO_CACHE, u'nocache.txt') _getOldSignalFile(GC_NO_UPDATE_CHECK, u'noupdatecheck.txt') # Assign directories first From a983d23f910222359de2142b7b9667685a66a160 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 09:52:18 -0800 Subject: [PATCH 010/100] Define GAM_ADMIN environment variable --- src/gam.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index aeefc918..1c3d0fee 100755 --- a/src/gam.py +++ b/src/gam.py @@ -125,6 +125,8 @@ GM_Globals = { # # When retrieving lists of Google Drive activities from API, how many should be retrieved in each chunk GC_ACTIVITY_MAX_RESULTS = u'activity_max_results' +# GAM admin user +GC_ADMIN = u'admin' # Automatically generate gam batch command if number of users specified in gam users xxx command exceeds this number # Default: 0, don't automatically generate gam batch commands GC_AUTO_BATCH_MIN = u'auto_batch_min' @@ -171,6 +173,7 @@ GC_USER_MAX_RESULTS = u'user_max_results' GC_Defaults = { GC_ACTIVITY_MAX_RESULTS: 100, + GC_ADMIN = u'', GC_AUTO_BATCH_MIN: 0, GC_CACHE_DIR: u'', GC_CHARSET: u'utf-8', @@ -210,6 +213,7 @@ GC_VAR_LIMITS_KEY = u'lmit' GC_VAR_INFO = { GC_ACTIVITY_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 500)}, + GC_ADMIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_AUTO_BATCH_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, GC_CACHE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_CHARSET: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, @@ -280,7 +284,7 @@ def win32_unicode_argv(): if argc.value > 0: # Remove Python executable and commands if present start = argc.value - len(sys.argv) - return [argv[i] for i in xrange(start, argc.value)] + return [argv[i] for i in range(start, argc.value)] from HTMLParser import HTMLParser from re import sub @@ -454,6 +458,7 @@ def SetGlobalVariables(): if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' _getOldEnvVar(GC_DOMAIN, u'GA_DOMAIN') + _getOldEnvVar(GC_ADMIN, u'GAM_ADMIN') _getOldEnvVar(GC_CUSTOMER_ID, u'CUSTOMER_ID') _getOldEnvVar(GC_CHARSET, u'GAM_CHARSET') _getOldEnvVar(GC_NUM_THREADS, u'GAM_THREADS') @@ -548,7 +553,7 @@ def doGAMVersion(): def tryOAuth(gdataObject, scope): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=u'me@u.jaylee.us') # TODO lookup admin user from file + scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) try: @@ -785,7 +790,7 @@ def getServiceFromDiscoveryDocument(api, version, http): def buildGAPIObject(api, act_as=None, soft_errors=False): if not act_as: - act_as = u'me@u.jaylee.us' # TODO lookup admin user from file + act_as = GC_Values[GC_ADMIN] # TODO lookup admin user from file if not GM_Globals[GM_OAUTH2SERVICE_KEY]: json_string = readFile(GC_Values[GC_OAUTH2SERVICE_JSON], continueOnError=True, displayError=True) if not json_string: From 9e09c06770d953bc8459f5d52283df6ed5d6f5c7 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 12:26:10 -0500 Subject: [PATCH 011/100] gdata scopes as globals --- src/gam.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/gam.py b/src/gam.py index 1c3d0fee..bf964cf3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -72,6 +72,11 @@ FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' + +GDATA_EMAIL_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/' +GDATA_ADMIN_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/domain/' +GDATA_EMAIL_AUDIT_SCOPE = u'https://apps-apis.google.com/a/feeds/compliance/audit/' + # # Global variables # @@ -551,6 +556,7 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope): + scope = [scope, u'email'] credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file @@ -849,16 +855,18 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service - return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), scope) + return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), + GDATA_ADMIN_SETTINGS_SCOPE) def getAuditObject(): import gdata.apps.audit.service - return commonAppsObjInit(gdata.apps.audit.service.AuditService()) + return commonAppsObjInit(gdata.apps.audit.service.AuditService(), + GDATA_EMAIL_AUDIT_SCOPE) def getEmailSettingsObject(): import gdata.apps.emailsettings.service - scope = [u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/', u'email'] - return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), scope) + return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), + GDATA_ADMIN_SETTINGS_SCOPE) def geturl(url, dst): import urllib2 From 0cd8246bdb21b7f14ad608a9d5006558374d1d65 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 12:58:21 -0500 Subject: [PATCH 012/100] remove unused G+ APIs --- src/gam.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index bf964cf3..9992265d 100755 --- a/src/gam.py +++ b/src/gam.py @@ -765,8 +765,6 @@ API_VER_MAPPING = { u'groupssettings': u'v1', u'licensing': u'v1', u'oauth2': u'v2', - u'plus': u'v1', - u'plusDomains': u'v1', u'reports': u'reports_v1', u'siteVerification': u'v1', } From 7515700b1adcd344ba3cfe8e2d1392b1e68d3c45 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 13:18:56 -0500 Subject: [PATCH 013/100] fix GC_Defaults dict init --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 9992265d..33d83f3c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -178,7 +178,7 @@ GC_USER_MAX_RESULTS = u'user_max_results' GC_Defaults = { GC_ACTIVITY_MAX_RESULTS: 100, - GC_ADMIN = u'', + GC_ADMIN: u'', GC_AUTO_BATCH_MIN: 0, GC_CACHE_DIR: u'', GC_CHARSET: u'utf-8', From f99add7a3face9b334d9c157e20305c9ad878c61 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 10:20:18 -0800 Subject: [PATCH 014/100] no_browser.txt still needed by output_csv --- src/gam.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/gam.py b/src/gam.py index bf964cf3..03e2f2d8 100755 --- a/src/gam.py +++ b/src/gam.py @@ -153,6 +153,8 @@ GC_DOMAIN = u'domain' GC_DRIVE_DIR = u'drive_dir' # When retrieving lists of Drive files/folders from API, how many should be retrieved in each chunk GC_DRIVE_MAX_RESULTS = u'drive_max_results' +# If no_browser is False, output_csv won't open a browser when todrive is set +GC_NO_BROWSER = u'no_browser' # Disable GAM API caching GC_NO_CACHE = u'no_cache' # Disable GAM update check @@ -189,6 +191,7 @@ GC_Defaults = { GC_DOMAIN: u'', GC_DRIVE_DIR: u'', GC_DRIVE_MAX_RESULTS: 1000, + GC_NO_BROWSER: FALSE, GC_NO_CACHE: FALSE, GC_NO_UPDATE_CHECK: FALSE, GC_NO_VERIFY_SSL: FALSE, @@ -229,6 +232,7 @@ GC_VAR_INFO = { GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, + GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_VERIFY_SSL: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, @@ -474,6 +478,7 @@ def SetGlobalVariables(): _getOldEnvVar(GC_USER_MAX_RESULTS, u'GAM_USER_MAX_RESULTS') _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', trueValue=4, falseValue=0) _getOldSignalFile(GC_NO_VERIFY_SSL, u'noverifyssl.txt') + _getOldSignalFile(GC_NO_BROWSER, u'nobrowser.txt') _getOldSignalFile(GC_NO_CACHE, u'nocache.txt') _getOldSignalFile(GC_NO_UPDATE_CHECK, u'noupdatecheck.txt') # Assign directories first From d871378336fec4b99d1a9122c1c211cc7f91ea7f Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 13:25:06 -0500 Subject: [PATCH 015/100] dynamically get API scopes for "gam oauth request" --- src/gam.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 33d83f3c..8eebc0f0 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8712,7 +8712,26 @@ def OAuthInfo(): print u'Google Apps Admin: Unknown' def doRequestOAuth(): - pass + admin_email = raw_input(u'Please enter your admin email address: ') + apis = API_VER_MAPPING.keys() + apis.remove(u'oauth2') + for api in apis: + version = getAPIVer(api) + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) + try: + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + except googleapiclient.errors.UnknownApiNameOrVersion: + service = getServiceFromDiscoveryDocument(api, version, http) + print u'%s: %s' % (service._rootDesc['title'], service._rootDesc['description']) + for scope in service._rootDesc[u'auth'][u'oauth2'][u'scopes'].items(): + scope_value = scope[0] + scope_description = scope[1][u'description'] + print u' %s\n %s' % (scope_value, scope_description) + print + print def batch_worker(): while True: From 7aaaaf91257adfd0e0e790ac31127269867cfc4e Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 10:59:04 -0800 Subject: [PATCH 016/100] Fix type, delete obsolete function --- src/gam.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index dfdbd295..1fa5b034 100755 --- a/src/gam.py +++ b/src/gam.py @@ -869,7 +869,7 @@ def getAuditObject(): def getEmailSettingsObject(): import gdata.apps.emailsettings.service return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - GDATA_ADMIN_SETTINGS_SCOPE) + GDATA_EMAIL_SETTINGS_SCOPE) def geturl(url, dst): import urllib2 @@ -9110,8 +9110,6 @@ try: doRequestOAuth() elif sys.argv[2].lower() == u'info': OAuthInfo() - elif sys.argv[2].lower() in [u'delete', u'revoke']: - doDeleteOAuth() else: print u'ERROR: %s is not a valid argument for "gam oauth"' % sys.argv[2] sys.exit(2) From 900fc9c4e318241695c6858d34d93f2004e4b4d6 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 14:22:33 -0500 Subject: [PATCH 017/100] sanitize some defaults --- src/gam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 1fa5b034..5da9d282 100755 --- a/src/gam.py +++ b/src/gam.py @@ -185,7 +185,7 @@ GC_Defaults = { GC_CACHE_DIR: u'', GC_CHARSET: u'utf-8', GC_CONFIG_DIR: u'', - GC_CUSTOMER_ID: u'', + GC_CUSTOMER_ID: u'my_customer', GC_DEBUG_LEVEL: 0, GC_DEVICE_MAX_RESULTS: 500, GC_DOMAIN: u'', @@ -580,7 +580,7 @@ def tryOAuth(gdataObject, scope): sys.exit(4) gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = credentials.id_token.get(u'hd', UNKNOWN).lower() + GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][GC_Values[GC_ADMIN].find(u'@'):].lower() if not GC_Values[GC_CUSTOMER_ID]: GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER gdataObject.domain = GC_Values[GC_DOMAIN] From 5ae1f3c44143bb2ea3dcb7664dabf9c2d23832fc Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 11:50:17 -0800 Subject: [PATCH 018/100] Cleanup --- src/gam.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index 5da9d282..2f80fd98 100755 --- a/src/gam.py +++ b/src/gam.py @@ -560,7 +560,7 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) -def tryOAuth(gdataObject, scope): +def tryOAuth(gdataObject, scope, soft_errors=False): scope = [scope, u'email'] credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], @@ -5330,7 +5330,6 @@ def doCreateUser(): def doCreateGroup(): cd = buildGAPIObject(u'directory') - use_gs_api = False body = dict() body[u'email'] = sys.argv[3] if body[u'email'].find(u'@') == -1: @@ -5993,7 +5992,6 @@ def doUpdateGroup(): print u'ERROR: %s is not a valid argument for "gam update group"' % sys.argv[i] sys.exit(2) gs_body[attrib] = value - use_gs_api = True i += 2 if group[:4].lower() == u'uid:': # group settings API won't take uid so we make sure cd API is used so that we can grab real email. use_cd_api = True @@ -6251,12 +6249,7 @@ def doGetUserInfo(user_email=None): user_email = sys.argv[3] i = 4 else: - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - user_email = credentials.id_token[u'email'] + user_email = GC_Values[GC_ADMIN] if user_email[:4].lower() == u'uid:': user_email = user_email[4:] elif user_email.find(u'@') == -1: From 3d6197307197dabb247ebcb6093688aafd531b60 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 15:30:58 -0500 Subject: [PATCH 019/100] discovery files for 2 other gdata apis to standardize scope discovery --- src/email-audit-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/email-settings-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/gam.py | 35 +++++++++++++++++++---------------- 3 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 src/email-audit-v1.json create mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json new file mode 100644 index 00000000..b617e1f5 --- /dev/null +++ b/src/email-audit-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-audit:v1", + "name": "email-audit", + "version": "v1", + "revision": "20130823", + "title": "Email Audit API", + "description": "Lets you peform Google Apps Email Audits", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-audit", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/compliance/audit/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/compliance/audit/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json new file mode 100644 index 00000000..d2ab10be --- /dev/null +++ b/src/email-settings-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-settings:v1", + "name": "email-settings", + "version": "v1", + "revision": "20130823", + "title": "Email Settings API", + "description": "Lets you manage Google Apps Email Settings", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-settings", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/emailsettings/2.0/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/gam.py b/src/gam.py index 5da9d282..0175f277 100755 --- a/src/gam.py +++ b/src/gam.py @@ -73,10 +73,6 @@ FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' -GDATA_EMAIL_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/' -GDATA_ADMIN_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/domain/' -GDATA_EMAIL_AUDIT_SCOPE = u'https://apps-apis.google.com/a/feeds/compliance/audit/' - # # Global variables # @@ -561,7 +557,7 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope): - scope = [scope, u'email'] + scope.append(u'email') credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file @@ -772,6 +768,8 @@ API_VER_MAPPING = { u'oauth2': u'v2', u'reports': u'reports_v1', u'siteVerification': u'v1', + u'email-settings': u'v1', + u'email-audit': u'v1' } def getAPIVer(api): @@ -782,7 +780,7 @@ def getAPIScope(service): granted_scopes = api_scopes # TODO fix to lookup from file return [val for val in api_scopes if val in granted_scopes] + [u'email'] -def getServiceFromDiscoveryDocument(api, version, http): +def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) if hasattr(sys, '_MEIPASS'): @@ -858,18 +856,24 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service + service = getServiceFromDiscoveryDocument(u'admin-settings', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), - GDATA_ADMIN_SETTINGS_SCOPE) + scope) def getAuditObject(): import gdata.apps.audit.service + service = getServiceFromDiscoveryDocument(u'email-audit', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.audit.service.AuditService(), - GDATA_EMAIL_AUDIT_SCOPE) + scope) def getEmailSettingsObject(): import gdata.apps.emailsettings.service + service = getServiceFromDiscoveryDocument(u'email-settings', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - GDATA_EMAIL_SETTINGS_SCOPE) + scope) def geturl(url, dst): import urllib2 @@ -8720,6 +8724,7 @@ def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() apis.remove(u'oauth2') + all_apis = {} for api in apis: version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: @@ -8730,13 +8735,11 @@ def doRequestOAuth(): service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) - print u'%s: %s' % (service._rootDesc['title'], service._rootDesc['description']) - for scope in service._rootDesc[u'auth'][u'oauth2'][u'scopes'].items(): - scope_value = scope[0] - scope_description = scope[1][u'description'] - print u' %s\n %s' % (scope_value, scope_description) - print - print + all_apis[api] = service._rootDesc + i = 0 + for api in all_apis.values(): + print u'[*] %s) %s' % (i, api[u'title']) + i += 1 def batch_worker(): while True: From 881cc4d2555a18601beaa29a872ecef9d02bf68b Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 15:36:24 -0500 Subject: [PATCH 020/100] redo of Ross' latest patch against my updates --- src/gam.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index 0175f277..b00f1d73 100755 --- a/src/gam.py +++ b/src/gam.py @@ -556,7 +556,7 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) -def tryOAuth(gdataObject, scope): +def tryOAuth(gdataObject, scope, soft_errors=False): scope.append(u'email') credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], @@ -5334,7 +5334,6 @@ def doCreateUser(): def doCreateGroup(): cd = buildGAPIObject(u'directory') - use_gs_api = False body = dict() body[u'email'] = sys.argv[3] if body[u'email'].find(u'@') == -1: @@ -5997,7 +5996,6 @@ def doUpdateGroup(): print u'ERROR: %s is not a valid argument for "gam update group"' % sys.argv[i] sys.exit(2) gs_body[attrib] = value - use_gs_api = True i += 2 if group[:4].lower() == u'uid:': # group settings API won't take uid so we make sure cd API is used so that we can grab real email. use_cd_api = True @@ -6255,12 +6253,7 @@ def doGetUserInfo(user_email=None): user_email = sys.argv[3] i = 4 else: - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - user_email = credentials.id_token[u'email'] + user_email = GC_Values[GC_ADMIN] if user_email[:4].lower() == u'uid:': user_email = user_email[4:] elif user_email.find(u'@') == -1: From b6c5f1b1e73b9b012808713e70c5488c351ba06c Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 15:39:40 -0500 Subject: [PATCH 021/100] gdata discovery files wrap into Windows build --- src/gam.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gam.spec b/src/gam.spec index 57abac1f..83620ea1 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,6 +11,8 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] +a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] +a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From ab0bec7a7b3727be67baed4ec6ba5802f03f1096 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 12:54:35 -0800 Subject: [PATCH 022/100] Merge remote-tracking branch 'jay0lee/master' # Conflicts: # src/gam.py --- src/email-audit-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/email-settings-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/gam.py | 35 +++++++++++++++++++---------------- src/gam.spec | 2 ++ 4 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 src/email-audit-v1.json create mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json new file mode 100644 index 00000000..b617e1f5 --- /dev/null +++ b/src/email-audit-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-audit:v1", + "name": "email-audit", + "version": "v1", + "revision": "20130823", + "title": "Email Audit API", + "description": "Lets you peform Google Apps Email Audits", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-audit", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/compliance/audit/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/compliance/audit/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json new file mode 100644 index 00000000..d2ab10be --- /dev/null +++ b/src/email-settings-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-settings:v1", + "name": "email-settings", + "version": "v1", + "revision": "20130823", + "title": "Email Settings API", + "description": "Lets you manage Google Apps Email Settings", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-settings", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/emailsettings/2.0/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/gam.py b/src/gam.py index 2f80fd98..b00f1d73 100755 --- a/src/gam.py +++ b/src/gam.py @@ -73,10 +73,6 @@ FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' -GDATA_EMAIL_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/' -GDATA_ADMIN_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/domain/' -GDATA_EMAIL_AUDIT_SCOPE = u'https://apps-apis.google.com/a/feeds/compliance/audit/' - # # Global variables # @@ -561,7 +557,7 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope, soft_errors=False): - scope = [scope, u'email'] + scope.append(u'email') credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file @@ -772,6 +768,8 @@ API_VER_MAPPING = { u'oauth2': u'v2', u'reports': u'reports_v1', u'siteVerification': u'v1', + u'email-settings': u'v1', + u'email-audit': u'v1' } def getAPIVer(api): @@ -782,7 +780,7 @@ def getAPIScope(service): granted_scopes = api_scopes # TODO fix to lookup from file return [val for val in api_scopes if val in granted_scopes] + [u'email'] -def getServiceFromDiscoveryDocument(api, version, http): +def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) if hasattr(sys, '_MEIPASS'): @@ -858,18 +856,24 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service + service = getServiceFromDiscoveryDocument(u'admin-settings', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), - GDATA_ADMIN_SETTINGS_SCOPE) + scope) def getAuditObject(): import gdata.apps.audit.service + service = getServiceFromDiscoveryDocument(u'email-audit', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.audit.service.AuditService(), - GDATA_EMAIL_AUDIT_SCOPE) + scope) def getEmailSettingsObject(): import gdata.apps.emailsettings.service + service = getServiceFromDiscoveryDocument(u'email-settings', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - GDATA_EMAIL_SETTINGS_SCOPE) + scope) def geturl(url, dst): import urllib2 @@ -8713,6 +8717,7 @@ def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() apis.remove(u'oauth2') + all_apis = {} for api in apis: version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: @@ -8723,13 +8728,11 @@ def doRequestOAuth(): service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) - print u'%s: %s' % (service._rootDesc['title'], service._rootDesc['description']) - for scope in service._rootDesc[u'auth'][u'oauth2'][u'scopes'].items(): - scope_value = scope[0] - scope_description = scope[1][u'description'] - print u' %s\n %s' % (scope_value, scope_description) - print - print + all_apis[api] = service._rootDesc + i = 0 + for api in all_apis.values(): + print u'[*] %s) %s' % (i, api[u'title']) + i += 1 def batch_worker(): while True: diff --git a/src/gam.spec b/src/gam.spec index 57abac1f..83620ea1 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,6 +11,8 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] +a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] +a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From 20bba75e41c3e39ad8ec6ea95c4d9264c5e4bf5b Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 16:12:49 -0500 Subject: [PATCH 023/100] default scope selections basic logic is: -if 1 scope for API, use it -skip over scopes ending in .readonly, .action or .verify_only UNLESS all scopes are readonly, use all scopes (this is case with reports api) -all other scopes are used by default. --- src/gam.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index b00f1d73..66dadf4d 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8728,10 +8728,32 @@ def doRequestOAuth(): service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) - all_apis[api] = service._rootDesc + all_apis[u'%s-%s' % (api, version)] = service._rootDesc i = 0 + # Default Scope Selections + for api_name, api in all_apis.items(): + all_apis[api_name][u'use_scopes'] = [] + scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() + if len(scopes) == 1: + all_apis[api_name][u'use_scopes'] += scopes + continue + all_readonly = True + for scope in api[u'auth'][u'oauth2'][u'scopes'].keys(): + if scope.endswith(u'.readonly'): + continue + elif scope.endswith(u'.action'): + all_readonly = False + continue + elif scope.endswith(u'verify_only'): + all_readonly = False + continue + else: + all_apis[api_name][u'use_scopes'].append(scope) + all_readonly = False + if all_readonly: + all_apis[api_name][u'use_scopes'] += scopes for api in all_apis.values(): - print u'[*] %s) %s' % (i, api[u'title']) + print u'[*] %s) %s (%s scopes)' % (i, api[u'title'], len(api[u'use_scopes'])) i += 1 def batch_worker(): From d2039e55668af9cd06b70e43208de2da7aea4a92 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 16:24:37 -0500 Subject: [PATCH 024/100] cleanup admin settings API name --- src/admin-settings-v1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin-settings-v1.json b/src/admin-settings-v1.json index 5f75e3cb..c1fa4e6e 100644 --- a/src/admin-settings-v1.json +++ b/src/admin-settings-v1.json @@ -5,7 +5,7 @@ "name": "admin-settings", "version": "v1", "revision": "20130823", - "title": "Admin Settings API (read-only calls)", + "title": "Admin Settings API", "description": "Lets you access Google Apps Admin Settings", "ownerDomain": "google.com", "ownerName": "Google", From 12166e2245da2b36b7d6d99ad70febf025049e2e Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 30 Dec 2015 17:10:06 -0500 Subject: [PATCH 025/100] more work on dynamic scope selections --- src/gam.py | 84 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 20 deletions(-) diff --git a/src/gam.py b/src/gam.py index 66dadf4d..e0c4ede0 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8713,24 +8713,7 @@ def OAuthInfo(): except KeyError: print u'Google Apps Admin: Unknown' -def doRequestOAuth(): - admin_email = raw_input(u'Please enter your admin email address: ') - apis = API_VER_MAPPING.keys() - apis.remove(u'oauth2') - all_apis = {} - for api in apis: - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) - try: - service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) - all_apis[u'%s-%s' % (api, version)] = service._rootDesc - i = 0 - # Default Scope Selections +def select_default_scopes(all_apis): for api_name, api in all_apis.items(): all_apis[api_name][u'use_scopes'] = [] scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() @@ -8752,9 +8735,70 @@ def doRequestOAuth(): all_readonly = False if all_readonly: all_apis[api_name][u'use_scopes'] += scopes - for api in all_apis.values(): - print u'[*] %s) %s (%s scopes)' % (i, api[u'title'], len(api[u'use_scopes'])) + return all_apis + +def doRequestOAuth(): + admin_email = raw_input(u'Please enter your admin email address: ') + apis = API_VER_MAPPING.keys() + apis.remove(u'oauth2') + all_apis = {} + for api in apis: + version = getAPIVer(api) + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) + try: + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + except googleapiclient.errors.UnknownApiNameOrVersion: + service = getServiceFromDiscoveryDocument(api, version, http) + all_apis[u'%s-%s' % (api, version)] = service._rootDesc + i = 0 + for api_name in all_apis.keys(): + all_apis[api_name][u'index'] = i i += 1 + all_apis = select_default_scopes(all_apis) + os.system([u'clear', u'cls'][os.name == u'nt']) + while True: + print u'Select the APIs to use with GAM.' + print + for api in all_apis.values(): + num_scopes_selected = len(api[u'use_scopes']) + num_scopes_total = len(api[u'auth'][u'oauth2'][u'scopes']) + if num_scopes_selected > 0: + select_value = u'*' + else: + select_value = u' ' + print u'[%s] %s) %s (%s/%s scopes)' % (select_value, api[u'index'], api[u'title'], num_scopes_selected, num_scopes_total) + print + print u' %s) Select Defaults' % (i+1) + print u' %s) Unselect all APIs' % (i+2) + print u' %s) Continue' % (i+3) + print + selection = int(raw_input(u'Your selection: ')) + if int(selection) == i+1: + all_apis = select_default_scopes(all_apis) + elif selection == i+2: + for api in all_apis.keys(): + all_apis[api][u'use_scopes'] = [] + for api in all_apis.keys(): + if all_apis[api][u'index'] == selection: + if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: + if len(all_apis[api][u'use_scopes']) == 1: + all_apis[api][u'use_scopes'] = [] + else: + all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'] + else: + print + x = 0 + for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): + if scope in all_apis[api][u'use_scopes']: + select_value = u'*' + else: + select_value = u' ' + print u'[%s] %s) %s\n %s\n' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) + x += 1 + print def batch_worker(): while True: From eca89ca5e9a86c15debc62935704d6e2673b1e77 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 15:17:43 -0800 Subject: [PATCH 026/100] Fix typo --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index e0c4ede0..d87f85f3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -576,7 +576,7 @@ def tryOAuth(gdataObject, scope, soft_errors=False): sys.exit(4) gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][GC_Values[GC_ADMIN].find(u'@'):].lower() + GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][GC_Values[GC_ADMIN].find(u'@')+1:].lower() if not GC_Values[GC_CUSTOMER_ID]: GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER gdataObject.domain = GC_Values[GC_DOMAIN] From abde922b496df09555fe4b9e1f8b70acba4a09f8 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 15:38:39 -0800 Subject: [PATCH 027/100] Only build object if necessary --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index d87f85f3..44c94d25 100755 --- a/src/gam.py +++ b/src/gam.py @@ -7089,10 +7089,10 @@ def doGetInstanceInfo(): geturl(url, target_file) sys.exit(0) print u'Google Apps Domain: %s' % (GC_Values[GC_DOMAIN]) - cd = buildGAPIObject(u'directory') if GC_Values[GC_CUSTOMER_ID] != MY_CUSTOMER: customerId = GC_Values[GC_CUSTOMER_ID] else: + cd = buildGAPIObject(u'directory') result = callGAPI(cd.users(), u'list', fields=u'users(customerId)', customer=GC_Values[GC_CUSTOMER_ID], maxResults=1) try: From bf6c2ef2661b52850982fd43de0b3cf618f8252c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 23:39:04 -0800 Subject: [PATCH 028/100] buildGAPIObject reworked This gets everything working but does not address the issue of matching the admins actual scope list. You'd better define GA_DOMAIN as it used to come out of oauth2.txt if it wasn't defined. I had to add Audit API and Site Verification API to the service account list of APIs and downloaded a new oauth2service.json. --- src/email-audit-v1.json | 34 -------------- src/email-settings-v1.json | 34 -------------- src/gam.py | 91 ++++++++++++++++++++++++-------------- src/gam.spec | 2 - 4 files changed, 57 insertions(+), 104 deletions(-) delete mode 100644 src/email-audit-v1.json delete mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json deleted file mode 100644 index b617e1f5..00000000 --- a/src/email-audit-v1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "kind": "discovery#restDescription", - "discoveryVersion": "v1", - "id": "email-audit:v1", - "name": "email-audit", - "version": "v1", - "revision": "20130823", - "title": "Email Audit API", - "description": "Lets you peform Google Apps Email Audits", - "ownerDomain": "google.com", - "ownerName": "Google", - "icons": { - "x16": "http://www.google.com/images/icons/product/search-16.gif", - "x32": "http://www.google.com/images/icons/product/search-32.gif" - }, - "documentationLink": "https://developers.google.com/admin-sdk/email-audit", - "protocol": "rest", - "baseUrl": "https://apps-apis.google.com/", - "rootUrl": "https://apps-apis.google.com/", - "servicePath": "/a/feeds/compliance/audit/", - "auth": { - "oauth2": { - "scopes": { - "https://apps-apis.google.com/a/feeds/compliance/audit/": { - "description": "Manage email audits" - } - } - } - }, - "schemas": { - }, - "resources": { - } -} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json deleted file mode 100644 index d2ab10be..00000000 --- a/src/email-settings-v1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "kind": "discovery#restDescription", - "discoveryVersion": "v1", - "id": "email-settings:v1", - "name": "email-settings", - "version": "v1", - "revision": "20130823", - "title": "Email Settings API", - "description": "Lets you manage Google Apps Email Settings", - "ownerDomain": "google.com", - "ownerName": "Google", - "icons": { - "x16": "http://www.google.com/images/icons/product/search-16.gif", - "x32": "http://www.google.com/images/icons/product/search-32.gif" - }, - "documentationLink": "https://developers.google.com/admin-sdk/email-settings", - "protocol": "rest", - "baseUrl": "https://apps-apis.google.com/", - "rootUrl": "https://apps-apis.google.com/", - "servicePath": "/a/feeds/emailsettings/2.0/", - "auth": { - "oauth2": { - "scopes": { - "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { - "description": "Manage email audits" - } - } - } - }, - "schemas": { - }, - "resources": { - } -} diff --git a/src/gam.py b/src/gam.py index 44c94d25..dd00497a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -104,6 +104,8 @@ GM_MAP_ROLE_ID_TO_NAME = u'ri2n' GM_MAP_ROLE_NAME_TO_ID = u'rn2i' # Dictionary mapping User ID to Name GM_MAP_USER_ID_TO_NAME = u'ui2n' +# Current API scope +GM_API_SCOPE = u'scop' # GM_Globals = { GM_SYSEXITRC: 0, @@ -120,6 +122,7 @@ GM_Globals = { GM_MAP_ROLE_ID_TO_NAME: None, GM_MAP_ROLE_NAME_TO_ID: None, GM_MAP_USER_ID_TO_NAME: None, + GM_API_SCOPE: None, } # # Global variables defined by environment variables/signal files @@ -557,12 +560,12 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope, soft_errors=False): - scope.append(u'email') +# scope.append(u'email') #TODO: What is this for? credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file + scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) + cache=GC_Values[GC_CACHE_DIR]) try: credentials.refresh(http) except oauth2client.client.AccessTokenRefreshError, e: @@ -586,7 +589,7 @@ def checkGDataError(e, service): # First check for errors that need special handling if e[0].get(u'reason', u'') in [u'Token invalid - Invalid token: Stateless token expired', u'Token invalid - Invalid token: Token not found']: keep_domain = service.domain - tryOAuth(service) + tryOAuth(service, GM_Globals[GM_API_SCOPE]) service.domain = keep_domain return False if e[0][u'body'].startswith(u'Required field must not be blank:') or e[0][u'body'].startswith(u'These characters are not allowed:'): @@ -762,19 +765,45 @@ API_VER_MAPPING = { u'datatransfer': u'datatransfer_v1', u'directory': u'directory_v1', u'drive': u'v2', + u'email-audit': u'v1', + u'email-settings': u'v1', u'gmail': u'v1', u'groupssettings': u'v1', u'licensing': u'v1', u'oauth2': u'v2', u'reports': u'reports_v1', u'siteVerification': u'v1', - u'email-settings': u'v1', - u'email-audit': u'v1' } def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') +SERVICE_API_SCOPE_MAPPING = { + u'admin-settings': [u'https://apps-apis.google.com/a/feeds/domain/',], + u'appsactivity': [u'https://www.googleapis.com/auth/activity', u'https://www.googleapis.com/auth/drive'], + u'calendar': [u'https://www.googleapis.com/auth/calendar',], + u'classroom': [u'https://www.googleapis.com/auth/classroom.courses', u'https://www.googleapis.com/auth/classroom.profile.emails', u'https://www.googleapis.com/auth/classroom.profile.photos',], + u'cloudprint': [u'https://www.googleapis.com/auth/cloudprint',], + u'datatransfer': [u'https://www.googleapis.com/auth/admin.datatransfer',], + u'drive': [u'https://www.googleapis.com/auth/drive',], + u'directory': [u'https://www.googleapis.com/auth/admin.datatransfer', u'https://www.googleapis.com/auth/admin.directory.customer', u'https://www.googleapis.com/auth/admin.directory.device.chromeos', + u'https://www.googleapis.com/auth/admin.directory.device.mobile', u'https://www.googleapis.com/auth/admin.directory.device.mobile.action', + u'https://www.googleapis.com/auth/admin.directory.domain', u'https://www.googleapis.com/auth/admin.directory.group', u'https://www.googleapis.com/auth/admin.directory.group.member', + u'https://www.googleapis.com/auth/admin.directory.notifications', u'https://www.googleapis.com/auth/admin.directory.orgunit', u'https://www.googleapis.com/auth/admin.directory.resource.calendar', + u'https://www.googleapis.com/auth/admin.directory.rolemanagement', u'https://www.googleapis.com/auth/admin.directory.user', u'https://www.googleapis.com/auth/admin.directory.user.alias', + u'https://www.googleapis.com/auth/admin.directory.user.security', u'https://www.googleapis.com/auth/admin.directory.userschema',], + u'email-audit': [u'https://apps-apis.google.com/a/feeds/compliance/audit/',], + u'email-settings': [u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/',], + u'gmail': [u'https://mail.google.com/',], + u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], + u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], + u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], + u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], + } + +def getServiceAPIScope(api): + return SERVICE_API_SCOPE_MAPPING.get(api, []) + def getAPIScope(service): api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] granted_scopes = api_scopes # TODO fix to lookup from file @@ -795,9 +824,7 @@ def getServiceFromDiscoveryDocument(api, version, http=None): systemErrorExit(4, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) return googleapiclient.discovery.build_from_document(discovery, base=u'https://www.googleapis.com', http=http) -def buildGAPIObject(api, act_as=None, soft_errors=False): - if not act_as: - act_as = GC_Values[GC_ADMIN] # TODO lookup admin user from file +def getOAuth2ServiceDetails(): if not GM_Globals[GM_OAUTH2SERVICE_KEY]: json_string = readFile(GC_Values[GC_OAUTH2SERVICE_JSON], continueOnError=True, displayError=True) if not json_string: @@ -814,39 +841,41 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) + +def buildGAPIObject(api, act_as=None, soft_errors=False): + sub = act_as if act_as else GC_Values[GC_ADMIN] + getOAuth2ServiceDetails() + GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api) + credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], + GM_Globals[GM_OAUTH2SERVICE_KEY], + scope=GM_Globals[GM_API_SCOPE], user_agent=GAM_INFO, sub=sub) + http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR])) version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) try: - service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + return googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) + return getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scope = getAPIScope(service) - credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], - GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=act_as) - try: - service._http = credentials.authorize(http) - service._http.request.credentials.refresh(http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(scope))) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_API_SCOPE]))) sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: return False sys.exit(4) - return service def commonAppsObjInit(appsObj, scope): - if not tryOAuth(appsObj, scope): + getOAuth2ServiceDetails() + GM_Globals[GM_API_SCOPE] = scope + if not tryOAuth(appsObj, GM_Globals[GM_API_SCOPE]): doRequestOAuth() - tryOAuth(appsObj, scope) + tryOAuth(appsObj, GM_Globals[GM_API_SCOPE]) #Identify GAM to Google's Servers appsObj.source = GAM_INFO #Show debugging output if debug.gam exists @@ -856,24 +885,18 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service - service = getServiceFromDiscoveryDocument(u'admin-settings', u'v1') - scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), - scope) + getServiceAPIScope(u'admin-settings')) def getAuditObject(): import gdata.apps.audit.service - service = getServiceFromDiscoveryDocument(u'email-audit', u'v1') - scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.audit.service.AuditService(), - scope) + getServiceAPIScope(u'email-audit')) def getEmailSettingsObject(): import gdata.apps.emailsettings.service - service = getServiceFromDiscoveryDocument(u'email-settings', u'v1') - scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - scope) + getServiceAPIScope(u'email-settings')) def geturl(url, dst): import urllib2 @@ -8747,7 +8770,7 @@ def doRequestOAuth(): if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) + cache=GC_Values[GC_CACHE_DIR]) try: service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: diff --git a/src/gam.spec b/src/gam.spec index 83620ea1..57abac1f 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,8 +11,6 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] -a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] -a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From b037333d2bfe62f82e742a2007ec3321a92bfa8c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 01:01:05 -0800 Subject: [PATCH 029/100] Clean up OAuthInfo --- src/gam.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/gam.py b/src/gam.py index dd00497a..2a986a1c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -797,6 +797,7 @@ SERVICE_API_SCOPE_MAPPING = { u'gmail': [u'https://mail.google.com/',], u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], + u'oauth2': [u'https://www.googleapis.com/auth/plus.login', u'https://www.googleapis.com/auth/plus.me',], u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], } @@ -8707,20 +8708,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): - if len(sys.argv) > 3: - access_token = sys.argv[3] - else: - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - credentials.user_agent = GAM_INFO - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) - if credentials.access_token_expired: - credentials.refresh(http) - access_token = credentials.access_token - print u"\nOAuth File: %s" % GC_Values[GC_OAUTH2_TXT] + access_token = sys.argv[3] oa2 = buildGAPIObject(u'oauth2') token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) print u"Client ID: %s" % token_info[u'issued_to'] From a3d0a0250a0d7f7675113af8781fa2448cbef93e Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Thu, 31 Dec 2015 08:52:05 -0500 Subject: [PATCH 030/100] Remove usage of oauth2 API for now as it's nearly useless --- src/gam.py | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/gam.py b/src/gam.py index e0c4ede0..0a068edb 100755 --- a/src/gam.py +++ b/src/gam.py @@ -765,7 +765,6 @@ API_VER_MAPPING = { u'gmail': u'v1', u'groupssettings': u'v1', u'licensing': u'v1', - u'oauth2': u'v2', u'reports': u'reports_v1', u'siteVerification': u'v1', u'email-settings': u'v1', @@ -8684,34 +8683,8 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): - if len(sys.argv) > 3: - access_token = sys.argv[3] - else: - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - credentials.user_agent = GAM_INFO - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) - if credentials.access_token_expired: - credentials.refresh(http) - access_token = credentials.access_token - print u"\nOAuth File: %s" % GC_Values[GC_OAUTH2_TXT] - oa2 = buildGAPIObject(u'oauth2') - token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) - print u"Client ID: %s" % token_info[u'issued_to'] - try: - print u"Secret: %s" % credentials.client_secret - except UnboundLocalError: - pass - print u'Scopes:' - for scope in token_info[u'scope'].split(u' '): - print u' %s' % scope - try: - print u'Google Apps Admin: %s' % token_info[u'email'] - except KeyError: - print u'Google Apps Admin: Unknown' + # TODO eventually would be good if this did something to test admin-selected scopes + pass def select_default_scopes(all_apis): for api_name, api in all_apis.items(): @@ -8740,7 +8713,6 @@ def select_default_scopes(all_apis): def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() - apis.remove(u'oauth2') all_apis = {} for api in apis: version = getAPIVer(api) From 433cdfe87dfc4401c9c85f898800a1de2de63adc Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Thu, 31 Dec 2015 08:57:16 -0500 Subject: [PATCH 031/100] define UBER_SCOPES to cut down on unnecessary scope overlap --- src/gam.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/gam.py b/src/gam.py index 0a068edb..920e86de 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8686,8 +8686,16 @@ def OAuthInfo(): # TODO eventually would be good if this did something to test admin-selected scopes pass +UBER_SCOPES = { + u'gmail-v1': [u'https://mail.google.com/'], + u'drive-v2': [u'https://www.googleapis.com/auth/drive'] + } + def select_default_scopes(all_apis): for api_name, api in all_apis.items(): + if api_name in UBER_SCOPES.keys(): + all_apis[api_name][u'use_scopes'] = UBER_SCOPES[api_name] + continue all_apis[api_name][u'use_scopes'] = [] scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() if len(scopes) == 1: From 16add1bf241ca3d17b31032a047e99340073a0b8 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Thu, 31 Dec 2015 09:34:03 -0500 Subject: [PATCH 032/100] UBER_SCOPE for appsactivity-v1 API --- src/gam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 920e86de..d93d3452 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8688,7 +8688,8 @@ def OAuthInfo(): UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], - u'drive-v2': [u'https://www.googleapis.com/auth/drive'] + u'drive-v2': [u'https://www.googleapis.com/auth/drive'], + u'appsactivity-v1': [u'https://www.googleapis.com/auth/activity'] } def select_default_scopes(all_apis): From 56732ea3e8aaa1555f9887ffe03deefa3074828d Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 08:18:28 -0800 Subject: [PATCH 033/100] Commit Jay's changes --- src/email-audit-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/email-settings-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/gam.py | 19 +------------------ src/gam.spec | 2 ++ 4 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 src/email-audit-v1.json create mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json new file mode 100644 index 00000000..b617e1f5 --- /dev/null +++ b/src/email-audit-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-audit:v1", + "name": "email-audit", + "version": "v1", + "revision": "20130823", + "title": "Email Audit API", + "description": "Lets you peform Google Apps Email Audits", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-audit", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/compliance/audit/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/compliance/audit/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json new file mode 100644 index 00000000..d2ab10be --- /dev/null +++ b/src/email-settings-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-settings:v1", + "name": "email-settings", + "version": "v1", + "revision": "20130823", + "title": "Email Settings API", + "description": "Lets you manage Google Apps Email Settings", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-settings", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/emailsettings/2.0/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/gam.py b/src/gam.py index ab3d468b..8a19a4e4 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8707,23 +8707,6 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): -<<<<<<< HEAD - access_token = sys.argv[3] - oa2 = buildGAPIObject(u'oauth2') - token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) - print u"Client ID: %s" % token_info[u'issued_to'] - try: - print u"Secret: %s" % credentials.client_secret - except UnboundLocalError: - pass - print u'Scopes:' - for scope in token_info[u'scope'].split(u' '): - print u' %s' % scope - try: - print u'Google Apps Admin: %s' % token_info[u'email'] - except KeyError: - print u'Google Apps Admin: Unknown' -======= # TODO eventually would be good if this did something to test admin-selected scopes pass @@ -8732,7 +8715,6 @@ UBER_SCOPES = { u'drive-v2': [u'https://www.googleapis.com/auth/drive'], u'appsactivity-v1': [u'https://www.googleapis.com/auth/activity'] } ->>>>>>> jay0lee/master def select_default_scopes(all_apis): for api_name, api in all_apis.items(): @@ -8764,6 +8746,7 @@ def select_default_scopes(all_apis): def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() + apis.remove(u'oauth2') all_apis = {} for api in apis: version = getAPIVer(api) diff --git a/src/gam.spec b/src/gam.spec index 57abac1f..83620ea1 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,6 +11,8 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] +a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] +a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From 2bf8f9164e7c3bbe05a42f2a8e363ed88bc7e283 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 08:59:38 -0800 Subject: [PATCH 034/100] Fix typo, drop unneeded API from table --- src/email-settings-v1.json | 2 +- src/gam.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json index d2ab10be..19ed87f2 100644 --- a/src/email-settings-v1.json +++ b/src/email-settings-v1.json @@ -22,7 +22,7 @@ "oauth2": { "scopes": { "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { - "description": "Manage email audits" + "description": "Manage email settings" } } } diff --git a/src/gam.py b/src/gam.py index 8a19a4e4..c16450f1 100755 --- a/src/gam.py +++ b/src/gam.py @@ -796,7 +796,6 @@ SERVICE_API_SCOPE_MAPPING = { u'gmail': [u'https://mail.google.com/',], u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], - u'oauth2': [u'https://www.googleapis.com/auth/plus.login', u'https://www.googleapis.com/auth/plus.me',], u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], } From 05a36d32454078c1ec2b83ac65a28920d84b53e8 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Thu, 31 Dec 2015 12:44:40 -0500 Subject: [PATCH 035/100] update Google OAuth URIs upstream pull request is: https://github.com/google/oauth2client/pull/368 --- src/oauth2client/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/oauth2client/__init__.py b/src/oauth2client/__init__.py index 4dda02a4..5fcffc46 100644 --- a/src/oauth2client/__init__.py +++ b/src/oauth2client/__init__.py @@ -16,8 +16,8 @@ __version__ = '1.5.1' -GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/auth' +GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/v2/auth' GOOGLE_DEVICE_URI = 'https://accounts.google.com/o/oauth2/device/code' GOOGLE_REVOKE_URI = 'https://accounts.google.com/o/oauth2/revoke' -GOOGLE_TOKEN_URI = 'https://accounts.google.com/o/oauth2/token' -GOOGLE_TOKEN_INFO_URI = 'https://www.googleapis.com/oauth2/v2/tokeninfo' +GOOGLE_TOKEN_URI = 'https://www.googleapis.com/oauth2/v4/token' +GOOGLE_TOKEN_INFO_URI = 'https://www.googleapis.com/oauth2/v3/tokeninfo' From 8501aec7bcf1817b0cea1ade77aee1abc00d0917 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Thu, 31 Dec 2015 12:47:03 -0500 Subject: [PATCH 036/100] gdata discovery file description updates --- src/admin-settings-v1.json | 2 +- src/email-audit-v1.json | 2 +- src/email-settings-v1.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/admin-settings-v1.json b/src/admin-settings-v1.json index c1fa4e6e..ec3b1b0a 100644 --- a/src/admin-settings-v1.json +++ b/src/admin-settings-v1.json @@ -6,7 +6,7 @@ "version": "v1", "revision": "20130823", "title": "Admin Settings API", - "description": "Lets you access Google Apps Admin Settings", + "description": "Lets you access Google Apps admin settings", "ownerDomain": "google.com", "ownerName": "Google", "icons": { diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json index b617e1f5..52fcf1e1 100644 --- a/src/email-audit-v1.json +++ b/src/email-audit-v1.json @@ -6,7 +6,7 @@ "version": "v1", "revision": "20130823", "title": "Email Audit API", - "description": "Lets you peform Google Apps Email Audits", + "description": "Lets you peform Google Apps email audits", "ownerDomain": "google.com", "ownerName": "Google", "icons": { diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json index d2ab10be..269b02f1 100644 --- a/src/email-settings-v1.json +++ b/src/email-settings-v1.json @@ -6,7 +6,7 @@ "version": "v1", "revision": "20130823", "title": "Email Settings API", - "description": "Lets you manage Google Apps Email Settings", + "description": "Lets you manage Google Apps email settings", "ownerDomain": "google.com", "ownerName": "Google", "icons": { @@ -22,7 +22,7 @@ "oauth2": { "scopes": { "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { - "description": "Manage email audits" + "description": "Manage email settings" } } } From 1c4424dd0b12aab62dca41f672d64051b5e685e1 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Thu, 31 Dec 2015 16:47:53 -0500 Subject: [PATCH 037/100] Get doRequestOAuth to actually print out list of scopes --- src/gam.py | 82 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/src/gam.py b/src/gam.py index d93d3452..d69c8ec9 100755 --- a/src/gam.py +++ b/src/gam.py @@ -805,7 +805,6 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): systemErrorExit(6, None) json_data = json.loads(json_string) try: - # new format with config and key in the .json file... GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = json_data[u'client_email'] GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = json_data[u'client_id'] GM_Globals[GM_OAUTH2SERVICE_KEY] = json_data[u'private_key'] @@ -8692,15 +8691,15 @@ UBER_SCOPES = { u'appsactivity-v1': [u'https://www.googleapis.com/auth/activity'] } -def select_default_scopes(all_apis): - for api_name, api in all_apis.items(): +def select_default_scopes(apis): + for api_name, api in apis.items(): if api_name in UBER_SCOPES.keys(): - all_apis[api_name][u'use_scopes'] = UBER_SCOPES[api_name] + apis[api_name][u'use_scopes'] = UBER_SCOPES[api_name] continue - all_apis[api_name][u'use_scopes'] = [] + apis[api_name][u'use_scopes'] = [] scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() if len(scopes) == 1: - all_apis[api_name][u'use_scopes'] += scopes + apis[api_name][u'use_scopes'] += scopes continue all_readonly = True for scope in api[u'auth'][u'oauth2'][u'scopes'].keys(): @@ -8713,11 +8712,11 @@ def select_default_scopes(all_apis): all_readonly = False continue else: - all_apis[api_name][u'use_scopes'].append(scope) + apis[api_name][u'use_scopes'].append(scope) all_readonly = False if all_readonly: - all_apis[api_name][u'use_scopes'] += scopes - return all_apis + apis[api_name][u'use_scopes'] += scopes + return apis def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') @@ -8752,24 +8751,35 @@ def doRequestOAuth(): select_value = u' ' print u'[%s] %s) %s (%s/%s scopes)' % (select_value, api[u'index'], api[u'title'], num_scopes_selected, num_scopes_total) print - print u' %s) Select Defaults' % (i+1) + print u' %s) Select defaults for all APIs (allow all GAM commands)' % (i+1) print u' %s) Unselect all APIs' % (i+2) print u' %s) Continue' % (i+3) print selection = int(raw_input(u'Your selection: ')) - if int(selection) == i+1: + if int(selection) == i+1: # defaults all_apis = select_default_scopes(all_apis) - elif selection == i+2: + elif selection == i+2: # unselect all for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] - for api in all_apis.keys(): - if all_apis[api][u'index'] == selection: - if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: - if len(all_apis[api][u'use_scopes']) == 1: - all_apis[api][u'use_scopes'] = [] - else: - all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'] + elif selection == i+3: + selected_scopes = [u'email'] + for api in all_apis.keys(): + selected_scopes += all_apis[api][u'use_scopes'] + selected_scopes = list(set(selected_scopes)) # unique only + if len(selected_scopes) < 2: + print u'YOU MUST SELECT AT LEAST ONE SCOPE' + continue + break + elif selection >= 0 and selection < len(all_apis.keys()): + api = all_apis.keys()[selection] + if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: + if len(all_apis[api][u'use_scopes']) == 1: + all_apis[api][u'use_scopes'] = [] else: + all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'] + else: + os.system([u'clear', u'cls'][os.name == u'nt']) + while True: print x = 0 for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): @@ -8779,7 +8789,41 @@ def doRequestOAuth(): select_value = u' ' print u'[%s] %s) %s\n %s\n' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) x += 1 + print u' %s) Select defaults for this API (allow all GAM commands)' % (x) + print u' %s) Select read-only scopes' % (x+1) + print u' %s) Unselect all scopes' % (x+2) + print u' %s) Back to all APIs' % (x+3) print + selection = raw_input(u'Your selection: ') + try: + selection = int(selection) + except: + print u'ERROR: please enter numbers only' + continue + num_scopes = len(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()) + if selection >= 0 and selection < num_scopes: + if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in all_apis[api][u'use_scopes']: + all_apis[api][u'use_scopes'].remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + else: + all_apis[api][u'use_scopes'].append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + elif selection == x: + just_this_api = {api: all_apis[api]} + just_this_api = select_default_scopes(just_this_api) + all_apis[api][u'use_scopes'] = just_this_api[api][u'use_scopes'] + elif selection == x+1: + all_apis[api][u'use_scopes'] = [] + for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): + if scope.endswith(u'.readonly'): + all_apis[api][u'use_scopes'].append(scope) + elif selection == x+2: + all_apis[api][u'use_scopes'] = [] + elif selection == x+3: + break + os.system([u'clear', u'cls'][os.name == u'nt']) + os.system([u'clear', u'cls'][os.name == u'nt']) + print u'Please authorize your client id for the %s scopes:' % (len(selected_scopes)) + print + print u','.join(selected_scopes) def batch_worker(): while True: From f6dd0ccd12b8e5368bee9e0d05fc971149ea3518 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 17:30:39 -0800 Subject: [PATCH 038/100] Cleanup doRequestOAuth admin_email isn't used; If it's going to be used I'd say: ` if GC-Values{GC_ADMIN): admin_email = GC_Values[GC_ADMIN] else: admin_email = raw_input(u'Please enter your admin email address: ') ` 'oauth2' isn't in API_VER_MAPPING so the remove fails. If it might go back in but you don't want it here, say: ` if u'oauth2' in apis: apis.remove(u'oauth2') ` --- src/gam.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index 54b94fbe..b34f0c0e 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8742,9 +8742,7 @@ def select_default_scopes(apis): return apis def doRequestOAuth(): - admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() - apis.remove(u'oauth2') all_apis = {} for api in apis: version = getAPIVer(api) @@ -8845,7 +8843,7 @@ def doRequestOAuth(): break os.system([u'clear', u'cls'][os.name == u'nt']) os.system([u'clear', u'cls'][os.name == u'nt']) - print u'Please authorize your client id for the %s scopes:' % (len(selected_scopes)) + print u'Please authorize your service account client id for the %s scopes:' % (len(selected_scopes)) print print u','.join(selected_scopes) From 3182ce031cd90fa84185a83bb9f65bcf1b12c03c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 18:00:29 -0800 Subject: [PATCH 039/100] Leave email scope in tryOAuth --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index b34f0c0e..3237ca99 100755 --- a/src/gam.py +++ b/src/gam.py @@ -560,7 +560,7 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope, soft_errors=False): -# scope.append(u'email') #TODO: What is this for? + scope.append(u'email') credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) From e0c52c8660c5abdbbca7b6f64c25bfd1e0cae37c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 20:20:38 -0800 Subject: [PATCH 040/100] First cut, dynamic scopes Environment variable GAMSCOPESFILE points to scopes file. Scopes file gamscopes.json --- src/gam.py | 63 ++++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/gam.py b/src/gam.py index 3237ca99..f22ddf42 100755 --- a/src/gam.py +++ b/src/gam.py @@ -68,6 +68,7 @@ ERROR_PREFIX = ERROR+u': ' WARNING = u'WARNING' WARNING_PREFIX = WARNING+u': ' FN_EXTRA_ARGS_TXT = u'extra-args.txt' +FN_GAMSCOPES_JSON = u'gamscopes.json' FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' @@ -90,6 +91,8 @@ GM_SYS_ENCODING = u'syen' GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' +# Scopes retrieved from gamscopes.json +GM_GAMSCOPES = u'scop' # Values retrieved from oauth2service.json GM_OAUTH2SERVICE_KEY = u'oauk' GM_OAUTH2SERVICE_ACCOUNT_EMAIL = u'oaae' @@ -105,7 +108,7 @@ GM_MAP_ROLE_NAME_TO_ID = u'rn2i' # Dictionary mapping User ID to Name GM_MAP_USER_ID_TO_NAME = u'ui2n' # Current API scope -GM_API_SCOPE = u'scop' +GM_API_SCOPE = u'csco' # GM_Globals = { GM_SYSEXITRC: 0, @@ -114,6 +117,7 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, + GM_GAMSCOPES: {}, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID: None, @@ -152,6 +156,8 @@ GC_DOMAIN = u'domain' GC_DRIVE_DIR = u'drive_dir' # When retrieving lists of Drive files/folders from API, how many should be retrieved in each chunk GC_DRIVE_MAX_RESULTS = u'drive_max_results' +# Path to gamscopes.json +GC_GAMSCOPES_JSON = u'gamscopes_json' # If no_browser is False, output_csv won't open a browser when todrive is set GC_NO_BROWSER = u'no_browser' # Disable GAM API caching @@ -190,6 +196,7 @@ GC_Defaults = { GC_DOMAIN: u'', GC_DRIVE_DIR: u'', GC_DRIVE_MAX_RESULTS: 1000, + GC_GAMSCOPES_JSON: FN_GAMSCOPES_JSON, GC_NO_BROWSER: FALSE, GC_NO_CACHE: FALSE, GC_NO_UPDATE_CHECK: FALSE, @@ -231,6 +238,7 @@ GC_VAR_INFO = { GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, + GC_GAMSCOPES_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, @@ -465,6 +473,7 @@ def SetGlobalVariables(): _getOldEnvVar(GC_OAUTH2SERVICE_JSON, u'OAUTHSERVICEFILE') if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' + _getOldEnvVar(GC_GAMSCOPES_JSON, u'GAMSCOPESFILE') _getOldEnvVar(GC_DOMAIN, u'GA_DOMAIN') _getOldEnvVar(GC_ADMIN, u'GAM_ADMIN') _getOldEnvVar(GC_CUSTOMER_ID, u'CUSTOMER_ID') @@ -507,6 +516,13 @@ def SetGlobalVariables(): GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None + while True: + json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) + if not json_string: + doRequestOAuth() + continue + GM_Globals[GM_GAMSCOPES] = json.loads(json_string) + break return True def doGAMCheckForUpdates(forceCheck=False): @@ -777,36 +793,10 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -SERVICE_API_SCOPE_MAPPING = { - u'admin-settings': [u'https://apps-apis.google.com/a/feeds/domain/',], - u'appsactivity': [u'https://www.googleapis.com/auth/activity', u'https://www.googleapis.com/auth/drive'], - u'calendar': [u'https://www.googleapis.com/auth/calendar',], - u'classroom': [u'https://www.googleapis.com/auth/classroom.courses', u'https://www.googleapis.com/auth/classroom.profile.emails', u'https://www.googleapis.com/auth/classroom.profile.photos',], - u'cloudprint': [u'https://www.googleapis.com/auth/cloudprint',], - u'datatransfer': [u'https://www.googleapis.com/auth/admin.datatransfer',], - u'drive': [u'https://www.googleapis.com/auth/drive',], - u'directory': [u'https://www.googleapis.com/auth/admin.datatransfer', u'https://www.googleapis.com/auth/admin.directory.customer', u'https://www.googleapis.com/auth/admin.directory.device.chromeos', - u'https://www.googleapis.com/auth/admin.directory.device.mobile', u'https://www.googleapis.com/auth/admin.directory.device.mobile.action', - u'https://www.googleapis.com/auth/admin.directory.domain', u'https://www.googleapis.com/auth/admin.directory.group', u'https://www.googleapis.com/auth/admin.directory.group.member', - u'https://www.googleapis.com/auth/admin.directory.notifications', u'https://www.googleapis.com/auth/admin.directory.orgunit', u'https://www.googleapis.com/auth/admin.directory.resource.calendar', - u'https://www.googleapis.com/auth/admin.directory.rolemanagement', u'https://www.googleapis.com/auth/admin.directory.user', u'https://www.googleapis.com/auth/admin.directory.user.alias', - u'https://www.googleapis.com/auth/admin.directory.user.security', u'https://www.googleapis.com/auth/admin.directory.userschema',], - u'email-audit': [u'https://apps-apis.google.com/a/feeds/compliance/audit/',], - u'email-settings': [u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/',], - u'gmail': [u'https://mail.google.com/',], - u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], - u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], - u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], - u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], - } - -def getServiceAPIScope(api): - return SERVICE_API_SCOPE_MAPPING.get(api, []) - -def getAPIScope(service): - api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] - granted_scopes = api_scopes # TODO fix to lookup from file - return [val for val in api_scopes if val in granted_scopes] + [u'email'] +def getServiceAPIScope(api, version=None): + if not version: + version = getAPIVer(api) + return GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -843,15 +833,15 @@ def getOAuth2ServiceDetails(): def buildGAPIObject(api, act_as=None, soft_errors=False): sub = act_as if act_as else GC_Values[GC_ADMIN] getOAuth2ServiceDetails() - GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api) + version = getAPIVer(api) + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api, version) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=GM_Globals[GM_API_SCOPE], user_agent=GAM_INFO, sub=sub) http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR])) - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' try: return googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: @@ -8785,12 +8775,15 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: selected_scopes = [u'email'] + json_scopes = {} for api in all_apis.keys(): selected_scopes += all_apis[api][u'use_scopes'] + json_scopes[api] = all_apis[api][u'use_scopes'] selected_scopes = list(set(selected_scopes)) # unique only if len(selected_scopes) < 2: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) break elif selection >= 0 and selection < len(all_apis.keys()): api = all_apis.keys()[selection] From 0470680a4d00cc0202ce2c5d57a2becad7d74fa1 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 20:43:33 -0800 Subject: [PATCH 041/100] Add some error checking for gamscopes.json --- src/gam.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index f22ddf42..8ed94a9a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -266,6 +266,7 @@ MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files we MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service Account.' +MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' def convertUTF8(data): @@ -521,7 +522,10 @@ def SetGlobalVariables(): if not json_string: doRequestOAuth() continue - GM_Globals[GM_GAMSCOPES] = json.loads(json_string) + json_data = json.loads(json_string) + if not isinstance(json_data, dict) or (u'scopes' not in json_data) or (not isinstance(json_data[u'scopes'], dict)): + systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) + GM_Globals[GM_GAMSCOPES] = json_data[u'scopes'] break return True @@ -8775,10 +8779,10 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: selected_scopes = [u'email'] - json_scopes = {} + json_scopes = {u'scopes': {}} for api in all_apis.keys(): selected_scopes += all_apis[api][u'use_scopes'] - json_scopes[api] = all_apis[api][u'use_scopes'] + json_scopes[u'scopes'][api] = all_apis[api][u'use_scopes'] selected_scopes = list(set(selected_scopes)) # unique only if len(selected_scopes) < 2: print u'YOU MUST SELECT AT LEAST ONE SCOPE' From ea6f49f7be743e668acb84a1396d855f656a98e1 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 22:54:07 -0800 Subject: [PATCH 042/100] Have OAuthInfo print API scope table --- src/gam.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 8ed94a9a..475feb5a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8699,8 +8699,11 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): - # TODO eventually would be good if this did something to test admin-selected scopes - pass + print u'API Scopes' + for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): + print u' API: {0}'.format(api) + for scope in GM_Globals[GM_GAMSCOPES][api]: + print u' {0}'.format(scope) UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], From a0ac6265e95e6c83c360f6a0182d0523a1b47e8b Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 23:18:44 -0800 Subject: [PATCH 043/100] If SetGlobalVariables calls doRequestOAuth, don't call again if command is oauth create --- src/gam.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 475feb5a..18c37add 100755 --- a/src/gam.py +++ b/src/gam.py @@ -73,7 +73,6 @@ FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' - # # Global variables # @@ -93,6 +92,8 @@ GM_BATCH_QUEUE = u'batq' GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json GM_GAMSCOPES = u'scop' +# gamscopes.json created +GM_GAMSCOPES_CREATED = u'gscr' # Values retrieved from oauth2service.json GM_OAUTH2SERVICE_KEY = u'oauk' GM_OAUTH2SERVICE_ACCOUNT_EMAIL = u'oaae' @@ -118,6 +119,7 @@ GM_Globals = { GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, GM_GAMSCOPES: {}, + GM_GAMSCOPES_CREATED: False, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID: None, @@ -517,6 +519,7 @@ def SetGlobalVariables(): GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None + GM_Globals[GM_GAMSCOPES_CREATED] = False while True: json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) if not json_string: @@ -8791,6 +8794,7 @@ def doRequestOAuth(): print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) + GM_Globals[GM_GAMSCOPES_CREATED] = True break elif selection >= 0 and selection < len(all_apis.keys()): api = all_apis.keys()[selection] @@ -9216,7 +9220,8 @@ try: sys.exit(0) elif sys.argv[1].lower() in [u'oauth', u'oauth2']: if sys.argv[2].lower() in [u'request', u'create']: - doRequestOAuth() + if not GM_Globals[GM_GAMSCOPES_CREATED]: + doRequestOAuth() elif sys.argv[2].lower() == u'info': OAuthInfo() else: From 571a9dcb3e839359173ce212b0665aa40282ee81 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 23:37:15 -0800 Subject: [PATCH 044/100] Handle no scopes for API This should probably call doRequestOAuth rather than bailing out --- src/gam.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 18c37add..f5e6d993 100755 --- a/src/gam.py +++ b/src/gam.py @@ -255,21 +255,22 @@ GC_VAR_INFO = { GC_USER_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 500)}, } -MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' +MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' +MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV headers of "{1}".' MESSAGE_HIT_CONTROL_C_TO_UPDATE = u'\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.' MESSAGE_NO_DISCOVERY_INFORMATION = u'No online discovery doc and {0} does not exist locally' MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' +MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for API {0}-{1}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' +MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service Account.' -MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' -MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' def convertUTF8(data): import collections @@ -519,7 +520,7 @@ def SetGlobalVariables(): GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None - GM_Globals[GM_GAMSCOPES_CREATED] = False + GM_Globals[GM_GAMSCOPES_CREATED] = False while True: json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) if not json_string: @@ -803,7 +804,10 @@ def getAPIVer(api): def getServiceAPIScope(api, version=None): if not version: version = getAPIVer(api) - return GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) + scopes = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) + if scopes: + return scopes + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -8706,7 +8710,7 @@ def OAuthInfo(): for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): print u' API: {0}'.format(api) for scope in GM_Globals[GM_GAMSCOPES][api]: - print u' {0}'.format(scope) + print u' {0}'.format(scope) UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], From 1ec164a25a3a7d2d724962057251a84bc33ab199 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 06:50:16 -0800 Subject: [PATCH 045/100] Bring gam oauth delete back, give creation message --- src/gam.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/gam.py b/src/gam.py index f5e6d993..6c83b4c3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8712,6 +8712,17 @@ def OAuthInfo(): for scope in GM_Globals[GM_GAMSCOPES][api]: print u' {0}'.format(scope) +def doDeleteOAuth(): + sys.stdout.write(u'Scopes file: {0}, will be Deleted in 3...'.format(GC_Values[GC_GAMSCOPES_JSON])) + time.sleep(1) + sys.stdout.write(u'2...') + time.sleep(1) + sys.stdout.write(u'1...') + time.sleep(1) + sys.stdout.write(u'boom!\n') + os.remove(GC_Values[GC_GAMSCOPES_JSON]) + sys.stdout.write(u'Scopes file: {0}, Deleted\n'.format(GC_Values[GC_GAMSCOPES_JSON])) + UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], u'drive-v2': [u'https://www.googleapis.com/auth/drive'], @@ -8798,6 +8809,7 @@ def doRequestOAuth(): print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) + print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) GM_Globals[GM_GAMSCOPES_CREATED] = True break elif selection >= 0 and selection < len(all_apis.keys()): @@ -9228,6 +9240,8 @@ try: doRequestOAuth() elif sys.argv[2].lower() == u'info': OAuthInfo() + elif sys.argv[2].lower() in [u'delete', u'revoke']: + doDeleteOAuth() else: print u'ERROR: %s is not a valid argument for "gam oauth"' % sys.argv[2] sys.exit(2) From bcb17cd0a580a3481ab81309e0f2d392fe2eadee Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 07:40:22 -0800 Subject: [PATCH 046/100] in doRequestOauth, initialize API use_scopes from gamscopes.json if available --- src/gam.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 6c83b4c3..1a09cbc6 100755 --- a/src/gam.py +++ b/src/gam.py @@ -527,9 +527,9 @@ def SetGlobalVariables(): doRequestOAuth() continue json_data = json.loads(json_string) - if not isinstance(json_data, dict) or (u'scopes' not in json_data) or (not isinstance(json_data[u'scopes'], dict)): + if not isinstance(json_data, dict): systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) - GM_Globals[GM_GAMSCOPES] = json_data[u'scopes'] + GM_Globals[GM_GAMSCOPES] = json_data break return True @@ -804,7 +804,8 @@ def getAPIVer(api): def getServiceAPIScope(api, version=None): if not version: version = getAPIVer(api) - scopes = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) + apiData = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), {}) + scopes = apiData.get(u'use_scopes', []) if scopes: return scopes systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) @@ -8709,17 +8710,21 @@ def OAuthInfo(): print u'API Scopes' for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): print u' API: {0}'.format(api) - for scope in GM_Globals[GM_GAMSCOPES][api]: + for scope in GM_Globals[GM_GAMSCOPES][api][u'use_scopes']: print u' {0}'.format(scope) def doDeleteOAuth(): sys.stdout.write(u'Scopes file: {0}, will be Deleted in 3...'.format(GC_Values[GC_GAMSCOPES_JSON])) + sys.stdout.flush() time.sleep(1) sys.stdout.write(u'2...') + sys.stdout.flush() time.sleep(1) sys.stdout.write(u'1...') + sys.stdout.flush() time.sleep(1) sys.stdout.write(u'boom!\n') + sys.stdout.flush() os.remove(GC_Values[GC_GAMSCOPES_JSON]) sys.stdout.write(u'Scopes file: {0}, Deleted\n'.format(GC_Values[GC_GAMSCOPES_JSON])) @@ -8775,6 +8780,9 @@ def doRequestOAuth(): all_apis[api_name][u'index'] = i i += 1 all_apis = select_default_scopes(all_apis) + if GM_Globals[GM_GAMSCOPES]: + for api in GM_Globals[GM_GAMSCOPES]: + all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES][api][u'use_scopes'] os.system([u'clear', u'cls'][os.name == u'nt']) while True: print u'Select the APIs to use with GAM.' @@ -8800,10 +8808,10 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: selected_scopes = [u'email'] - json_scopes = {u'scopes': {}} + json_scopes = {} for api in all_apis.keys(): selected_scopes += all_apis[api][u'use_scopes'] - json_scopes[u'scopes'][api] = all_apis[api][u'use_scopes'] + json_scopes[api] = {u'use_scopes': all_apis[api][u'use_scopes']} selected_scopes = list(set(selected_scopes)) # unique only if len(selected_scopes) < 2: print u'YOU MUST SELECT AT LEAST ONE SCOPE' From 7d69c8e3bd3dda71037d265253611b6fc8deef8f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 12:13:02 -0800 Subject: [PATCH 047/100] Dynamic scopes cleanup I deleted the 'email' scope because it caused everything to fail as it's not (and never was) authorized. What was/is it for? --- src/gam.py | 111 ++++++++++++++++++++++++++++------------------------- 1 file changed, 58 insertions(+), 53 deletions(-) diff --git a/src/gam.py b/src/gam.py index 1a09cbc6..b5f01735 100755 --- a/src/gam.py +++ b/src/gam.py @@ -91,7 +91,10 @@ GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json -GM_GAMSCOPES = u'scop' +GM_GAMSCOPES_BY_API = u'scop' +GM_GAMSCOPES_LIST = u'scof' +# Current API scope +GM_CURRENT_API_SCOPES = u'scoc' # gamscopes.json created GM_GAMSCOPES_CREATED = u'gscr' # Values retrieved from oauth2service.json @@ -108,8 +111,6 @@ GM_MAP_ROLE_ID_TO_NAME = u'ri2n' GM_MAP_ROLE_NAME_TO_ID = u'rn2i' # Dictionary mapping User ID to Name GM_MAP_USER_ID_TO_NAME = u'ui2n' -# Current API scope -GM_API_SCOPE = u'csco' # GM_Globals = { GM_SYSEXITRC: 0, @@ -118,7 +119,9 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, - GM_GAMSCOPES: {}, + GM_GAMSCOPES_BY_API: {}, + GM_GAMSCOPES_LIST: [], + GM_CURRENT_API_SCOPES: None, GM_GAMSCOPES_CREATED: False, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, @@ -128,7 +131,6 @@ GM_Globals = { GM_MAP_ROLE_ID_TO_NAME: None, GM_MAP_ROLE_NAME_TO_ID: None, GM_MAP_USER_ID_TO_NAME: None, - GM_API_SCOPE: None, } # # Global variables defined by environment variables/signal files @@ -521,16 +523,11 @@ def SetGlobalVariables(): if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None GM_Globals[GM_GAMSCOPES_CREATED] = False - while True: - json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) - if not json_string: - doRequestOAuth() - continue - json_data = json.loads(json_string) - if not isinstance(json_data, dict): - systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) - GM_Globals[GM_GAMSCOPES] = json_data - break + json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) + if not json_string: + doRequestOAuth() + elif not validateSetGAMScopes(json.loads(json_string)): + systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) return True def doGAMCheckForUpdates(forceCheck=False): @@ -583,11 +580,10 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) -def tryOAuth(gdataObject, scope, soft_errors=False): - scope.append(u'email') +def tryOAuth(gdataObject, soft_errors=False): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) + scope=GM_Globals[GM_GAMSCOPES_LIST], user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) try: @@ -596,7 +592,7 @@ def tryOAuth(gdataObject, scope, soft_errors=False): if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(scope))) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: return False @@ -613,7 +609,7 @@ def checkGDataError(e, service): # First check for errors that need special handling if e[0].get(u'reason', u'') in [u'Token invalid - Invalid token: Stateless token expired', u'Token invalid - Invalid token: Token not found']: keep_domain = service.domain - tryOAuth(service, GM_Globals[GM_API_SCOPE]) + tryOAuth(service) service.domain = keep_domain return False if e[0][u'body'].startswith(u'Required field must not be blank:') or e[0][u'body'].startswith(u'These characters are not allowed:'): @@ -801,14 +797,13 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -def getServiceAPIScope(api, version=None): +def setCurrentAPIScopes(api, version=None): if not version: version = getAPIVer(api) - apiData = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), {}) - scopes = apiData.get(u'use_scopes', []) - if scopes: - return scopes - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) + apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) + GM_Globals[GM_CURRENT_API_SCOPES] = apiData.get(u'use_scopes', []) + if not GM_Globals[GM_CURRENT_API_SCOPES]: + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -848,10 +843,10 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' - GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api, version) + setCurrentAPIScopes(api, version) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=GM_Globals[GM_API_SCOPE], user_agent=GAM_INFO, sub=sub) + scope=GM_Globals[GM_GAMSCOPES_LIST], user_agent=GAM_INFO, sub=sub) http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR])) try: @@ -864,18 +859,18 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_API_SCOPE]))) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: return False sys.exit(4) -def commonAppsObjInit(appsObj, scope): +def commonAppsObjInit(appsObj, api): getOAuth2ServiceDetails() - GM_Globals[GM_API_SCOPE] = scope - if not tryOAuth(appsObj, GM_Globals[GM_API_SCOPE]): + setCurrentAPIScopes(api) + if not tryOAuth(appsObj): doRequestOAuth() - tryOAuth(appsObj, GM_Globals[GM_API_SCOPE]) + tryOAuth(appsObj) #Identify GAM to Google's Servers appsObj.source = GAM_INFO #Show debugging output if debug.gam exists @@ -885,18 +880,15 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service - return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), - getServiceAPIScope(u'admin-settings')) + return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), u'admin-settings') def getAuditObject(): import gdata.apps.audit.service - return commonAppsObjInit(gdata.apps.audit.service.AuditService(), - getServiceAPIScope(u'email-audit')) + return commonAppsObjInit(gdata.apps.audit.service.AuditService(), u'email-audit') def getEmailSettingsObject(): import gdata.apps.emailsettings.service - return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - getServiceAPIScope(u'email-settings')) + return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), u'email-settings') def geturl(url, dst): import urllib2 @@ -8706,11 +8698,24 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa full_users = new_full_users return full_users +def validateSetGAMScopes(json_data): + GM_Globals[GM_GAMSCOPES_BY_API] = {} + GM_Globals[GM_GAMSCOPES_LIST] = [] + if not isinstance(json_data, dict): + return False + for api, value in json_data.items(): + if (not isinstance(value, dict)) or (u'use_scopes' not in value) or (not isinstance(value[u'use_scopes'], list)): + return False + GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': value[u'use_scopes']} + GM_Globals[GM_GAMSCOPES_LIST] += value[u'use_scopes'] + GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only + return len(GM_Globals[GM_GAMSCOPES_LIST]) > 0 + def OAuthInfo(): print u'API Scopes' - for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): + for api in sorted(GM_Globals[GM_GAMSCOPES_BY_API].keys()): print u' API: {0}'.format(api) - for scope in GM_Globals[GM_GAMSCOPES][api][u'use_scopes']: + for scope in GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes']: print u' {0}'.format(scope) def doDeleteOAuth(): @@ -8780,9 +8785,9 @@ def doRequestOAuth(): all_apis[api_name][u'index'] = i i += 1 all_apis = select_default_scopes(all_apis) - if GM_Globals[GM_GAMSCOPES]: - for api in GM_Globals[GM_GAMSCOPES]: - all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES][api][u'use_scopes'] + if GM_Globals[GM_GAMSCOPES_BY_API]: + for api in GM_Globals[GM_GAMSCOPES_BY_API]: + all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes'] os.system([u'clear', u'cls'][os.name == u'nt']) while True: print u'Select the APIs to use with GAM.' @@ -8807,16 +8812,16 @@ def doRequestOAuth(): for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: - selected_scopes = [u'email'] - json_scopes = {} + GM_Globals[GM_GAMSCOPES_BY_API] = {} + GM_Globals[GM_GAMSCOPES_LIST] = [] for api in all_apis.keys(): - selected_scopes += all_apis[api][u'use_scopes'] - json_scopes[api] = {u'use_scopes': all_apis[api][u'use_scopes']} - selected_scopes = list(set(selected_scopes)) # unique only - if len(selected_scopes) < 2: + GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': all_apis[api][u'use_scopes']} + GM_Globals[GM_GAMSCOPES_LIST] += all_apis[api][u'use_scopes'] + GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only + if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) GM_Globals[GM_GAMSCOPES_CREATED] = True break @@ -8871,9 +8876,9 @@ def doRequestOAuth(): break os.system([u'clear', u'cls'][os.name == u'nt']) os.system([u'clear', u'cls'][os.name == u'nt']) - print u'Please authorize your service account client id for the %s scopes:' % (len(selected_scopes)) + print u'Please authorize your service account client id for the %s scopes:' % (len(GM_Globals[GM_GAMSCOPES_LIST])) print - print u','.join(selected_scopes) + print u','.join(GM_Globals[GM_GAMSCOPES_LIST]) def batch_worker(): while True: From c0ee6740601dc82a3c34d2fced3bc6f6658217f2 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 14:04:01 -0800 Subject: [PATCH 048/100] Full scopes list causes internal error, back (for now) to API specific list --- src/gam.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index b5f01735..9d386584 100755 --- a/src/gam.py +++ b/src/gam.py @@ -121,7 +121,7 @@ GM_Globals = { GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, GM_GAMSCOPES_BY_API: {}, GM_GAMSCOPES_LIST: [], - GM_CURRENT_API_SCOPES: None, + GM_CURRENT_API_SCOPES: [], GM_GAMSCOPES_CREATED: False, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, @@ -269,6 +269,7 @@ MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for API {0}-{1}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' +MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT = u'Please authorize your service account client id for the {} scopes:' MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' @@ -527,7 +528,7 @@ def SetGlobalVariables(): if not json_string: doRequestOAuth() elif not validateSetGAMScopes(json.loads(json_string)): - systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) + systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) return True def doGAMCheckForUpdates(forceCheck=False): @@ -583,7 +584,7 @@ def doGAMVersion(): def tryOAuth(gdataObject, soft_errors=False): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=GM_Globals[GM_GAMSCOPES_LIST], user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) + scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) try: @@ -846,7 +847,7 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): setCurrentAPIScopes(api, version) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=GM_Globals[GM_GAMSCOPES_LIST], user_agent=GAM_INFO, sub=sub) + scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=sub) http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR])) try: @@ -8788,7 +8789,7 @@ def doRequestOAuth(): if GM_Globals[GM_GAMSCOPES_BY_API]: for api in GM_Globals[GM_GAMSCOPES_BY_API]: all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes'] - os.system([u'clear', u'cls'][os.name == u'nt']) + os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) while True: print u'Select the APIs to use with GAM.' print @@ -8833,7 +8834,7 @@ def doRequestOAuth(): else: all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'] else: - os.system([u'clear', u'cls'][os.name == u'nt']) + os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) while True: print x = 0 @@ -8874,9 +8875,9 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == x+3: break - os.system([u'clear', u'cls'][os.name == u'nt']) - os.system([u'clear', u'cls'][os.name == u'nt']) - print u'Please authorize your service account client id for the %s scopes:' % (len(GM_Globals[GM_GAMSCOPES_LIST])) + os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) + os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) + print MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST])) print print u','.join(GM_Globals[GM_GAMSCOPES_LIST]) From acb21cb926d77482b8ef73045e30f453cad23787 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 14:39:10 -0800 Subject: [PATCH 049/100] Clean signature printing --- src/gam.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 9d386584..424588ec 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4812,12 +4812,12 @@ def getSignature(users): user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - signature = callGData(emailsettings, u'GetSignature', soft_errors=True, username=user) - try: - sys.stderr.write(u"User %s signature:\n " % (user+u'@'+emailsettings.domain)) - print convertUTF8(u" %s" % signature[u'signature']) - except TypeError: - pass + result = callGData(emailsettings, u'GetSignature', soft_errors=True, username=user) + signature = result.get(u'signature', u'None') if result else u'None' + if not signature: + signature = u'None' + sys.stdout.write(u"User %s signature:\n " % (user+u'@'+emailsettings.domain)) + print convertUTF8(u" %s" % signature) def doWebClips(users): if sys.argv[4].lower() in true_values: From 75483185d694f2056212ce44d12f3cb504638137 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 07:09:15 -0800 Subject: [PATCH 050/100] Clean up Clean up handling of missing json files Clean up processing environment variables, signal files --- src/gam.py | 153 ++++++++++++++++++++++++++--------------------------- 1 file changed, 75 insertions(+), 78 deletions(-) diff --git a/src/gam.py b/src/gam.py index 424588ec..c2aa6727 100755 --- a/src/gam.py +++ b/src/gam.py @@ -95,8 +95,6 @@ GM_GAMSCOPES_BY_API = u'scop' GM_GAMSCOPES_LIST = u'scof' # Current API scope GM_CURRENT_API_SCOPES = u'scoc' -# gamscopes.json created -GM_GAMSCOPES_CREATED = u'gscr' # Values retrieved from oauth2service.json GM_OAUTH2SERVICE_KEY = u'oauk' GM_OAUTH2SERVICE_ACCOUNT_EMAIL = u'oaae' @@ -122,7 +120,6 @@ GM_Globals = { GM_GAMSCOPES_BY_API: {}, GM_GAMSCOPES_LIST: [], GM_CURRENT_API_SCOPES: [], - GM_GAMSCOPES_CREATED: False, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID: None, @@ -174,16 +171,8 @@ GC_NO_VERIFY_SSL = u'no_verify_ssl' GC_NUM_THREADS = u'num_threads' # Path to oauth2service.json GC_OAUTH2SERVICE_JSON = u'oauth2service_json' -# Default section to use for processing -GC_SECTION = u'section' -# Add (n/m) to end of messages if number of items to be processed exceeds this number -GC_SHOW_COUNTS_MIN = u'show_counts_min' -# Enable/disable "Getting ... " messages -GC_SHOW_GETTINGS = u'show_gettings' # GAM config directory containing admin-settings-v1.json, cloudprint-v2.json GC_SITE_DIR = u'site_dir' -# When adding Users to Groups/Org Units, how many should be processed in each batch -GC_USER_BATCH_SIZE = u'user_batch_size' # When retrieving lists of Users from API, how many should be retrieved in each chunk GC_USER_MAX_RESULTS = u'user_max_results' @@ -207,11 +196,7 @@ GC_Defaults = { GC_NO_VERIFY_SSL: FALSE, GC_NUM_THREADS: 5, GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON, - GC_SECTION: u'', - GC_SHOW_COUNTS_MIN: 1, - GC_SHOW_GETTINGS: TRUE, GC_SITE_DIR: u'', - GC_USER_BATCH_SIZE: 50, GC_USER_MAX_RESULTS: 500, } @@ -227,36 +212,34 @@ GC_TYPE_LANGUAGE = u'lang' GC_TYPE_STRING = u'stri' GC_VAR_TYPE_KEY = u'type' +GC_VAR_ENVVAR_KEY = u'enva' GC_VAR_LIMITS_KEY = u'lmit' GC_VAR_INFO = { - GC_ACTIVITY_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 500)}, - GC_ADMIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, - GC_AUTO_BATCH_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, - GC_CACHE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, - GC_CHARSET: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, - GC_CONFIG_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, - GC_CUSTOMER_ID: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, - GC_DEBUG_LEVEL: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, - GC_DEVICE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, - GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, - GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, - GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, - GC_GAMSCOPES_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, - GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, - GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, - GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, - GC_NO_VERIFY_SSL: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, - GC_NUM_THREADS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, None)}, - GC_OAUTH2SERVICE_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, - GC_SECTION: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, - GC_SHOW_COUNTS_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (0, None)}, - GC_SHOW_GETTINGS: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, - GC_SITE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, - GC_USER_BATCH_SIZE: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, - GC_USER_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 500)}, + GC_ACTIVITY_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_ACTIVITY_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 500)}, + GC_ADMIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GAM_ADMIN'}, + GC_AUTO_BATCH_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_AUTOBATCH', GC_VAR_LIMITS_KEY: (0, None)}, + GC_CACHE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMCACHEDIR'}, + GC_CHARSET: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GAM_CHARSET'}, + GC_CONFIG_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMUSERCONFIGDIR'}, + GC_CUSTOMER_ID: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'CUSTOMER_ID'}, + GC_DEBUG_LEVEL: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'debug.gam', GC_VAR_LIMITS_KEY: (0, None)}, + GC_DEVICE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_DEVICE_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 1000)}, + GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GA_DOMAIN'}, + GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMDRIVEDIR'}, + GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_DRIVE_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 1000)}, + GC_GAMSCOPES_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE, GC_VAR_ENVVAR_KEY: u'GAMSCOPESFILE'}, + GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'nobrowser.txt'}, + GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'nocache.txt'}, + GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'noupdatecheck.txt'}, + GC_NO_VERIFY_SSL: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'noverifyssl.txt'}, + GC_NUM_THREADS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_THREADS', GC_VAR_LIMITS_KEY: (1, None)}, + GC_OAUTH2SERVICE_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE, GC_VAR_ENVVAR_KEY: u'OAUTHSERVICEFILE'}, + GC_SITE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMSITECONFIGDIR'}, + GC_USER_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_USER_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 500)}, } + MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' @@ -441,8 +424,8 @@ def writeFile(filename, data, mode=u'wb', continueOnError=False, displayError=Tr # def SetGlobalVariables(): - def _getOldEnvVar(itemName, envVar): - value = os.environ.get(envVar, GC_Defaults[itemName]) + def _getOldEnvVar(itemName): + value = os.environ.get(GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY], GC_Defaults[itemName]) if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_INTEGER: try: number = int(value) @@ -456,8 +439,8 @@ def SetGlobalVariables(): value = number GC_Defaults[itemName] = value - def _getOldSignalFile(itemName, fileName, trueValue=True, falseValue=False): - GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], fileName)) else falseValue + def _getOldSignalFile(itemName, trueValue=True, falseValue=False): + GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY])) else falseValue def _getCfgDirectory(itemName): return GC_Defaults[itemName] @@ -468,34 +451,48 @@ def SetGlobalVariables(): value = os.path.expanduser(os.path.join(GC_Values[GC_CONFIG_DIR], value)) return value + def _chkCfgDirectories(): + for itemName in GC_VAR_INFO: + if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_DIRECTORY: + dirPath = GC_Values[itemName] + if not os.path.isdir(dirPath): + sys.stderr.write(u'{0}{1}={2}, Invalid Path\n'.format(WARNING_PREFIX, GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY], dirPath)) + + def _chkCfgFiles(): + for itemName in GC_VAR_INFO: + if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_FILE: + fileName = GC_Values[itemName] + if not os.path.isfile(fileName): + sys.stderr.write(u'{0}{1}={2}, Not Found\n'.format(WARNING_PREFIX, GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY], fileName)) + GC_Defaults[GC_CONFIG_DIR] = GM_Globals[GM_GAM_PATH] GC_Defaults[GC_CACHE_DIR] = os.path.join(GM_Globals[GM_GAM_PATH], u'gamcache') GC_Defaults[GC_DRIVE_DIR] = GM_Globals[GM_GAM_PATH] GC_Defaults[GC_SITE_DIR] = GM_Globals[GM_GAM_PATH] - _getOldEnvVar(GC_CONFIG_DIR, u'GAMUSERCONFIGDIR') - _getOldEnvVar(GC_SITE_DIR, u'GAMSITECONFIGDIR') - _getOldEnvVar(GC_CACHE_DIR, u'GAMCACHEDIR') - _getOldEnvVar(GC_DRIVE_DIR, u'GAMDRIVEDIR') - _getOldEnvVar(GC_OAUTH2SERVICE_JSON, u'OAUTHSERVICEFILE') + _getOldEnvVar(GC_CONFIG_DIR) + _getOldEnvVar(GC_SITE_DIR) + _getOldEnvVar(GC_CACHE_DIR) + _getOldEnvVar(GC_DRIVE_DIR) + _getOldEnvVar(GC_OAUTH2SERVICE_JSON) if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' - _getOldEnvVar(GC_GAMSCOPES_JSON, u'GAMSCOPESFILE') - _getOldEnvVar(GC_DOMAIN, u'GA_DOMAIN') - _getOldEnvVar(GC_ADMIN, u'GAM_ADMIN') - _getOldEnvVar(GC_CUSTOMER_ID, u'CUSTOMER_ID') - _getOldEnvVar(GC_CHARSET, u'GAM_CHARSET') - _getOldEnvVar(GC_NUM_THREADS, u'GAM_THREADS') - _getOldEnvVar(GC_AUTO_BATCH_MIN, u'GAM_AUTOBATCH') - _getOldEnvVar(GC_ACTIVITY_MAX_RESULTS, u'GAM_ACTIVITY_MAX_RESULTS') - _getOldEnvVar(GC_DEVICE_MAX_RESULTS, u'GAM_DEVICE_MAX_RESULTS') - _getOldEnvVar(GC_DRIVE_MAX_RESULTS, u'GAM_DRIVE_MAX_RESULTS') - _getOldEnvVar(GC_USER_MAX_RESULTS, u'GAM_USER_MAX_RESULTS') - _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', trueValue=4, falseValue=0) - _getOldSignalFile(GC_NO_VERIFY_SSL, u'noverifyssl.txt') - _getOldSignalFile(GC_NO_BROWSER, u'nobrowser.txt') - _getOldSignalFile(GC_NO_CACHE, u'nocache.txt') - _getOldSignalFile(GC_NO_UPDATE_CHECK, u'noupdatecheck.txt') + _getOldEnvVar(GC_GAMSCOPES_JSON) + _getOldEnvVar(GC_DOMAIN) + _getOldEnvVar(GC_ADMIN) + _getOldEnvVar(GC_CUSTOMER_ID) + _getOldEnvVar(GC_CHARSET) + _getOldEnvVar(GC_NUM_THREADS) + _getOldEnvVar(GC_AUTO_BATCH_MIN) + _getOldEnvVar(GC_ACTIVITY_MAX_RESULTS) + _getOldEnvVar(GC_DEVICE_MAX_RESULTS) + _getOldEnvVar(GC_DRIVE_MAX_RESULTS) + _getOldEnvVar(GC_USER_MAX_RESULTS) + _getOldSignalFile(GC_DEBUG_LEVEL, trueValue=4, falseValue=0) + _getOldSignalFile(GC_NO_VERIFY_SSL) + _getOldSignalFile(GC_NO_BROWSER) + _getOldSignalFile(GC_NO_CACHE) + _getOldSignalFile(GC_NO_UPDATE_CHECK) # Assign directories first for itemName in GC_VAR_INFO: if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_DIRECTORY: @@ -510,9 +507,6 @@ def SetGlobalVariables(): if not GC_Values[GC_NO_UPDATE_CHECK]: doGAMCheckForUpdates() # Globals derived from config file values - GM_Globals[GM_OAUTH2SERVICE_KEY] = None - GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = None - GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = None GM_Globals[GM_EXTRA_ARGS_DICT] = {u'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} httplib2.debuglevel = GC_Values[GC_DEBUG_LEVEL] if os.path.isfile(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)): @@ -521,14 +515,16 @@ def SetGlobalVariables(): ea_config.optionxform = str ea_config.read(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)) GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) + GM_Globals[GM_OAUTH2SERVICE_KEY] = None + GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = None + GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = None if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None - GM_Globals[GM_GAMSCOPES_CREATED] = False - json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) - if not json_string: - doRequestOAuth() - elif not validateSetGAMScopes(json.loads(json_string)): + json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) + if json_string and not validateSetGAMScopes(json.loads(json_string)): systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) + _chkCfgDirectories() + _chkCfgFiles() return True def doGAMCheckForUpdates(forceCheck=False): @@ -8731,8 +8727,11 @@ def doDeleteOAuth(): time.sleep(1) sys.stdout.write(u'boom!\n') sys.stdout.flush() - os.remove(GC_Values[GC_GAMSCOPES_JSON]) - sys.stdout.write(u'Scopes file: {0}, Deleted\n'.format(GC_Values[GC_GAMSCOPES_JSON])) + try: + os.remove(GC_Values[GC_GAMSCOPES_JSON]) + sys.stdout.write(u'Scopes file: {0}, Deleted\n'.format(GC_Values[GC_GAMSCOPES_JSON])) + except OSError as e: + sys.stderr.write(u'{0}{1}\n'.format(WARNING_PREFIX, e)) UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], @@ -8824,7 +8823,6 @@ def doRequestOAuth(): continue writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) - GM_Globals[GM_GAMSCOPES_CREATED] = True break elif selection >= 0 and selection < len(all_apis.keys()): api = all_apis.keys()[selection] @@ -9250,8 +9248,7 @@ try: sys.exit(0) elif sys.argv[1].lower() in [u'oauth', u'oauth2']: if sys.argv[2].lower() in [u'request', u'create']: - if not GM_Globals[GM_GAMSCOPES_CREATED]: - doRequestOAuth() + doRequestOAuth() elif sys.argv[2].lower() == u'info': OAuthInfo() elif sys.argv[2].lower() in [u'delete', u'revoke']: From 2390c4284e4f41a3cc47d052d19f555b8d422a38 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 07:44:48 -0800 Subject: [PATCH 051/100] Clearing cache_dir has to come after directory check --- src/gam.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index c2aa6727..729fa022 100755 --- a/src/gam.py +++ b/src/gam.py @@ -239,7 +239,6 @@ GC_VAR_INFO = { GC_USER_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_USER_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 500)}, } - MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' @@ -518,13 +517,13 @@ def SetGlobalVariables(): GM_Globals[GM_OAUTH2SERVICE_KEY] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = None - if GC_Values[GC_NO_CACHE]: - GC_Values[GC_CACHE_DIR] = None json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) if json_string and not validateSetGAMScopes(json.loads(json_string)): systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) _chkCfgDirectories() _chkCfgFiles() + if GC_Values[GC_NO_CACHE]: + GC_Values[GC_CACHE_DIR] = None return True def doGAMCheckForUpdates(forceCheck=False): From 008a65329e7c8f7ae341b60f7c997eb8c8bead77 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 09:10:22 -0800 Subject: [PATCH 052/100] gam oauth info now verifies scopes --- src/gam.py | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 729fa022..eb74414f 100755 --- a/src/gam.py +++ b/src/gam.py @@ -240,7 +240,8 @@ GC_VAR_INFO = { } MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' -MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' +MESSAGE_CLIENT_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' +MESSAGE_CLIENT_API_ACCESS_DENIED = u'API access denied. Please make sure the service account ID: {0} is authorized for the API Scope(s): {1}' MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' @@ -588,7 +589,8 @@ def tryOAuth(gdataObject, soft_errors=False): if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES])))) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_CONFIG) sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: return False @@ -855,7 +857,8 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES])))) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_CONFIG) sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: return False @@ -8708,11 +8711,42 @@ def validateSetGAMScopes(json_data): return len(GM_Globals[GM_GAMSCOPES_LIST]) > 0 def OAuthInfo(): + configRequired = False + getOAuth2ServiceDetails() print u'API Scopes' - for api in sorted(GM_Globals[GM_GAMSCOPES_BY_API].keys()): + for api in sorted(API_VER_MAPPING.keys()): print u' API: {0}'.format(api) - for scope in GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes']: - print u' {0}'.format(scope) + version = getAPIVer(api) + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) + scopes = apiData.get(u'use_scopes', []) + if scopes: + for scope in scopes: + print u' {0}'.format(scope) + credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], + GM_Globals[GM_OAUTH2SERVICE_KEY], + scope=scopes, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) + http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR])) + try: + googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + except googleapiclient.errors.UnknownApiNameOrVersion: + getServiceFromDiscoveryDocument(api, version, http) + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, e) + except oauth2client.client.AccessTokenRefreshError, e: + if e.message in [u'access_denied', + u'unauthorized_client: Unauthorized client or scope in request.', + u'access_denied: Requested client not authorized.']: + print u' {0}{1}'.format(WARNING_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u'listed above')) + configRequired = True + else: + print u' {0}{1}'.format(ERROR_PREFIX, e) + else: + print u' Not Requested' + if configRequired: + print MESSAGE_CLIENT_API_ACCESS_CONFIG def doDeleteOAuth(): sys.stdout.write(u'Scopes file: {0}, will be Deleted in 3...'.format(GC_Values[GC_GAMSCOPES_JSON])) From a7a3f2eef66dd38309366ba6437ca0164facf47a Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 09:52:10 -0800 Subject: [PATCH 053/100] Clean up input handling in doRequestOauth --- src/gam.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/gam.py b/src/gam.py index eb74414f..6c07b125 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8799,6 +8799,14 @@ def select_default_scopes(apis): apis[api_name][u'use_scopes'] += scopes return apis +def getSelection(): + while True: + selection = raw_input(u'Your selection: ') + if selection: + if selection.isdigit(): + return int(selection) + print u'ERROR: please enter numbers only' + def doRequestOAuth(): apis = API_VER_MAPPING.keys() all_apis = {} @@ -8821,8 +8829,8 @@ def doRequestOAuth(): if GM_Globals[GM_GAMSCOPES_BY_API]: for api in GM_Globals[GM_GAMSCOPES_BY_API]: all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes'] - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) while True: + os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' print for api in all_apis.values(): @@ -8838,8 +8846,8 @@ def doRequestOAuth(): print u' %s) Unselect all APIs' % (i+2) print u' %s) Continue' % (i+3) print - selection = int(raw_input(u'Your selection: ')) - if int(selection) == i+1: # defaults + selection = getSelection() + if selection == i+1: # defaults all_apis = select_default_scopes(all_apis) elif selection == i+2: # unselect all for api in all_apis.keys(): @@ -8865,8 +8873,8 @@ def doRequestOAuth(): else: all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'] else: - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) while True: + os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print x = 0 for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): @@ -8881,12 +8889,7 @@ def doRequestOAuth(): print u' %s) Unselect all scopes' % (x+2) print u' %s) Back to all APIs' % (x+3) print - selection = raw_input(u'Your selection: ') - try: - selection = int(selection) - except: - print u'ERROR: please enter numbers only' - continue + selection = getSelection() num_scopes = len(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()) if selection >= 0 and selection < num_scopes: if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in all_apis[api][u'use_scopes']: @@ -8906,8 +8909,6 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == x+3: break - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST])) print print u','.join(GM_Globals[GM_GAMSCOPES_LIST]) @@ -9282,7 +9283,7 @@ try: elif sys.argv[1].lower() in [u'oauth', u'oauth2']: if sys.argv[2].lower() in [u'request', u'create']: doRequestOAuth() - elif sys.argv[2].lower() == u'info': + elif sys.argv[2].lower() in [u'info', u'verify']: OAuthInfo() elif sys.argv[2].lower() in [u'delete', u'revoke']: doDeleteOAuth() From 90fd503838591fadf11a9363ec6d3a549b58bfb0 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 11:59:37 -0800 Subject: [PATCH 054/100] If GA_DOMAIN not set, derive domain from GAM_ADMIN if defined If neither set, we have to decide what to do. --- src/gam.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gam.py b/src/gam.py index 6c07b125..238545de 100755 --- a/src/gam.py +++ b/src/gam.py @@ -506,6 +506,10 @@ def SetGlobalVariables(): GM_Globals[GM_LAST_UPDATE_CHECK_TXT] = os.path.join(GC_Values[GC_CONFIG_DIR], FN_LAST_UPDATE_CHECK_TXT) if not GC_Values[GC_NO_UPDATE_CHECK]: doGAMCheckForUpdates() + if (not GC_Values[GC_DOMAIN]) and GC_Values[GC_ADMIN]: + loc = GC_Values[GC_ADMIN].find(u'@') + if loc > 0: + GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][loc+1:] # Globals derived from config file values GM_Globals[GM_EXTRA_ARGS_DICT] = {u'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} httplib2.debuglevel = GC_Values[GC_DEBUG_LEVEL] From 9f7596868489754cb476d49826801ac8d43902ad Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 12:38:45 -0800 Subject: [PATCH 055/100] More clean up of doRequestOAuth --- src/gam.py | 57 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/src/gam.py b/src/gam.py index 238545de..2265e20c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8803,13 +8803,17 @@ def select_default_scopes(apis): apis[api_name][u'use_scopes'] += scopes return apis -def getSelection(): +def getSelection(limit): while True: selection = raw_input(u'Your selection: ') if selection: if selection.isdigit(): - return int(selection) - print u'ERROR: please enter numbers only' + selection = int(selection) + if (selection >= 0) and (selection <= limit): + return selection + print u'ERROR: enter number in range 0-{0}'.format(limit) + else: + print u'ERROR: please enter numbers only' def doRequestOAuth(): apis = API_VER_MAPPING.keys() @@ -8844,19 +8848,20 @@ def doRequestOAuth(): select_value = u'*' else: select_value = u' ' - print u'[%s] %s) %s (%s/%s scopes)' % (select_value, api[u'index'], api[u'title'], num_scopes_selected, num_scopes_total) + print u'[%s] %2d) %s (%d/%d scopes)' % (select_value, api[u'index'], api[u'title'], num_scopes_selected, num_scopes_total) print - print u' %s) Select defaults for all APIs (allow all GAM commands)' % (i+1) - print u' %s) Unselect all APIs' % (i+2) - print u' %s) Continue' % (i+3) + print u' %2d) Select defaults for all APIs (allow all GAM commands)' % (i) + print u' %2d) Unselect all APIs' % (i+1) + print u' %2d) Continue' % (i+2) + print u' %2d) Cancel' % (i+3) print - selection = getSelection() - if selection == i+1: # defaults + selection = getSelection(i+3) + if selection == i: # defaults all_apis = select_default_scopes(all_apis) - elif selection == i+2: # unselect all + elif selection == i+1: # unselect all for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] - elif selection == i+3: + elif selection == i+2: # continue GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] for api in all_apis.keys(): @@ -8869,7 +8874,9 @@ def doRequestOAuth(): writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break - elif selection >= 0 and selection < len(all_apis.keys()): + elif selection == i+3: # cancel + return + else: # select api = all_apis.keys()[selection] if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: if len(all_apis[api][u'use_scopes']) == 1: @@ -8886,33 +8893,35 @@ def doRequestOAuth(): select_value = u'*' else: select_value = u' ' - print u'[%s] %s) %s\n %s\n' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) + print u'[%s] %2d) %s\n %s\n' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) x += 1 - print u' %s) Select defaults for this API (allow all GAM commands)' % (x) - print u' %s) Select read-only scopes' % (x+1) - print u' %s) Unselect all scopes' % (x+2) - print u' %s) Back to all APIs' % (x+3) + print u' %2d) Select defaults for this API (allow all GAM commands)' % (x) + print u' %2d) Select read-only scopes' % (x+1) + print u' %2d) Unselect all scopes' % (x+2) + print u' %2d) Back to all APIs' % (x+3) + print u' %2d) Cancel' % (x+4) print - selection = getSelection() - num_scopes = len(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()) - if selection >= 0 and selection < num_scopes: + selection = getSelection(x+4) + if selection < x: # select if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in all_apis[api][u'use_scopes']: all_apis[api][u'use_scopes'].remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) else: all_apis[api][u'use_scopes'].append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) - elif selection == x: + elif selection == x: # defaults just_this_api = {api: all_apis[api]} just_this_api = select_default_scopes(just_this_api) all_apis[api][u'use_scopes'] = just_this_api[api][u'use_scopes'] - elif selection == x+1: + elif selection == x+1: # read-only all_apis[api][u'use_scopes'] = [] for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): if scope.endswith(u'.readonly'): all_apis[api][u'use_scopes'].append(scope) - elif selection == x+2: + elif selection == x+2: # unselect all all_apis[api][u'use_scopes'] = [] - elif selection == x+3: + elif selection == x+3: # back break + else: # cancel + return print MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST])) print print u','.join(GM_Globals[GM_GAMSCOPES_LIST]) From 0424ced6493680c37bb5255cd8a3a3319722e9e2 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 2 Jan 2016 13:35:08 -0800 Subject: [PATCH 056/100] Reorder cancel and continue/back in doRequestOAuth, add missing .keys() --- src/gam.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index 2265e20c..76b0e94c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -242,7 +242,7 @@ GC_VAR_INFO = { MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' MESSAGE_CLIENT_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' MESSAGE_CLIENT_API_ACCESS_DENIED = u'API access denied. Please make sure the service account ID: {0} is authorized for the API Scope(s): {1}' -MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' +MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV headers of "{1}".' @@ -8852,8 +8852,8 @@ def doRequestOAuth(): print print u' %2d) Select defaults for all APIs (allow all GAM commands)' % (i) print u' %2d) Unselect all APIs' % (i+1) - print u' %2d) Continue' % (i+2) - print u' %2d) Cancel' % (i+3) + print u' %2d) Cancel' % (i+2) + print u' %2d) Continue' % (i+3) print selection = getSelection(i+3) if selection == i: # defaults @@ -8861,7 +8861,7 @@ def doRequestOAuth(): elif selection == i+1: # unselect all for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] - elif selection == i+2: # continue + elif selection == i+3: # continue GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] for api in all_apis.keys(): @@ -8874,7 +8874,7 @@ def doRequestOAuth(): writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break - elif selection == i+3: # cancel + elif selection == i+2: # cancel return else: # select api = all_apis.keys()[selection] @@ -8882,7 +8882,7 @@ def doRequestOAuth(): if len(all_apis[api][u'use_scopes']) == 1: all_apis[api][u'use_scopes'] = [] else: - all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'] + all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() else: while True: os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) @@ -8898,8 +8898,8 @@ def doRequestOAuth(): print u' %2d) Select defaults for this API (allow all GAM commands)' % (x) print u' %2d) Select read-only scopes' % (x+1) print u' %2d) Unselect all scopes' % (x+2) - print u' %2d) Back to all APIs' % (x+3) - print u' %2d) Cancel' % (x+4) + print u' %2d) Cancel' % (x+3) + print u' %2d) Back to all APIs' % (x+4) print selection = getSelection(x+4) if selection < x: # select @@ -8918,7 +8918,7 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'].append(scope) elif selection == x+2: # unselect all all_apis[api][u'use_scopes'] = [] - elif selection == x+3: # back + elif selection == x+4: # back break else: # cancel return From 59a0aadd72e56c0c33fa9b9766ea88be63cd83a1 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Sun, 3 Jan 2016 09:20:56 -0500 Subject: [PATCH 057/100] ignore gamscopes.json --- src/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/src/.gitignore b/src/.gitignore index cdf2a6bb..34bf7990 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -67,3 +67,4 @@ gamcache/ gam/ gam-64/ *.zip +gamscopes.json From b7d537471805846d1803b3d4824930cb67c9ac2f Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Sun, 3 Jan 2016 10:11:12 -0500 Subject: [PATCH 058/100] refactor saving of scopes to be a single list, not per API -allows for APIs which have overlapping scopes (Drive, Apps Activity) -buildGAPIObject() can dynamically decide which scopes to ask for based on: -scopes that API claims to use (discovery document) -scopes that user has prevously selected -still need to fixup GData API calls to make do same. --- src/gam.py | 56 +++++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/gam.py b/src/gam.py index 76b0e94c..059da48e 100755 --- a/src/gam.py +++ b/src/gam.py @@ -91,7 +91,6 @@ GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json -GM_GAMSCOPES_BY_API = u'scop' GM_GAMSCOPES_LIST = u'scof' # Current API scope GM_CURRENT_API_SCOPES = u'scoc' @@ -117,7 +116,6 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, - GM_GAMSCOPES_BY_API: {}, GM_GAMSCOPES_LIST: [], GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, @@ -799,13 +797,13 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -def setCurrentAPIScopes(api, version=None): - if not version: - version = getAPIVer(api) - apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) - GM_Globals[GM_CURRENT_API_SCOPES] = apiData.get(u'use_scopes', []) - if not GM_Globals[GM_CURRENT_API_SCOPES]: +def setCurrentAPIScopes(service): + selected_scopes = GM_Globals[GM_GAMSCOPES_LIST] + all_api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys() + selected_api_scopes = list(set(all_api_scopes).intersection(selected_scopes)) + if len(selected_api_scopes) < 1: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) + return selected_api_scopes + [u'email'] def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -845,18 +843,21 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' - setCurrentAPIScopes(api, version) - credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], - GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=sub) - http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR])) + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) try: - return googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: - return getServiceFromDiscoveryDocument(api, version, http) + service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) + scopes = setCurrentAPIScopes(service) + + credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], + GM_Globals[GM_OAUTH2SERVICE_KEY], + scope=scopes, user_agent=GAM_INFO, sub=sub) + try: + service._http = credentials.authorize(http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', @@ -867,6 +868,7 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): if soft_errors: return False sys.exit(4) + return service def commonAppsObjInit(appsObj, api): getOAuth2ServiceDetails() @@ -8702,16 +8704,10 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def validateSetGAMScopes(json_data): - GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] - if not isinstance(json_data, dict): + if not isinstance(json_data, list): return False - for api, value in json_data.items(): - if (not isinstance(value, dict)) or (u'use_scopes' not in value) or (not isinstance(value[u'use_scopes'], list)): - return False - GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': value[u'use_scopes']} - GM_Globals[GM_GAMSCOPES_LIST] += value[u'use_scopes'] - GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only + GM_Globals[GM_GAMSCOPES_LIST] = json_data return len(GM_Globals[GM_GAMSCOPES_LIST]) > 0 def OAuthInfo(): @@ -8834,9 +8830,11 @@ def doRequestOAuth(): all_apis[api_name][u'index'] = i i += 1 all_apis = select_default_scopes(all_apis) - if GM_Globals[GM_GAMSCOPES_BY_API]: - for api in GM_Globals[GM_GAMSCOPES_BY_API]: - all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes'] + if GM_Globals[GM_GAMSCOPES_LIST]: + for api in all_apis.keys(): + for scope in GM_Globals[GM_GAMSCOPES_LIST]: + if scope in all_apis[api][u'auth'][u'oauth2'][u'scopes']: + all_apis[u'use_scopes'].append(scope) while True: os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' @@ -8862,16 +8860,14 @@ def doRequestOAuth(): for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: # continue - GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] for api in all_apis.keys(): - GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': all_apis[api][u'use_scopes']} GM_Globals[GM_GAMSCOPES_LIST] += all_apis[api][u'use_scopes'] GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_LIST])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break elif selection == i+2: # cancel From 3a38dceb5f7ce43fb06227b06a0a129ad03819db Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 10:00:12 -0800 Subject: [PATCH 059/100] Dynamic scope repair The last changes broke GData services and gam oath info --- src/gam.py | 69 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/src/gam.py b/src/gam.py index 059da48e..109adda8 100755 --- a/src/gam.py +++ b/src/gam.py @@ -91,6 +91,7 @@ GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json +GM_GAMSCOPES_BY_API = u'scop' GM_GAMSCOPES_LIST = u'scof' # Current API scope GM_CURRENT_API_SCOPES = u'scoc' @@ -116,6 +117,7 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, + GM_GAMSCOPES_BY_API: {}, GM_GAMSCOPES_LIST: [], GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, @@ -238,8 +240,10 @@ GC_VAR_INFO = { } MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' +MESSAGE_CLIENT_API_ACCESS_AUTHORIZED = u'API access authorized' MESSAGE_CLIENT_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' -MESSAGE_CLIENT_API_ACCESS_DENIED = u'API access denied. Please make sure the service account ID: {0} is authorized for the API Scope(s): {1}' +MESSAGE_CLIENT_API_ACCESS_DENIED = u'API access denied. Please make sure the Service account Client ID: {0} is authorized for the API Scope(s): {1}' +MESSAGE_CLIENT_API_ACCESS_NOT_REQUESTED = u'API access not requested' MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' @@ -250,11 +254,11 @@ MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for API {0}-{1}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' -MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT = u'Please authorize your service account client id for the {} scopes:' +MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT = u'Please authorize your Service account Client ID for the {} scopes:' MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' -MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service Account.' +MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service account.' def convertUTF8(data): import collections @@ -797,13 +801,21 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -def setCurrentAPIScopes(service): - selected_scopes = GM_Globals[GM_GAMSCOPES_LIST] - all_api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys() - selected_api_scopes = list(set(all_api_scopes).intersection(selected_scopes)) - if len(selected_api_scopes) < 1: +def setCurrentServiceScopes(service, api, version): + if GM_Globals[GM_GAMSCOPES_LIST]: + GM_Globals[GM_CURRENT_API_SCOPES] = list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST]))+[u'email',] + else: + GM_Globals[GM_CURRENT_API_SCOPES] = service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()+['email',] + if len(GM_Globals[GM_CURRENT_API_SCOPES]) < 2: + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) + +def setCurrentAPIScopes(api, version=None): + if not version: + version = getAPIVer(api) + apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) + GM_Globals[GM_CURRENT_API_SCOPES] = apiData.get(u'use_scopes', []) + if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) - return selected_api_scopes + [u'email'] def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -844,18 +856,17 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) + cache=GC_Values[GC_CACHE_DIR]) try: service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service) - + setCurrentServiceScopes(service, api, version) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scopes, user_agent=GAM_INFO, sub=sub) + scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=sub) try: service._http = credentials.authorize(http) except oauth2client.client.AccessTokenRefreshError, e: @@ -864,10 +875,10 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): u'access_denied: Requested client not authorized.']: sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES])))) systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_CONFIG) - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) return False - sys.exit(4) + systemErrorExit(4, e) return service def commonAppsObjInit(appsObj, api): @@ -8704,16 +8715,22 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def validateSetGAMScopes(json_data): + GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] - if not isinstance(json_data, list): + if not isinstance(json_data, dict): return False - GM_Globals[GM_GAMSCOPES_LIST] = json_data + for api, value in json_data.items(): + if (not isinstance(value, dict)) or (u'use_scopes' not in value) or (not isinstance(value[u'use_scopes'], list)): + return False + GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': value[u'use_scopes']} + GM_Globals[GM_GAMSCOPES_LIST] += value[u'use_scopes'] + GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only return len(GM_Globals[GM_GAMSCOPES_LIST]) > 0 def OAuthInfo(): configRequired = False getOAuth2ServiceDetails() - print u'API Scopes' + print u'API Access' for api in sorted(API_VER_MAPPING.keys()): print u' API: {0}'.format(api) version = getAPIVer(api) @@ -8743,8 +8760,10 @@ def OAuthInfo(): configRequired = True else: print u' {0}{1}'.format(ERROR_PREFIX, e) + continue + print u' {0}'.format(MESSAGE_CLIENT_API_ACCESS_AUTHORIZED) else: - print u' Not Requested' + print u' {0}'.format(MESSAGE_CLIENT_API_ACCESS_NOT_REQUESTED) if configRequired: print MESSAGE_CLIENT_API_ACCESS_CONFIG @@ -8830,11 +8849,9 @@ def doRequestOAuth(): all_apis[api_name][u'index'] = i i += 1 all_apis = select_default_scopes(all_apis) - if GM_Globals[GM_GAMSCOPES_LIST]: - for api in all_apis.keys(): - for scope in GM_Globals[GM_GAMSCOPES_LIST]: - if scope in all_apis[api][u'auth'][u'oauth2'][u'scopes']: - all_apis[u'use_scopes'].append(scope) + if GM_Globals[GM_GAMSCOPES_BY_API]: + for api in GM_Globals[GM_GAMSCOPES_BY_API]: + all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes'] while True: os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' @@ -8860,14 +8877,16 @@ def doRequestOAuth(): for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: # continue + GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] for api in all_apis.keys(): + GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': all_apis[api][u'use_scopes']} GM_Globals[GM_GAMSCOPES_LIST] += all_apis[api][u'use_scopes'] GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_LIST])) + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break elif selection == i+2: # cancel From 20e8175ae17ba7bf110e7d1263efed2aac31456e Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Sun, 3 Jan 2016 14:10:20 -0500 Subject: [PATCH 060/100] further updates to allow scopes to be saved as list also determine default scopes based on substrings --- src/gam.py | 101 ++++++++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 55 deletions(-) diff --git a/src/gam.py b/src/gam.py index 059da48e..a5f4294e 100755 --- a/src/gam.py +++ b/src/gam.py @@ -852,7 +852,6 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) scopes = setCurrentAPIScopes(service) - credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=sub) @@ -8719,22 +8718,25 @@ def OAuthInfo(): version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' - apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) - scopes = apiData.get(u'use_scopes', []) + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) + try: + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + except googleapiclient.errors.UnknownApiNameOrVersion: + service = getServiceFromDiscoveryDocument(api, version, http) + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, e) + scopes = setCurrentAPIScopes(service) if scopes: for scope in scopes: + if scope == u'email': + continue print u' {0}'.format(scope) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) - http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR])) try: - googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - getServiceFromDiscoveryDocument(api, version, http) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) + service._http = credentials.authorize(service._http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', @@ -8773,31 +8775,22 @@ UBER_SCOPES = { } def select_default_scopes(apis): + scopes = [] for api_name, api in apis.items(): if api_name in UBER_SCOPES.keys(): - apis[api_name][u'use_scopes'] = UBER_SCOPES[api_name] - continue - apis[api_name][u'use_scopes'] = [] - scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() - if len(scopes) == 1: - apis[api_name][u'use_scopes'] += scopes - continue - all_readonly = True - for scope in api[u'auth'][u'oauth2'][u'scopes'].keys(): - if scope.endswith(u'.readonly'): - continue - elif scope.endswith(u'.action'): - all_readonly = False - continue - elif scope.endswith(u'verify_only'): - all_readonly = False - continue - else: - apis[api_name][u'use_scopes'].append(scope) - all_readonly = False - if all_readonly: - apis[api_name][u'use_scopes'] += scopes - return apis + scopes += UBER_SCOPES[api_name] + else: + scopes += api[u'auth'][u'oauth2'][u'scopes'].keys() + selected_scopes = scopes + # reduce # of scopes by checking if a scope is a substring of another + # which should mean it covers same API operations. Add a . at end + # to prevent things like directory.users removing directory.userschema + for check_me in scopes: + check_me += u'.' + for against_me in scopes: + if check_me in against_me and check_me != against_me: + selected_scopes.remove(against_me) + return selected_scopes def getSelection(limit): while True: @@ -8829,19 +8822,18 @@ def doRequestOAuth(): for api_name in all_apis.keys(): all_apis[api_name][u'index'] = i i += 1 - all_apis = select_default_scopes(all_apis) if GM_Globals[GM_GAMSCOPES_LIST]: - for api in all_apis.keys(): - for scope in GM_Globals[GM_GAMSCOPES_LIST]: - if scope in all_apis[api][u'auth'][u'oauth2'][u'scopes']: - all_apis[u'use_scopes'].append(scope) + selected_scopes = GM_Globals[GM_GAMSCOPES_LIST] + else: + selected_scopes = select_default_scopes(all_apis) while True: - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) + #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' print for api in all_apis.values(): - num_scopes_selected = len(api[u'use_scopes']) - num_scopes_total = len(api[u'auth'][u'oauth2'][u'scopes']) + api_scopes = api[u'auth'][u'oauth2'][u'scopes'] + num_scopes_selected = len(set(api_scopes).intersection(selected_scopes)) + num_scopes_total = len(api_scopes) if num_scopes_selected > 0: select_value = u'*' else: @@ -8855,15 +8847,13 @@ def doRequestOAuth(): print selection = getSelection(i+3) if selection == i: # defaults - all_apis = select_default_scopes(all_apis) + selected_scopes = select_default_scopes(all_apis) elif selection == i+1: # unselect all for api in all_apis.keys(): - all_apis[api][u'use_scopes'] = [] + selected_scopes = [] elif selection == i+3: # continue - GM_Globals[GM_GAMSCOPES_LIST] = [] - for api in all_apis.keys(): - GM_Globals[GM_GAMSCOPES_LIST] += all_apis[api][u'use_scopes'] - GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only + selected_scopes = list(set(selected_scopes)) + GM_Globals[GM_GAMSCOPES_LIST] = selected_scopes # unique only if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue @@ -8875,17 +8865,18 @@ def doRequestOAuth(): else: # select api = all_apis.keys()[selection] if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: - if len(all_apis[api][u'use_scopes']) == 1: - all_apis[api][u'use_scopes'] = [] + one_scope = all_apis[api][u'auth'][u'oauth2'][u'scopes'][0] + if one_scope in selected_scopes: + selected_scopes.remove(one_scope) else: - all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() + selected_scopes.append(one_scope) else: while True: - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) + #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print x = 0 for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): - if scope in all_apis[api][u'use_scopes']: + if scope in selected_scopes: select_value = u'*' else: select_value = u' ' @@ -8899,10 +8890,10 @@ def doRequestOAuth(): print selection = getSelection(x+4) if selection < x: # select - if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in all_apis[api][u'use_scopes']: - all_apis[api][u'use_scopes'].remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in selected_scopes: + selected_scopes.remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) else: - all_apis[api][u'use_scopes'].append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + selected_scopes.append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) elif selection == x: # defaults just_this_api = {api: all_apis[api]} just_this_api = select_default_scopes(just_this_api) From 23cb9afec76a3e733aa41ce6e7bde898dba55bef Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 16:04:44 -0800 Subject: [PATCH 061/100] Merge remote-tracking branch 'jay0lee/master' # Conflicts: # src/gam.py --- src/gam.py | 104 +++++++++++++++++++++++++---------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/src/gam.py b/src/gam.py index 109adda8..03ded040 100755 --- a/src/gam.py +++ b/src/gam.py @@ -863,7 +863,7 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - setCurrentServiceScopes(service, api, version) + scopes = setCurrentAPIScopes(service) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=sub) @@ -8736,22 +8736,25 @@ def OAuthInfo(): version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' - apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) - scopes = apiData.get(u'use_scopes', []) + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR]) + try: + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + except googleapiclient.errors.UnknownApiNameOrVersion: + service = getServiceFromDiscoveryDocument(api, version, http) + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, e) + scopes = setCurrentAPIScopes(service) if scopes: for scope in scopes: + if scope == u'email': + continue print u' {0}'.format(scope) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) - http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR])) try: - googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - getServiceFromDiscoveryDocument(api, version, http) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) + service._http = credentials.authorize(service._http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', @@ -8792,31 +8795,22 @@ UBER_SCOPES = { } def select_default_scopes(apis): + scopes = [] for api_name, api in apis.items(): if api_name in UBER_SCOPES.keys(): - apis[api_name][u'use_scopes'] = UBER_SCOPES[api_name] - continue - apis[api_name][u'use_scopes'] = [] - scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() - if len(scopes) == 1: - apis[api_name][u'use_scopes'] += scopes - continue - all_readonly = True - for scope in api[u'auth'][u'oauth2'][u'scopes'].keys(): - if scope.endswith(u'.readonly'): - continue - elif scope.endswith(u'.action'): - all_readonly = False - continue - elif scope.endswith(u'verify_only'): - all_readonly = False - continue - else: - apis[api_name][u'use_scopes'].append(scope) - all_readonly = False - if all_readonly: - apis[api_name][u'use_scopes'] += scopes - return apis + scopes += UBER_SCOPES[api_name] + else: + scopes += api[u'auth'][u'oauth2'][u'scopes'].keys() + selected_scopes = scopes + # reduce # of scopes by checking if a scope is a substring of another + # which should mean it covers same API operations. Add a . at end + # to prevent things like directory.users removing directory.userschema + for check_me in scopes: + check_me += u'.' + for against_me in scopes: + if check_me in against_me and check_me != against_me: + selected_scopes.remove(against_me) + return selected_scopes def getSelection(limit): while True: @@ -8848,17 +8842,18 @@ def doRequestOAuth(): for api_name in all_apis.keys(): all_apis[api_name][u'index'] = i i += 1 - all_apis = select_default_scopes(all_apis) - if GM_Globals[GM_GAMSCOPES_BY_API]: - for api in GM_Globals[GM_GAMSCOPES_BY_API]: - all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api][u'use_scopes'] + if GM_Globals[GM_GAMSCOPES_LIST]: + selected_scopes = GM_Globals[GM_GAMSCOPES_LIST] + else: + selected_scopes = select_default_scopes(all_apis) while True: - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) + #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' print for api in all_apis.values(): - num_scopes_selected = len(api[u'use_scopes']) - num_scopes_total = len(api[u'auth'][u'oauth2'][u'scopes']) + api_scopes = api[u'auth'][u'oauth2'][u'scopes'] + num_scopes_selected = len(set(api_scopes).intersection(selected_scopes)) + num_scopes_total = len(api_scopes) if num_scopes_selected > 0: select_value = u'*' else: @@ -8872,17 +8867,13 @@ def doRequestOAuth(): print selection = getSelection(i+3) if selection == i: # defaults - all_apis = select_default_scopes(all_apis) + selected_scopes = select_default_scopes(all_apis) elif selection == i+1: # unselect all for api in all_apis.keys(): - all_apis[api][u'use_scopes'] = [] + selected_scopes = [] elif selection == i+3: # continue - GM_Globals[GM_GAMSCOPES_BY_API] = {} - GM_Globals[GM_GAMSCOPES_LIST] = [] - for api in all_apis.keys(): - GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': all_apis[api][u'use_scopes']} - GM_Globals[GM_GAMSCOPES_LIST] += all_apis[api][u'use_scopes'] - GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only + selected_scopes = list(set(selected_scopes)) + GM_Globals[GM_GAMSCOPES_LIST] = selected_scopes # unique only if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue @@ -8894,17 +8885,18 @@ def doRequestOAuth(): else: # select api = all_apis.keys()[selection] if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: - if len(all_apis[api][u'use_scopes']) == 1: - all_apis[api][u'use_scopes'] = [] + one_scope = all_apis[api][u'auth'][u'oauth2'][u'scopes'][0] + if one_scope in selected_scopes: + selected_scopes.remove(one_scope) else: - all_apis[api][u'use_scopes'] = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() + selected_scopes.append(one_scope) else: while True: - os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) + #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print x = 0 for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): - if scope in all_apis[api][u'use_scopes']: + if scope in selected_scopes: select_value = u'*' else: select_value = u' ' @@ -8918,10 +8910,10 @@ def doRequestOAuth(): print selection = getSelection(x+4) if selection < x: # select - if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in all_apis[api][u'use_scopes']: - all_apis[api][u'use_scopes'].remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in selected_scopes: + selected_scopes.remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) else: - all_apis[api][u'use_scopes'].append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + selected_scopes.append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) elif selection == x: # defaults just_this_api = {api: all_apis[api]} just_this_api = select_default_scopes(just_this_api) From 2288e99e56c63b349ed7979f0acc21a4ea8129e5 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 20:19:40 -0800 Subject: [PATCH 062/100] Dynamic scope cleanup --- src/gam.py | 80 ++++++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/gam.py b/src/gam.py index 03ded040..1b39087b 100755 --- a/src/gam.py +++ b/src/gam.py @@ -91,7 +91,6 @@ GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json -GM_GAMSCOPES_BY_API = u'scop' GM_GAMSCOPES_LIST = u'scof' # Current API scope GM_CURRENT_API_SCOPES = u'scoc' @@ -117,7 +116,6 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, - GM_GAMSCOPES_BY_API: {}, GM_GAMSCOPES_LIST: [], GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, @@ -801,21 +799,14 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -def setCurrentServiceScopes(service, api, version): +def setCurrentAPIScopes(service, api, version): if GM_Globals[GM_GAMSCOPES_LIST]: - GM_Globals[GM_CURRENT_API_SCOPES] = list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST]))+[u'email',] + selected_api_scopes = list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) else: - GM_Globals[GM_CURRENT_API_SCOPES] = service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()+['email',] - if len(GM_Globals[GM_CURRENT_API_SCOPES]) < 2: - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) - -def setCurrentAPIScopes(api, version=None): - if not version: - version = getAPIVer(api) - apiData = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), {}) - GM_Globals[GM_CURRENT_API_SCOPES] = apiData.get(u'use_scopes', []) - if not GM_Globals[GM_CURRENT_API_SCOPES]: + selected_api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys() + if len(selected_api_scopes) < 1: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) + return selected_api_scopes + [u'email'] def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -863,17 +854,17 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service) + scopes = setCurrentAPIScopes(service, api, version) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=sub) + scope=scopes, user_agent=GAM_INFO, sub=sub) try: service._http = credentials.authorize(http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES])))) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(scopes)))) systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_CONFIG) if soft_errors: sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) @@ -881,9 +872,17 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): systemErrorExit(4, e) return service +GDATA_API_SCOPES = { + u'admin-settings': u'https://apps-apis.google.com/a/feeds/domain/', + u'email-audit': u'https://apps-apis.google.com/a/feeds/compliance/audit/', + u'email-settings': u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/' + } + def commonAppsObjInit(appsObj, api): getOAuth2ServiceDetails() - setCurrentAPIScopes(api) + GM_Globals[GM_CURRENT_API_SCOPES] = GDATA_API_SCOPES[api] + if GM_Globals[GM_GAMSCOPES_LIST] and (GM_Globals[GM_CURRENT_API_SCOPES] not in GM_Globals[GM_GAMSCOPES_LIST]): + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, getAPIVer(api))) if not tryOAuth(appsObj): doRequestOAuth() tryOAuth(appsObj) @@ -8715,16 +8714,10 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def validateSetGAMScopes(json_data): - GM_Globals[GM_GAMSCOPES_BY_API] = {} GM_Globals[GM_GAMSCOPES_LIST] = [] - if not isinstance(json_data, dict): + if not isinstance(json_data, list): return False - for api, value in json_data.items(): - if (not isinstance(value, dict)) or (u'use_scopes' not in value) or (not isinstance(value[u'use_scopes'], list)): - return False - GM_Globals[GM_GAMSCOPES_BY_API][api] = {u'use_scopes': value[u'use_scopes']} - GM_Globals[GM_GAMSCOPES_LIST] += value[u'use_scopes'] - GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) # unique only + GM_Globals[GM_GAMSCOPES_LIST] = list(set(json_data)) # unique only return len(GM_Globals[GM_GAMSCOPES_LIST]) > 0 def OAuthInfo(): @@ -8737,24 +8730,24 @@ def OAuthInfo(): if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) + cache=GC_Values[GC_CACHE_DIR]) try: service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service) + systemErrorExit(4, e) + scopes = setCurrentAPIScopes(service, api, version) + if u'email' in scopes: + scopes.remove(u'email') if scopes: for scope in scopes: - if scope == u'email': - continue print u' {0}'.format(scope) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) try: - service._http = credentials.authorize(service._http) + service._http = credentials.refresh(http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', @@ -8763,7 +8756,7 @@ def OAuthInfo(): configRequired = True else: print u' {0}{1}'.format(ERROR_PREFIX, e) - continue + continue print u' {0}'.format(MESSAGE_CLIENT_API_ACCESS_AUTHORIZED) else: print u' {0}'.format(MESSAGE_CLIENT_API_ACCESS_NOT_REQUESTED) @@ -8801,15 +8794,20 @@ def select_default_scopes(apis): scopes += UBER_SCOPES[api_name] else: scopes += api[u'auth'][u'oauth2'][u'scopes'].keys() - selected_scopes = scopes + scopes.sort() + selected_scopes = [] # reduce # of scopes by checking if a scope is a substring of another # which should mean it covers same API operations. Add a . at end # to prevent things like directory.users removing directory.userschema - for check_me in scopes: - check_me += u'.' - for against_me in scopes: - if check_me in against_me and check_me != against_me: - selected_scopes.remove(against_me) + i = 0 + count = len(scopes) + while i < count: + scope = scopes[i] + selected_scopes.append(scope) + i += 1 + scope += u'.' + while (i < count) and scopes[i].startswith(scope): + i += 1 return selected_scopes def getSelection(limit): @@ -8877,7 +8875,7 @@ def doRequestOAuth(): if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_LIST])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break elif selection == i+2: # cancel @@ -8885,7 +8883,7 @@ def doRequestOAuth(): else: # select api = all_apis.keys()[selection] if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: - one_scope = all_apis[api][u'auth'][u'oauth2'][u'scopes'][0] + one_scope = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[0] if one_scope in selected_scopes: selected_scopes.remove(one_scope) else: From 5697580bd03b6323cad6eab0b2afe5c97683640f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 20:21:33 -0800 Subject: [PATCH 063/100] Fix typo --- src/gam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 1b39087b..e0e688e1 100755 --- a/src/gam.py +++ b/src/gam.py @@ -252,7 +252,7 @@ MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for API {0}-{1}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' -MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT = u'Please authorize your Service account Client ID for the {} scopes:' +MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service account Client ID for the {} scopes:' MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' @@ -8927,7 +8927,7 @@ def doRequestOAuth(): break else: # cancel return - print MESSAGE_PLEASE_AUTHORIZE_SERVIE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST])) + print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST])) print print u','.join(GM_Globals[GM_GAMSCOPES_LIST]) From 69689e286b02ca7abf0174b4febb579effb203cb Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 20:39:34 -0800 Subject: [PATCH 064/100] Fix setCurrentAPIScopes to work for both buildGAPIObject and OAuthInfo --- src/gam.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/gam.py b/src/gam.py index e0e688e1..968b9955 100755 --- a/src/gam.py +++ b/src/gam.py @@ -801,12 +801,9 @@ def getAPIVer(api): def setCurrentAPIScopes(service, api, version): if GM_Globals[GM_GAMSCOPES_LIST]: - selected_api_scopes = list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) + return list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) else: - selected_api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys() - if len(selected_api_scopes) < 1: - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) - return selected_api_scopes + [u'email'] + return service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys() def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -854,7 +851,9 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service, api, version) + scopes = setCurrentAPIScopes(service, api, version) + [u'email'] + if len(scopes) == 1: + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=sub) @@ -8738,8 +8737,6 @@ def OAuthInfo(): except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) scopes = setCurrentAPIScopes(service, api, version) - if u'email' in scopes: - scopes.remove(u'email') if scopes: for scope in scopes: print u' {0}'.format(scope) From c00d820c7552aa1b52ea8561b34e31b965fbed6a Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 21:42:32 -0800 Subject: [PATCH 065/100] Eliminate extraneous loop --- src/gam.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 968b9955..a39454c3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8864,8 +8864,7 @@ def doRequestOAuth(): if selection == i: # defaults selected_scopes = select_default_scopes(all_apis) elif selection == i+1: # unselect all - for api in all_apis.keys(): - selected_scopes = [] + selected_scopes = [] elif selection == i+3: # continue selected_scopes = list(set(selected_scopes)) GM_Globals[GM_GAMSCOPES_LIST] = selected_scopes # unique only From f35c1884965268a9ba17904822cf6e5e760247d5 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 3 Jan 2016 23:24:40 -0800 Subject: [PATCH 066/100] Finish refactoring doRequestOAuth, make selected_scopes a set --- src/gam.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/gam.py b/src/gam.py index a39454c3..ccb4989f 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8792,7 +8792,7 @@ def select_default_scopes(apis): else: scopes += api[u'auth'][u'oauth2'][u'scopes'].keys() scopes.sort() - selected_scopes = [] + selected_scopes = set() # reduce # of scopes by checking if a scope is a substring of another # which should mean it covers same API operations. Add a . at end # to prevent things like directory.users removing directory.userschema @@ -8800,7 +8800,7 @@ def select_default_scopes(apis): count = len(scopes) while i < count: scope = scopes[i] - selected_scopes.append(scope) + selected_scopes.add(scope) i += 1 scope += u'.' while (i < count) and scopes[i].startswith(scope): @@ -8838,7 +8838,7 @@ def doRequestOAuth(): all_apis[api_name][u'index'] = i i += 1 if GM_Globals[GM_GAMSCOPES_LIST]: - selected_scopes = GM_Globals[GM_GAMSCOPES_LIST] + selected_scopes = set(GM_Globals[GM_GAMSCOPES_LIST]) else: selected_scopes = select_default_scopes(all_apis) while True: @@ -8864,13 +8864,12 @@ def doRequestOAuth(): if selection == i: # defaults selected_scopes = select_default_scopes(all_apis) elif selection == i+1: # unselect all - selected_scopes = [] + selected_scopes.clear() elif selection == i+3: # continue - selected_scopes = list(set(selected_scopes)) - GM_Globals[GM_GAMSCOPES_LIST] = selected_scopes # unique only - if len(GM_Globals[GM_GAMSCOPES_LIST]) == 0: + if not selected_scopes: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue + GM_Globals[GM_GAMSCOPES_LIST] = list(selected_scopes) writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_LIST])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break @@ -8878,18 +8877,19 @@ def doRequestOAuth(): return else: # select api = all_apis.keys()[selection] - if len(all_apis[api][u'auth'][u'oauth2'][u'scopes']) == 1: - one_scope = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[0] + api_scopes = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() + if len(api_scopes) == 1: + one_scope = api_scopes[0] if one_scope in selected_scopes: selected_scopes.remove(one_scope) else: - selected_scopes.append(one_scope) + selected_scopes.add(one_scope) else: while True: #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print x = 0 - for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): + for scope in api_scopes: if scope in selected_scopes: select_value = u'*' else: @@ -8904,21 +8904,20 @@ def doRequestOAuth(): print selection = getSelection(x+4) if selection < x: # select - if all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection] in selected_scopes: - selected_scopes.remove(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + if api_scopes[selection] in selected_scopes: + selected_scopes.remove(api_scopes[selection]) else: - selected_scopes.append(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()[selection]) + selected_scopes.add(api_scopes[selection]) elif selection == x: # defaults - just_this_api = {api: all_apis[api]} - just_this_api = select_default_scopes(just_this_api) - all_apis[api][u'use_scopes'] = just_this_api[api][u'use_scopes'] + selected_scopes = selected_scopes.difference(api_scopes) + selected_scopes = selected_scopes.union(select_default_scopes({api: all_apis[api]})) elif selection == x+1: # read-only - all_apis[api][u'use_scopes'] = [] - for scope in all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys(): + selected_scopes = selected_scopes.difference(api_scopes) + for scope in api_scopes: if scope.endswith(u'.readonly'): - all_apis[api][u'use_scopes'].append(scope) + selected_scopes.add(scope) elif selection == x+2: # unselect all - all_apis[api][u'use_scopes'] = [] + selected_scopes = selected_scopes.difference(api_scopes) elif selection == x+4: # back break else: # cancel From 0209b51c4d6a88263125ca989365f6cafbf387a0 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 4 Jan 2016 05:48:07 -0800 Subject: [PATCH 067/100] Clean up no setCurrentAPIScopes, api, version not needed No scopes error message changed to take API title --- src/gam.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/gam.py b/src/gam.py index ccb4989f..bf5679f7 100755 --- a/src/gam.py +++ b/src/gam.py @@ -249,7 +249,7 @@ MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV header MESSAGE_HIT_CONTROL_C_TO_UPDATE = u'\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.' MESSAGE_NO_DISCOVERY_INFORMATION = u'No online discovery doc and {0} does not exist locally' MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' -MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for API {0}-{1}; please run gam oauth create' +MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for {0}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service account Client ID for the {} scopes:' @@ -799,7 +799,7 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -def setCurrentAPIScopes(service, api, version): +def setCurrentAPIScopes(service): if GM_Globals[GM_GAMSCOPES_LIST]: return list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) else: @@ -851,9 +851,10 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service, api, version) + [u'email'] - if len(scopes) == 1: - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) + scopes = setCurrentAPIScopes(service) + if not scopes: + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) + scopes += [u'email',] credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=sub) @@ -871,17 +872,17 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): systemErrorExit(4, e) return service -GDATA_API_SCOPES = { - u'admin-settings': u'https://apps-apis.google.com/a/feeds/domain/', - u'email-audit': u'https://apps-apis.google.com/a/feeds/compliance/audit/', - u'email-settings': u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/' +GDATA_API_INFO = { + u'admin-settings': {u'scopes': u'https://apps-apis.google.com/a/feeds/domain/', u'title': u'Admin Settings API'}, + u'email-audit': {u'scopes': u'https://apps-apis.google.com/a/feeds/compliance/audit/', u'title': u'Email Audit API'}, + u'email-settings': {u'scopes': u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/', u'title': u'Email Settings API'}, } def commonAppsObjInit(appsObj, api): getOAuth2ServiceDetails() - GM_Globals[GM_CURRENT_API_SCOPES] = GDATA_API_SCOPES[api] + GM_Globals[GM_CURRENT_API_SCOPES] = GDATA_API_INFO[api][u'scopes'] if GM_Globals[GM_GAMSCOPES_LIST] and (GM_Globals[GM_CURRENT_API_SCOPES] not in GM_Globals[GM_GAMSCOPES_LIST]): - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, getAPIVer(api))) + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(GDATA_API_INFO[api][u'title'])) if not tryOAuth(appsObj): doRequestOAuth() tryOAuth(appsObj) @@ -8736,7 +8737,7 @@ def OAuthInfo(): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service, api, version) + scopes = setCurrentAPIScopes(service) if scopes: for scope in scopes: print u' {0}'.format(scope) From dae75f6234055c3be0ce54d5f850cf03d09019f9 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 4 Jan 2016 06:02:10 -0800 Subject: [PATCH 068/100] Fix typo in email-audit-v1.jsom --- src/email-audit-v1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json index 52fcf1e1..afc5a89f 100644 --- a/src/email-audit-v1.json +++ b/src/email-audit-v1.json @@ -6,7 +6,7 @@ "version": "v1", "revision": "20130823", "title": "Email Audit API", - "description": "Lets you peform Google Apps email audits", + "description": "Lets you perform Google Apps email audits", "ownerDomain": "google.com", "ownerName": "Google", "icons": { From cb79688a734d11712334d03d30d1035e6d577909 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 4 Jan 2016 10:52:33 -0800 Subject: [PATCH 069/100] in doRequestOAuth, sort list by API title --- src/gam.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index bf5679f7..3b0afe38 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8823,6 +8823,7 @@ def getSelection(limit): def doRequestOAuth(): apis = API_VER_MAPPING.keys() all_apis = {} + api_titles = {} for api in apis: version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: @@ -8833,11 +8834,13 @@ def doRequestOAuth(): service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) - all_apis[u'%s-%s' % (api, version)] = service._rootDesc - i = 0 - for api_name in all_apis.keys(): - all_apis[api_name][u'index'] = i - i += 1 + api_name = u'%s-%s' % (api, version) + all_apis[api_name] = service._rootDesc + api_titles[api_name] = api_name + api_index = [] + for title, api_name in sorted(api_titles.items()): + api_index.append(api_name) + i = len(api_index) if GM_Globals[GM_GAMSCOPES_LIST]: selected_scopes = set(GM_Globals[GM_GAMSCOPES_LIST]) else: @@ -8846,7 +8849,8 @@ def doRequestOAuth(): #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' print - for api in all_apis.values(): + for n in range(i): + api = all_apis[api_index[n]] api_scopes = api[u'auth'][u'oauth2'][u'scopes'] num_scopes_selected = len(set(api_scopes).intersection(selected_scopes)) num_scopes_total = len(api_scopes) @@ -8854,7 +8858,7 @@ def doRequestOAuth(): select_value = u'*' else: select_value = u' ' - print u'[%s] %2d) %s (%d/%d scopes)' % (select_value, api[u'index'], api[u'title'], num_scopes_selected, num_scopes_total) + print u'[%s] %2d) %s (%d/%d scopes)' % (select_value, n, api[u'title'], num_scopes_selected, num_scopes_total) print print u' %2d) Select defaults for all APIs (allow all GAM commands)' % (i) print u' %2d) Unselect all APIs' % (i+1) @@ -8877,7 +8881,7 @@ def doRequestOAuth(): elif selection == i+2: # cancel return else: # select - api = all_apis.keys()[selection] + api = api_index[selection] api_scopes = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() if len(api_scopes) == 1: one_scope = api_scopes[0] @@ -8895,8 +8899,9 @@ def doRequestOAuth(): select_value = u'*' else: select_value = u' ' - print u'[%s] %2d) %s\n %s\n' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) + print u'[%s] %2d) %s\n %s' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) x += 1 + print print u' %2d) Select defaults for this API (allow all GAM commands)' % (x) print u' %2d) Select read-only scopes' % (x+1) print u' %2d) Unselect all scopes' % (x+2) From d258d4da63f7c98146460fa810e5c5befe3294e7 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 4 Jan 2016 12:23:54 -0800 Subject: [PATCH 070/100] Refactor doRequestOAuth The selected scopes list can't be created until completion; otherwise turning off a scope in one API turns it off in all other APIs. Elimination of child scopes is still supported in select-default. Top level API list is sorted. --- src/gam.py | 85 +++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/src/gam.py b/src/gam.py index 3b0afe38..b2670b5d 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8781,32 +8781,28 @@ def doDeleteOAuth(): UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], - u'drive-v2': [u'https://www.googleapis.com/auth/drive'], - u'appsactivity-v1': [u'https://www.googleapis.com/auth/activity'] } def select_default_scopes(apis): - scopes = [] for api_name, api in apis.items(): - if api_name in UBER_SCOPES.keys(): - scopes += UBER_SCOPES[api_name] + if api_name in UBER_SCOPES: + api[u'use_scopes'] = UBER_SCOPES[api_name] else: - scopes += api[u'auth'][u'oauth2'][u'scopes'].keys() - scopes.sort() - selected_scopes = set() - # reduce # of scopes by checking if a scope is a substring of another - # which should mean it covers same API operations. Add a . at end - # to prevent things like directory.users removing directory.userschema - i = 0 - count = len(scopes) - while i < count: - scope = scopes[i] - selected_scopes.add(scope) - i += 1 - scope += u'.' - while (i < count) and scopes[i].startswith(scope): - i += 1 - return selected_scopes + scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() + scopes.sort() + api[u'use_scopes'] = [] + # reduce # of scopes by checking if a scope is a substring of another + # which should mean it covers same API operations. Add a . at end + # to prevent things like directory.users removing directory.userschema + i = 0 + count = len(scopes) + while i < count: + scope = scopes[i] + api[u'use_scopes'].append(scope) + i += 1 + scope += u'.' + while (i < count) and scopes[i].startswith(scope): + i += 1 def getSelection(limit): while True: @@ -8838,21 +8834,22 @@ def doRequestOAuth(): all_apis[api_name] = service._rootDesc api_titles[api_name] = api_name api_index = [] - for title, api_name in sorted(api_titles.items()): + for _, api_name in sorted(api_titles.items()): api_index.append(api_name) i = len(api_index) if GM_Globals[GM_GAMSCOPES_LIST]: - selected_scopes = set(GM_Globals[GM_GAMSCOPES_LIST]) + for api in all_apis: + all_apis[api][u'use_scopes'] = list(set(GM_Globals[GM_GAMSCOPES_LIST]) & set(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys())) else: - selected_scopes = select_default_scopes(all_apis) + select_default_scopes(all_apis) while True: #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' print for n in range(i): api = all_apis[api_index[n]] - api_scopes = api[u'auth'][u'oauth2'][u'scopes'] - num_scopes_selected = len(set(api_scopes).intersection(selected_scopes)) + api_scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() + num_scopes_selected = len(api[u'use_scopes']) num_scopes_total = len(api_scopes) if num_scopes_selected > 0: select_value = u'*' @@ -8867,14 +8864,18 @@ def doRequestOAuth(): print selection = getSelection(i+3) if selection == i: # defaults - selected_scopes = select_default_scopes(all_apis) + select_default_scopes(all_apis) elif selection == i+1: # unselect all - selected_scopes.clear() + for api in all_apis.keys(): + all_apis[api][u'use_scopes'] = [] elif selection == i+3: # continue + selected_scopes = [] + for api in all_apis.keys(): + selected_scopes += all_apis[api][u'use_scopes'] if not selected_scopes: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - GM_Globals[GM_GAMSCOPES_LIST] = list(selected_scopes) + GM_Globals[GM_GAMSCOPES_LIST] = list(set(selected_scopes)) writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_LIST])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) break @@ -8884,18 +8885,17 @@ def doRequestOAuth(): api = api_index[selection] api_scopes = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() if len(api_scopes) == 1: - one_scope = api_scopes[0] - if one_scope in selected_scopes: - selected_scopes.remove(one_scope) + if len(all_apis[api][u'use_scopes']) == 1: + all_apis[api][u'use_scopes'] = [] else: - selected_scopes.add(one_scope) + all_apis[api][u'use_scopes'] = api_scopes else: while True: #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print x = 0 for scope in api_scopes: - if scope in selected_scopes: + if scope in all_apis[api][u'use_scopes']: select_value = u'*' else: select_value = u' ' @@ -8910,20 +8910,21 @@ def doRequestOAuth(): print selection = getSelection(x+4) if selection < x: # select - if api_scopes[selection] in selected_scopes: - selected_scopes.remove(api_scopes[selection]) + if api_scopes[selection] in all_apis[api][u'use_scopes']: + all_apis[api][u'use_scopes'].remove(api_scopes[selection]) else: - selected_scopes.add(api_scopes[selection]) + all_apis[api][u'use_scopes'].append(api_scopes[selection]) elif selection == x: # defaults - selected_scopes = selected_scopes.difference(api_scopes) - selected_scopes = selected_scopes.union(select_default_scopes({api: all_apis[api]})) + just_this_api = {api: all_apis[api]} + select_default_scopes(just_this_api) + all_apis[api][u'use_scopes'] = just_this_api[api][u'use_scopes'] elif selection == x+1: # read-only - selected_scopes = selected_scopes.difference(api_scopes) + all_apis[api][u'use_scopes'] = [] for scope in api_scopes: if scope.endswith(u'.readonly'): - selected_scopes.add(scope) + all_apis[api][u'use_scopes'].append(scope) elif selection == x+2: # unselect all - selected_scopes = selected_scopes.difference(api_scopes) + all_apis[api][u'use_scopes'] = [] elif selection == x+4: # back break else: # cancel From 25e97b97d4634d4ec053d228a919647191a0e983 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 4 Jan 2016 12:57:23 -0800 Subject: [PATCH 071/100] Sort scopes in doRequestOauth, move scope over for readability --- src/gam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index b2670b5d..8986e94a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8883,7 +8883,7 @@ def doRequestOAuth(): return else: # select api = api_index[selection] - api_scopes = all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys() + api_scopes = sorted(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()) if len(api_scopes) == 1: if len(all_apis[api][u'use_scopes']) == 1: all_apis[api][u'use_scopes'] = [] @@ -8899,7 +8899,7 @@ def doRequestOAuth(): select_value = u'*' else: select_value = u' ' - print u'[%s] %2d) %s\n %s' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) + print u'[%s] %2d) %s\n %s' % (select_value, x, all_apis[api][u'auth'][u'oauth2'][u'scopes'][scope][u'description'], scope) x += 1 print print u' %2d) Select defaults for this API (allow all GAM commands)' % (x) From 5c64f0825fb450bd610bcb8c804b151a0b20f700 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 4 Jan 2016 13:10:25 -0800 Subject: [PATCH 072/100] Fix UTF error in getDelegates --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 8986e94a..8671d8bf 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1227,7 +1227,7 @@ def getDelegates(users): if csv_format: print u'%s,%s,%s' % (user + u'@' + emailsettings.domain, delegate[u'address'], delegate[u'status']) else: - print u"Delegator: %s\n Delegate: %s\n Status: %s\n Delegate Email: %s\n Delegate ID: %s\n" % (user, delegate[u'delegate'], delegate[u'status'], delegate[u'address'], delegate[u'delegationId']) + print convertUTF8(u"Delegator: %s\n Delegate: %s\n Status: %s\n Delegate Email: %s\n Delegate ID: %s\n" % (user, delegate[u'delegate'], delegate[u'status'], delegate[u'address'], delegate[u'delegationId'])) except TypeError: pass From 95bb288e38d5d31ea07c1ae85d7a810c5f9ba403 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 5 Jan 2016 08:18:19 -0800 Subject: [PATCH 073/100] Back to scopes by api --- src/gam.py | 91 ++++++++++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/src/gam.py b/src/gam.py index 8671d8bf..1feab3ae 100755 --- a/src/gam.py +++ b/src/gam.py @@ -91,7 +91,7 @@ GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json -GM_GAMSCOPES_LIST = u'scof' +GM_GAMSCOPES_BY_API = u'scop' # Current API scope GM_CURRENT_API_SCOPES = u'scoc' # Values retrieved from oauth2service.json @@ -116,7 +116,7 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, - GM_GAMSCOPES_LIST: [], + GM_GAMSCOPES_BY_API: {}, GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, @@ -252,7 +252,7 @@ MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for {0}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' -MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service account Client ID for the {} scopes:' +MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service account Client ID for the {0} scopes:\n\n{1}' MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' @@ -522,6 +522,7 @@ def SetGlobalVariables(): GM_Globals[GM_OAUTH2SERVICE_KEY] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = None + GM_Globals[GM_GAMSCOPES_BY_API] = {} json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) if json_string and not validateSetGAMScopes(json.loads(json_string)): systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) @@ -799,12 +800,6 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -def setCurrentAPIScopes(service): - if GM_Globals[GM_GAMSCOPES_LIST]: - return list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) - else: - return service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys() - def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) @@ -851,7 +846,7 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service) + scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), []) if not scopes: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) scopes += [u'email',] @@ -873,16 +868,16 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): return service GDATA_API_INFO = { - u'admin-settings': {u'scopes': u'https://apps-apis.google.com/a/feeds/domain/', u'title': u'Admin Settings API'}, - u'email-audit': {u'scopes': u'https://apps-apis.google.com/a/feeds/compliance/audit/', u'title': u'Email Audit API'}, - u'email-settings': {u'scopes': u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/', u'title': u'Email Settings API'}, + u'admin-settings': u'Admin Settings API', + u'email-audit': u'Email Audit API', + u'email-settings': u'Email Settings API', } def commonAppsObjInit(appsObj, api): getOAuth2ServiceDetails() - GM_Globals[GM_CURRENT_API_SCOPES] = GDATA_API_INFO[api][u'scopes'] - if GM_Globals[GM_GAMSCOPES_LIST] and (GM_Globals[GM_CURRENT_API_SCOPES] not in GM_Globals[GM_GAMSCOPES_LIST]): - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(GDATA_API_INFO[api][u'title'])) + GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, getAPIVer(api)), []) + if not GM_Globals[GM_CURRENT_API_SCOPES]: + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(GDATA_API_INFO[api])) if not tryOAuth(appsObj): doRequestOAuth() tryOAuth(appsObj) @@ -8714,11 +8709,14 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def validateSetGAMScopes(json_data): - GM_Globals[GM_GAMSCOPES_LIST] = [] - if not isinstance(json_data, list): + GM_Globals[GM_GAMSCOPES_BY_API] = {} + if not isinstance(json_data, dict): return False - GM_Globals[GM_GAMSCOPES_LIST] = list(set(json_data)) # unique only - return len(GM_Globals[GM_GAMSCOPES_LIST]) > 0 + for api, value in json_data.items(): + if not isinstance(value, list): + return False + GM_Globals[GM_GAMSCOPES_BY_API][api] = set(value) + return len(GM_Globals[GM_GAMSCOPES_BY_API]) > 0 def OAuthInfo(): configRequired = False @@ -8737,27 +8735,27 @@ def OAuthInfo(): service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = setCurrentAPIScopes(service) - if scopes: - for scope in scopes: - print u' {0}'.format(scope) - credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], - GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scopes, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) - try: - service._http = credentials.refresh(http) - except oauth2client.client.AccessTokenRefreshError, e: - if e.message in [u'access_denied', - u'unauthorized_client: Unauthorized client or scope in request.', - u'access_denied: Requested client not authorized.']: - print u' {0}{1}'.format(WARNING_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u'listed above')) + api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] + requested_scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), []) + if requested_scopes: + for scope in requested_scopes: + credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], + GM_Globals[GM_OAUTH2SERVICE_KEY], + scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) + try: + service._http = credentials.refresh(http) + status = u'Authorized' + except oauth2client.client.AccessTokenRefreshError, e: configRequired = True - else: - print u' {0}{1}'.format(ERROR_PREFIX, e) - continue - print u' {0}'.format(MESSAGE_CLIENT_API_ACCESS_AUTHORIZED) + if e.message in [u'access_denied', + u'unauthorized_client: Unauthorized client or scope in request.', + u'access_denied: Requested client not authorized.']: + status = u'DENIED' + else: + status = u'{0}{1}'.format(ERROR_PREFIX, e) + print u' {0}\n {1}\n Access: {2}'.format(api_scopes[scope][u'description'], scope, status) else: - print u' {0}'.format(MESSAGE_CLIENT_API_ACCESS_NOT_REQUESTED) + print u' Access: Not requested' if configRequired: print MESSAGE_CLIENT_API_ACCESS_CONFIG @@ -8837,9 +8835,9 @@ def doRequestOAuth(): for _, api_name in sorted(api_titles.items()): api_index.append(api_name) i = len(api_index) - if GM_Globals[GM_GAMSCOPES_LIST]: + if GM_Globals[GM_GAMSCOPES_BY_API]: for api in all_apis: - all_apis[api][u'use_scopes'] = list(set(GM_Globals[GM_GAMSCOPES_LIST]) & set(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys())) + all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api] else: select_default_scopes(all_apis) while True: @@ -8869,16 +8867,18 @@ def doRequestOAuth(): for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: # continue + GM_Globals[GM_GAMSCOPES_BY_API] = {} selected_scopes = [] for api in all_apis.keys(): + GM_Globals[GM_GAMSCOPES_BY_API][api] = all_apis[api][u'use_scopes'] selected_scopes += all_apis[api][u'use_scopes'] if not selected_scopes: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - GM_Globals[GM_GAMSCOPES_LIST] = list(set(selected_scopes)) - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_LIST])) + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) - break + print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(selected_scopes), u','.join(selected_scopes)) + return elif selection == i+2: # cancel return else: # select @@ -8929,9 +8929,6 @@ def doRequestOAuth(): break else: # cancel return - print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST])) - print - print u','.join(GM_Globals[GM_GAMSCOPES_LIST]) def batch_worker(): while True: From 12ccd58eae07e8fa4535a81fbb14c201e7bf9ff9 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 5 Jan 2016 08:54:14 -0800 Subject: [PATCH 074/100] Fix error --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 1feab3ae..61c69711 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8715,7 +8715,7 @@ def validateSetGAMScopes(json_data): for api, value in json_data.items(): if not isinstance(value, list): return False - GM_Globals[GM_GAMSCOPES_BY_API][api] = set(value) + GM_Globals[GM_GAMSCOPES_BY_API][api] = list(set(value)) return len(GM_Globals[GM_GAMSCOPES_BY_API]) > 0 def OAuthInfo(): From f8642a18dfefd9594bf1be57a1b3ee2341c2a172 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 5 Jan 2016 12:06:18 -0800 Subject: [PATCH 075/100] Delete 'email' scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If this ‘email’ scope is included here then it better be output in the scopes list in doRequestOAuth otherwise nothing works. Why was it even here? --- src/gam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 61c69711..b04b55bc 100755 --- a/src/gam.py +++ b/src/gam.py @@ -849,7 +849,6 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), []) if not scopes: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) - scopes += [u'email',] credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scopes, user_agent=GAM_INFO, sub=sub) From 8eb347488fbca8354b607bdf4150a9f28ab28908 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 6 Jan 2016 06:23:57 -0800 Subject: [PATCH 076/100] Eliminate 'email' scope, may sure that selected_scopes has unique elements --- src/gam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gam.py b/src/gam.py index b04b55bc..42edc5dc 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8871,6 +8871,7 @@ def doRequestOAuth(): for api in all_apis.keys(): GM_Globals[GM_GAMSCOPES_BY_API][api] = all_apis[api][u'use_scopes'] selected_scopes += all_apis[api][u'use_scopes'] + selected_scopes = set(selected_scopes) if not selected_scopes: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue From 8929ee534fd8629bda3b5da7dd5aec89af1ebd34 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 7 Jan 2016 13:07:43 -0800 Subject: [PATCH 077/100] Ease transition to new all service model Get admin email address and domain from oauth2.txt if it exists --- src/gam.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/gam.py b/src/gam.py index 42edc5dc..47e8c339 100755 --- a/src/gam.py +++ b/src/gam.py @@ -238,10 +238,8 @@ GC_VAR_INFO = { } MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' -MESSAGE_CLIENT_API_ACCESS_AUTHORIZED = u'API access authorized' MESSAGE_CLIENT_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' MESSAGE_CLIENT_API_ACCESS_DENIED = u'API access denied. Please make sure the Service account Client ID: {0} is authorized for the API Scope(s): {1}' -MESSAGE_CLIENT_API_ACCESS_NOT_REQUESTED = u'API access not requested' MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' @@ -442,6 +440,20 @@ def SetGlobalVariables(): def _getOldSignalFile(itemName, trueValue=True, falseValue=False): GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY])) else falseValue + def _getAdminDomainFromOldOauth2Txt(): + if (not GC_Defaults[GC_ADMIN]) or (not GC_Defaults[GC_DOMAIN]): + srcFile = os.path.expanduser(os.environ.get(u'OAUTHFILE', u'oauth2.txt')) + if not os.path.isabs(srcFile): + srcFile = os.path.expanduser(os.path.join(GC_Defaults[GC_CONFIG_DIR], srcFile)) + if os.path.isfile(srcFile): + json_string = readFile(srcFile, continueOnError=True, displayError=False) + if json_string: + json_data = json.loads(json_string) + if not GC_Defaults[GC_ADMIN]: + GC_Defaults[GC_ADMIN] = json_data.get(u'id_token', {}).get(u'email', u'') + if not GC_Defaults[GC_DOMAIN]: + GC_Defaults[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', u'') + def _getCfgDirectory(itemName): return GC_Defaults[itemName] @@ -493,6 +505,7 @@ def SetGlobalVariables(): _getOldSignalFile(GC_NO_BROWSER) _getOldSignalFile(GC_NO_CACHE) _getOldSignalFile(GC_NO_UPDATE_CHECK) + _getAdminDomainFromOldOauth2Txt() # Assign directories first for itemName in GC_VAR_INFO: if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_DIRECTORY: @@ -833,7 +846,7 @@ def getOAuth2ServiceDetails(): systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) def buildGAPIObject(api, act_as=None, soft_errors=False): - sub = act_as if act_as else GC_Values[GC_ADMIN] + svcsub = act_as if act_as else GC_Values[GC_ADMIN] getOAuth2ServiceDetails() version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: @@ -851,7 +864,7 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scopes, user_agent=GAM_INFO, sub=sub) + scope=scopes, user_agent=GAM_INFO, sub=svcsub) try: service._http = credentials.authorize(http) except oauth2client.client.AccessTokenRefreshError, e: From 0333e29eef388c203d718977155e20aa41b29cd0 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 10 Jan 2016 09:58:21 -0800 Subject: [PATCH 078/100] Service creation, dynamic scope cleanup Make routine getAPIversionHttpService to handle all steps to get a service. Make routine handleOAuthTokenError to handle OAuth token errors. In doRequestOAuth, get admin email address from oauth2.txt if it exists, otherwise prompt for it. Move reading of gamscopes,json into SetGlobalVariables as a local routine _getScopesAdminDomainFromGamScopesJson. --- src/gam.py | 334 +++++++++++++++++++++++++++-------------------------- 1 file changed, 170 insertions(+), 164 deletions(-) diff --git a/src/gam.py b/src/gam.py index 47e8c339..ecffe2df 100755 --- a/src/gam.py +++ b/src/gam.py @@ -181,7 +181,7 @@ GC_Defaults = { GC_CACHE_DIR: u'', GC_CHARSET: u'utf-8', GC_CONFIG_DIR: u'', - GC_CUSTOMER_ID: u'my_customer', + GC_CUSTOMER_ID: MY_CUSTOMER, GC_DEBUG_LEVEL: 0, GC_DEVICE_MAX_RESULTS: 500, GC_DOMAIN: u'', @@ -238,8 +238,8 @@ GC_VAR_INFO = { } MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' -MESSAGE_CLIENT_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' -MESSAGE_CLIENT_API_ACCESS_DENIED = u'API access denied. Please make sure the Service account Client ID: {0} is authorized for the API Scope(s): {1}' +MESSAGE_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' +MESSAGE_API_ACCESS_DENIED = u'API access denied.\n\nPlease make sure the Service account Client ID: {0} is authorized for the API Scope(s): {1}\n\nPlease make sure the Admin email address: {2} is valid' MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' @@ -256,6 +256,8 @@ MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloadin MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service account.' +OAUTH_TOKEN_ERRORS = [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.', u'invalid_grant: Not a valid email.', u'invalid_request: Invalid impersonation prn email address.'] + def convertUTF8(data): import collections if isinstance(data, str): @@ -355,6 +357,9 @@ def systemErrorExit(sysRC, message): sys.stderr.write(u'\n{0}{1}\n'.format(ERROR_PREFIX, message)) sys.exit(sysRC) +def badGAMScopesExit(): + systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) + def noPythonSSLExit(): systemErrorExit(8, MESSAGE_NO_PYTHON_SSL) @@ -416,7 +421,15 @@ def writeFile(filename, data, mode=u'wb', continueOnError=False, displayError=Tr sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) return False systemErrorExit(6, e) + +# Get global domain from global admin email address # +def getDomainFromAdmin(): + if GC_Values[GC_ADMIN]: + loc = GC_Values[GC_ADMIN].find(u'@') + if loc > 0: + GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][loc+1:] + # Set global variables # Check for GAM updates based on status of noupdatecheck.txt # @@ -440,19 +453,25 @@ def SetGlobalVariables(): def _getOldSignalFile(itemName, trueValue=True, falseValue=False): GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY])) else falseValue - def _getAdminDomainFromOldOauth2Txt(): - if (not GC_Defaults[GC_ADMIN]) or (not GC_Defaults[GC_DOMAIN]): - srcFile = os.path.expanduser(os.environ.get(u'OAUTHFILE', u'oauth2.txt')) - if not os.path.isabs(srcFile): - srcFile = os.path.expanduser(os.path.join(GC_Defaults[GC_CONFIG_DIR], srcFile)) - if os.path.isfile(srcFile): - json_string = readFile(srcFile, continueOnError=True, displayError=False) - if json_string: - json_data = json.loads(json_string) - if not GC_Defaults[GC_ADMIN]: - GC_Defaults[GC_ADMIN] = json_data.get(u'id_token', {}).get(u'email', u'') - if not GC_Defaults[GC_DOMAIN]: - GC_Defaults[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', u'') + def _getScopesAdminDomainFromGamScopesJson(): + GM_Globals[GM_GAMSCOPES_BY_API] = {} + json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) + if json_string == None: + return + json_data = json.loads(json_string) + if not isinstance(json_data, dict): + badGAMScopesExit() + scopes = json_data.get(u'scopes', None) + if (not scopes) or (not isinstance(scopes, dict)): + badGAMScopesExit() + for api, value in scopes.items(): + if not isinstance(value, list): + badGAMScopesExit() + GM_Globals[GM_GAMSCOPES_BY_API][api] = list(set(value)) + if not GC_Values[GC_ADMIN]: + GC_Values[GC_ADMIN] = json_data.get(u'admin', GC_Defaults[GC_ADMIN]) + if not GC_Values[GC_DOMAIN]: + GC_Values[GC_DOMAIN] = json_data.get(u'domain', GC_Defaults[GC_DOMAIN]) def _getCfgDirectory(itemName): return GC_Defaults[itemName] @@ -505,7 +524,6 @@ def SetGlobalVariables(): _getOldSignalFile(GC_NO_BROWSER) _getOldSignalFile(GC_NO_CACHE) _getOldSignalFile(GC_NO_UPDATE_CHECK) - _getAdminDomainFromOldOauth2Txt() # Assign directories first for itemName in GC_VAR_INFO: if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_DIRECTORY: @@ -519,10 +537,6 @@ def SetGlobalVariables(): GM_Globals[GM_LAST_UPDATE_CHECK_TXT] = os.path.join(GC_Values[GC_CONFIG_DIR], FN_LAST_UPDATE_CHECK_TXT) if not GC_Values[GC_NO_UPDATE_CHECK]: doGAMCheckForUpdates() - if (not GC_Values[GC_DOMAIN]) and GC_Values[GC_ADMIN]: - loc = GC_Values[GC_ADMIN].find(u'@') - if loc > 0: - GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][loc+1:] # Globals derived from config file values GM_Globals[GM_EXTRA_ARGS_DICT] = {u'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} httplib2.debuglevel = GC_Values[GC_DEBUG_LEVEL] @@ -535,10 +549,9 @@ def SetGlobalVariables(): GM_Globals[GM_OAUTH2SERVICE_KEY] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = None - GM_Globals[GM_GAMSCOPES_BY_API] = {} - json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) - if json_string and not validateSetGAMScopes(json.loads(json_string)): - systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) + _getScopesAdminDomainFromGamScopesJson() + if not GC_Values[GC_DOMAIN]: + getDomainFromAdmin() _chkCfgDirectories() _chkCfgFiles() if GC_Values[GC_NO_CACHE]: @@ -595,6 +608,16 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) + +def handleOAuthTokenError(e, soft_errors): + if e.message in OAUTH_TOKEN_ERRORS: + if soft_errors: + return None + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], + u','.join(GM_Globals[GM_CURRENT_API_SCOPES]), GC_Values[GC_ADMIN]))) + systemErrorExit(12, MESSAGE_API_ACCESS_CONFIG) + systemErrorExit(18, u'Authentication Token Error - {0}'.format(e)) + def tryOAuth(gdataObject, soft_errors=False): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], @@ -603,21 +626,11 @@ def tryOAuth(gdataObject, soft_errors=False): cache=GC_Values[GC_CACHE_DIR]) try: credentials.refresh(http) + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, e) except oauth2client.client.AccessTokenRefreshError, e: - if e.message in [u'access_denied', - u'unauthorized_client: Unauthorized client or scope in request.', - u'access_denied: Requested client not authorized.']: - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_CURRENT_API_SCOPES])))) - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_CONFIG) - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) - if soft_errors: - return False - sys.exit(4) + return handleOAuthTokenError(e, soft_errors) gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} - if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][GC_Values[GC_ADMIN].find(u'@')+1:].lower() - if not GC_Values[GC_CUSTOMER_ID]: - GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER gdataObject.domain = GC_Values[GC_DOMAIN] return True @@ -699,6 +712,8 @@ def callGData(service, function, soft_errors=False, throw_errors=[], **kwargs): sys.stderr.write(u' - Giving up.\n') return None sys.exit(int(e.error_code)) + except oauth2client.client.AccessTokenRefreshError as e: + return handleOAuthTokenError(e, soft_errors) def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_reasons=[], retry_reasons=[], **kwargs): method = getattr(service, function) @@ -747,8 +762,7 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re return None sys.exit(int(http_status)) except oauth2client.client.AccessTokenRefreshError, e: - sys.stderr.write(u'{0}Authentication Token Error: {1}\n'.format(ERROR_PREFIX, e)) - sys.exit(403) + return handleOAuthTokenError(e, False) except httplib2.CertificateValidationUnsupported: noPythonSSLExit() except TypeError, e: @@ -810,23 +824,11 @@ API_VER_MAPPING = { u'siteVerification': u'v1', } -def getAPIVer(api): - return API_VER_MAPPING.get(api, u'v1') - -def getServiceFromDiscoveryDocument(api, version, http=None): - disc_filename = u'%s-%s.json' % (api, version) - disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) - if hasattr(sys, '_MEIPASS'): - pyinstaller_disc_file = os.path.join(sys._MEIPASS, disc_filename) - else: - pyinstaller_disc_file = None - if os.path.isfile(disc_file): - discovery = readFile(disc_file) - elif pyinstaller_disc_file: - discovery = readFile(pyinstaller_disc_file) - else: - systemErrorExit(4, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) - return googleapiclient.discovery.build_from_document(discovery, base=u'https://www.googleapis.com', http=http) +def getAPIVersion(api): + version = API_VER_MAPPING.get(api, u'v1') + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + return (api, version, u'{0}-{1}'.format(api, version)) def getOAuth2ServiceDetails(): if not GM_Globals[GM_OAUTH2SERVICE_KEY]: @@ -845,38 +847,46 @@ def getOAuth2ServiceDetails(): printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) -def buildGAPIObject(api, act_as=None, soft_errors=False): - svcsub = act_as if act_as else GC_Values[GC_ADMIN] +def getAPIversionHttpService(api): getOAuth2ServiceDetails() - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' + api, version, api_version = getAPIVersion(api) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) try: service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) + return (api_version, http, service) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), []) - if not scopes: + except googleapiclient.errors.UnknownApiNameOrVersion: + pass + disc_filename = u'%s.json' % (api_version) + disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) + if hasattr(sys, '_MEIPASS'): + pyinstaller_disc_file = os.path.join(sys._MEIPASS, disc_filename) + else: + pyinstaller_disc_file = None + if os.path.isfile(disc_file): + discovery = readFile(disc_file) + elif pyinstaller_disc_file: + discovery = readFile(pyinstaller_disc_file) + else: + systemErrorExit(11, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) + service = googleapiclient.discovery.build_from_document(discovery, http=http) + return (api_version, http, service) + +def buildGAPIObject(api, act_as=None, soft_errors=False): + svcsub = act_as if act_as else GC_Values[GC_ADMIN] + api_version, http, service = getAPIversionHttpService(api) + GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) + if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scopes, user_agent=GAM_INFO, sub=svcsub) + scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=svcsub) try: service._http = credentials.authorize(http) except oauth2client.client.AccessTokenRefreshError, e: - if e.message in [u'access_denied', - u'unauthorized_client: Unauthorized client or scope in request.', - u'access_denied: Requested client not authorized.']: - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(scopes)))) - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_CONFIG) - if soft_errors: - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) - return False - systemErrorExit(4, e) + return handleOAuthTokenError(e, soft_errors) return service GDATA_API_INFO = { @@ -887,12 +897,11 @@ GDATA_API_INFO = { def commonAppsObjInit(appsObj, api): getOAuth2ServiceDetails() - GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, getAPIVer(api)), []) + api, _, api_version = getAPIVersion(api) + GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(GDATA_API_INFO[api])) - if not tryOAuth(appsObj): - doRequestOAuth() - tryOAuth(appsObj) + tryOAuth(appsObj) #Identify GAM to Google's Servers appsObj.source = GAM_INFO #Show debugging output if debug.gam exists @@ -8720,35 +8729,14 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa full_users = new_full_users return full_users -def validateSetGAMScopes(json_data): - GM_Globals[GM_GAMSCOPES_BY_API] = {} - if not isinstance(json_data, dict): - return False - for api, value in json_data.items(): - if not isinstance(value, list): - return False - GM_Globals[GM_GAMSCOPES_BY_API][api] = list(set(value)) - return len(GM_Globals[GM_GAMSCOPES_BY_API]) > 0 - def OAuthInfo(): configRequired = False - getOAuth2ServiceDetails() - print u'API Access' + print u'API Access, Admin: {0}'.format(GC_Values[GC_ADMIN]) for api in sorted(API_VER_MAPPING.keys()): print u' API: {0}'.format(api) - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) - try: - service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) + api_version, http, service = getAPIversionHttpService(api) api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] - requested_scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(u'{0}-{1}'.format(api, version), []) + requested_scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) if requested_scopes: for scope in requested_scopes: credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], @@ -8759,17 +8747,15 @@ def OAuthInfo(): status = u'Authorized' except oauth2client.client.AccessTokenRefreshError, e: configRequired = True - if e.message in [u'access_denied', - u'unauthorized_client: Unauthorized client or scope in request.', - u'access_denied: Requested client not authorized.']: + if e.message in OAUTH_TOKEN_ERRORS: status = u'DENIED' else: - status = u'{0}{1}'.format(ERROR_PREFIX, e) + status = u'{0}Authentication Token Error - {1}'.format(ERROR_PREFIX, e) print u' {0}\n {1}\n Access: {2}'.format(api_scopes[scope][u'description'], scope, status) else: print u' Access: Not requested' if configRequired: - print MESSAGE_CLIENT_API_ACCESS_CONFIG + print MESSAGE_API_ACCESS_CONFIG def doDeleteOAuth(): sys.stdout.write(u'Scopes file: {0}, will be Deleted in 3...'.format(GC_Values[GC_GAMSCOPES_JSON])) @@ -8789,69 +8775,87 @@ def doDeleteOAuth(): except OSError as e: sys.stderr.write(u'{0}{1}\n'.format(WARNING_PREFIX, e)) -UBER_SCOPES = { - u'gmail-v1': [u'https://mail.google.com/'], - } +EMAIL_PATTERN = re.compile(r'^(.+)@(.+\..+)$') +EMAIL_FORMAT_REQUIRED = u'@.' -def select_default_scopes(apis): - for api_name, api in apis.items(): - if api_name in UBER_SCOPES: - api[u'use_scopes'] = UBER_SCOPES[api_name] - else: - scopes = api[u'auth'][u'oauth2'][u'scopes'].keys() - scopes.sort() - api[u'use_scopes'] = [] - # reduce # of scopes by checking if a scope is a substring of another - # which should mean it covers same API operations. Add a . at end - # to prevent things like directory.users removing directory.userschema - i = 0 - count = len(scopes) - while i < count: - scope = scopes[i] - api[u'use_scopes'].append(scope) - i += 1 - scope += u'.' - while (i < count) and scopes[i].startswith(scope): - i += 1 - -def getSelection(limit): - while True: - selection = raw_input(u'Your selection: ') - if selection: - if selection.isdigit(): - selection = int(selection) - if (selection >= 0) and (selection <= limit): - return selection - print u'ERROR: enter number in range 0-{0}'.format(limit) - else: - print u'ERROR: please enter numbers only' +UBER_SCOPES = {u'gmail-v1': [u'https://mail.google.com/'],} def doRequestOAuth(): + + def _getAdminDomain(): + srcFile = os.path.expanduser(os.environ.get(u'OAUTHFILE', u'oauth2.txt')) + if not os.path.isabs(srcFile): + srcFile = os.path.expanduser(os.path.join(GC_Values[GC_CONFIG_DIR], srcFile)) + if os.path.isfile(srcFile): + json_string = readFile(srcFile, continueOnError=True, displayError=False) + if json_string: + json_data = json.loads(json_string) + GC_Values[GC_ADMIN] = json_data.get(u'id_token', {}).get(u'email', GC_Defaults[GC_ADMIN]) + if not GC_Values[GC_DOMAIN]: + GC_Values[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', GC_Defaults[GC_DOMAIN]) + if GC_Values[GC_ADMIN]: + return + print u'' + while True: + value = raw_input(u'Enter Admin email address: ').strip().lower() + ema = EMAIL_PATTERN.match(value) + if ema: + GC_Values[GC_ADMIN] = value + if not GC_Values[GC_DOMAIN]: + GC_Values[GC_DOMAIN] = ema.group(2) + return + print u'{0}Enter full email address: {1}'.format(ERROR_PREFIX, EMAIL_FORMAT_REQUIRED) + + def _select_default_scopes(apis): + for api_name, api in apis.items(): + if api_name in UBER_SCOPES: + api[u'use_scopes'] = UBER_SCOPES[api_name] + else: + scopes = sorted(api[u'auth'][u'oauth2'][u'scopes'].keys()) + api[u'use_scopes'] = [] + # reduce # of scopes by checking if a scope is a substring of another + # which should mean it covers same API operations. Add a . at end + # to prevent things like directory.users removing directory.userschema + i = 0 + count = len(scopes) + while i < count: + scope = scopes[i] + api[u'use_scopes'].append(scope) + i += 1 + scope += u'.' + while (i < count) and scopes[i].startswith(scope): + i += 1 + + def _getSelection(limit): + while True: + selection = raw_input(u'Your selection: ') + if selection: + if selection.isdigit(): + selection = int(selection) + if (selection >= 0) and (selection <= limit): + return selection + print u'ERROR: enter number in range 0-{0}'.format(limit) + else: + print u'ERROR: please enter numbers only' + apis = API_VER_MAPPING.keys() all_apis = {} api_titles = {} for api in apis: - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) - try: - service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) - except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) - api_name = u'%s-%s' % (api, version) - all_apis[api_name] = service._rootDesc - api_titles[api_name] = api_name + api_version, _, service = getAPIversionHttpService(api) + all_apis[api_version] = service._rootDesc + api_titles[api_version] = api_version api_index = [] - for _, api_name in sorted(api_titles.items()): - api_index.append(api_name) + for _, api_version in sorted(api_titles.items()): + api_index.append(api_version) i = len(api_index) if GM_Globals[GM_GAMSCOPES_BY_API]: for api in all_apis: all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api] else: - select_default_scopes(all_apis) + _select_default_scopes(all_apis) + if not GC_Values[GC_ADMIN]: + _getAdminDomain() while True: #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) print u'Select the APIs to use with GAM.' @@ -8872,9 +8876,9 @@ def doRequestOAuth(): print u' %2d) Cancel' % (i+2) print u' %2d) Continue' % (i+3) print - selection = getSelection(i+3) + selection = _getSelection(i+3) if selection == i: # defaults - select_default_scopes(all_apis) + _select_default_scopes(all_apis) elif selection == i+1: # unselect all for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] @@ -8888,7 +8892,9 @@ def doRequestOAuth(): if not selected_scopes: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(GM_Globals[GM_GAMSCOPES_BY_API])) + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps({u'scopes': GM_Globals[GM_GAMSCOPES_BY_API], + u'admin': GC_Values[GC_ADMIN], + u'domain': GC_Values[GC_DOMAIN]})) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(selected_scopes), u','.join(selected_scopes)) return @@ -8921,7 +8927,7 @@ def doRequestOAuth(): print u' %2d) Cancel' % (x+3) print u' %2d) Back to all APIs' % (x+4) print - selection = getSelection(x+4) + selection = _getSelection(x+4) if selection < x: # select if api_scopes[selection] in all_apis[api][u'use_scopes']: all_apis[api][u'use_scopes'].remove(api_scopes[selection]) @@ -8929,7 +8935,7 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'].append(api_scopes[selection]) elif selection == x: # defaults just_this_api = {api: all_apis[api]} - select_default_scopes(just_this_api) + _select_default_scopes(just_this_api) all_apis[api][u'use_scopes'] = just_this_api[api][u'use_scopes'] elif selection == x+1: # read-only all_apis[api][u'use_scopes'] = [] From aa04e3ec1d2b6b51b0191b1fc54298d7f01fce8f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 11 Jan 2016 08:35:01 -0800 Subject: [PATCH 079/100] Fix select_default_scopes in doRequestOAuth https://www.googleapis.com/auth/admin.directory.user.security is not a subset of https://www.googleapis.com/auth/admin.directory.user --- src/gam.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gam.py b/src/gam.py index ecffe2df..4ffe8e1f 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8824,6 +8824,8 @@ def doRequestOAuth(): i += 1 scope += u'.' while (i < count) and scopes[i].startswith(scope): + if scopes[i].endswith(u'.security'): + api[u'use_scopes'].append(scopes[i]) i += 1 def _getSelection(limit): From eb5cfde6305603c0b6758401b5f9ccfd0567372f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 11 Jan 2016 10:06:53 -0800 Subject: [PATCH 080/100] Handle JSON format errors --- src/gam.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/gam.py b/src/gam.py index 4ffe8e1f..eb7c17e3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -240,11 +240,11 @@ GC_VAR_INFO = { MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' MESSAGE_API_ACCESS_CONFIG = u'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access' MESSAGE_API_ACCESS_DENIED = u'API access denied.\n\nPlease make sure the Service account Client ID: {0} is authorized for the API Scope(s): {1}\n\nPlease make sure the Admin email address: {2} is valid' -MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV headers of "{1}".' MESSAGE_HIT_CONTROL_C_TO_UPDATE = u'\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.' +MESSAGE_INVALID_JSON = u'The file {0} has an invalid format.' MESSAGE_NO_DISCOVERY_INFORMATION = u'No online discovery doc and {0} does not exist locally' MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for {0}; please run gam oauth create' @@ -357,8 +357,8 @@ def systemErrorExit(sysRC, message): sys.stderr.write(u'\n{0}{1}\n'.format(ERROR_PREFIX, message)) sys.exit(sysRC) -def badGAMScopesExit(): - systemErrorExit(19, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) +def invalidJSONExit(fileName): + systemErrorExit(17, MESSAGE_INVALID_JSON.format(fileName)) def noPythonSSLExit(): systemErrorExit(8, MESSAGE_NO_PYTHON_SSL) @@ -458,15 +458,16 @@ def SetGlobalVariables(): json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) if json_string == None: return - json_data = json.loads(json_string) - if not isinstance(json_data, dict): - badGAMScopesExit() + try: + json_data = json.loads(json_string) + except ValueError: + invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) scopes = json_data.get(u'scopes', None) if (not scopes) or (not isinstance(scopes, dict)): - badGAMScopesExit() + invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) for api, value in scopes.items(): if not isinstance(value, list): - badGAMScopesExit() + invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) GM_Globals[GM_GAMSCOPES_BY_API][api] = list(set(value)) if not GC_Values[GC_ADMIN]: GC_Values[GC_ADMIN] = json_data.get(u'admin', GC_Defaults[GC_ADMIN]) @@ -837,15 +838,15 @@ def getOAuth2ServiceDetails(): printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) systemErrorExit(6, None) - json_data = json.loads(json_string) try: + json_data = json.loads(json_string) GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = json_data[u'client_email'] GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = json_data[u'client_id'] GM_Globals[GM_OAUTH2SERVICE_KEY] = json_data[u'private_key'] - except KeyError: + except (ValueError, KeyError): printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) - systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) + invalidJSONExit(GC_Values[GC_OAUTH2SERVICE_JSON]) def getAPIversionHttpService(api): getOAuth2ServiceDetails() @@ -871,8 +872,11 @@ def getAPIversionHttpService(api): discovery = readFile(pyinstaller_disc_file) else: systemErrorExit(11, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) - service = googleapiclient.discovery.build_from_document(discovery, http=http) - return (api_version, http, service) + try: + service = googleapiclient.discovery.build_from_document(discovery, http=http) + return (api_version, http, service) + except (ValueError, KeyError): + invalidJSONExit(disc_file) def buildGAPIObject(api, act_as=None, soft_errors=False): svcsub = act_as if act_as else GC_Values[GC_ADMIN] @@ -8789,10 +8793,13 @@ def doRequestOAuth(): if os.path.isfile(srcFile): json_string = readFile(srcFile, continueOnError=True, displayError=False) if json_string: - json_data = json.loads(json_string) - GC_Values[GC_ADMIN] = json_data.get(u'id_token', {}).get(u'email', GC_Defaults[GC_ADMIN]) - if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', GC_Defaults[GC_DOMAIN]) + try: + json_data = json.loads(json_string) + GC_Values[GC_ADMIN] = json_data.get(u'id_token', {}).get(u'email', GC_Defaults[GC_ADMIN]) + if not GC_Values[GC_DOMAIN]: + GC_Values[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', GC_Defaults[GC_DOMAIN]) + except ValueError: + pass if GC_Values[GC_ADMIN]: return print u'' From 0d94dd3fa51070a2d2f3649f3e709f7ec8d87d41 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 11 Jan 2016 22:21:01 -0800 Subject: [PATCH 081/100] Handle out of domain users better --- src/gam.py | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/gam.py b/src/gam.py index eb7c17e3..29427106 100755 --- a/src/gam.py +++ b/src/gam.py @@ -92,8 +92,10 @@ GM_BATCH_QUEUE = u'batq' GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json GM_GAMSCOPES_BY_API = u'scop' +# Current API user +GM_CURRENT_API_USER = u'capu' # Current API scope -GM_CURRENT_API_SCOPES = u'scoc' +GM_CURRENT_API_SCOPES = u'caps' # Values retrieved from oauth2service.json GM_OAUTH2SERVICE_KEY = u'oauk' GM_OAUTH2SERVICE_ACCOUNT_EMAIL = u'oaae' @@ -117,6 +119,7 @@ GM_Globals = { GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, GM_GAMSCOPES_BY_API: {}, + GM_CURRENT_API_USER: None, GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, @@ -254,6 +257,7 @@ MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service accou MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' +MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service account.' OAUTH_TOKEN_ERRORS = [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.', u'invalid_grant: Not a valid email.', u'invalid_request: Invalid impersonation prn email address.'] @@ -609,17 +613,20 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) - def handleOAuthTokenError(e, soft_errors): if e.message in OAUTH_TOKEN_ERRORS: - if soft_errors: - return None - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], - u','.join(GM_Globals[GM_CURRENT_API_SCOPES]), GC_Values[GC_ADMIN]))) - systemErrorExit(12, MESSAGE_API_ACCESS_CONFIG) + if not GM_Globals[GM_CURRENT_API_USER]: + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], + u','.join(GM_Globals[GM_CURRENT_API_SCOPES]), GC_Values[GC_ADMIN]))) + systemErrorExit(12, MESSAGE_API_ACCESS_CONFIG) + else: + systemErrorExit(19, MESSAGE_SERVICE_NOT_APPLICABLE.format(GM_Globals[GM_CURRENT_API_USER])) + if soft_errors: + sys.stderr.write(u'{0}Authentication Token Error - {1}\n'.format(ERROR_PREFIX, e)) + return None systemErrorExit(18, u'Authentication Token Error - {0}'.format(e)) -def tryOAuth(gdataObject, soft_errors=False): +def getGDataOAuthToken(gdataObject): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) @@ -630,17 +637,14 @@ def tryOAuth(gdataObject, soft_errors=False): except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) except oauth2client.client.AccessTokenRefreshError, e: - return handleOAuthTokenError(e, soft_errors) + return handleOAuthTokenError(e, False) gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} - gdataObject.domain = GC_Values[GC_DOMAIN] return True def checkGDataError(e, service): # First check for errors that need special handling if e[0].get(u'reason', u'') in [u'Token invalid - Invalid token: Stateless token expired', u'Token invalid - Invalid token: Token not found']: - keep_domain = service.domain - tryOAuth(service) - service.domain = keep_domain + getGDataOAuthToken(service) return False if e[0][u'body'].startswith(u'Required field must not be blank:') or e[0][u'body'].startswith(u'These characters are not allowed:'): return e[0]['body'] @@ -763,7 +767,7 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re return None sys.exit(int(http_status)) except oauth2client.client.AccessTokenRefreshError, e: - return handleOAuthTokenError(e, False) + return handleOAuthTokenError(e, soft_errors) except httplib2.CertificateValidationUnsupported: noPythonSSLExit() except TypeError, e: @@ -881,6 +885,7 @@ def getAPIversionHttpService(api): def buildGAPIObject(api, act_as=None, soft_errors=False): svcsub = act_as if act_as else GC_Values[GC_ADMIN] api_version, http, service = getAPIversionHttpService(api) + GM_Globals[GM_CURRENT_API_USER] = act_as GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) @@ -889,6 +894,8 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=svcsub) try: service._http = credentials.authorize(http) + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, e) except oauth2client.client.AccessTokenRefreshError, e: return handleOAuthTokenError(e, soft_errors) return service @@ -899,31 +906,33 @@ GDATA_API_INFO = { u'email-settings': u'Email Settings API', } -def commonAppsObjInit(appsObj, api): +def initGDataObject(gdataObj, api): getOAuth2ServiceDetails() api, _, api_version = getAPIVersion(api) + GM_Globals[GM_CURRENT_API_USER] = None GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(GDATA_API_INFO[api])) - tryOAuth(appsObj) + getGDataOAuthToken(gdataObj) + gdataObj.domain = GC_Values[GC_DOMAIN] #Identify GAM to Google's Servers - appsObj.source = GAM_INFO + gdataObj.source = GAM_INFO #Show debugging output if debug.gam exists if GC_Values[GC_DEBUG_LEVEL] > 0: - appsObj.debug = True - return appsObj + gdataObj.debug = True + return gdataObj def getAdminSettingsObject(): import gdata.apps.adminsettings.service - return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), u'admin-settings') + return initGDataObject(gdata.apps.adminsettings.service.AdminSettingsService(), u'admin-settings') def getAuditObject(): import gdata.apps.audit.service - return commonAppsObjInit(gdata.apps.audit.service.AuditService(), u'email-audit') + return initGDataObject(gdata.apps.audit.service.AuditService(), u'email-audit') def getEmailSettingsObject(): import gdata.apps.emailsettings.service - return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), u'email-settings') + return initGDataObject(gdata.apps.emailsettings.service.EmailSettingsService(), u'email-settings') def geturl(url, dst): import urllib2 From 671f7d810cb81b469e69da2c1f9d3d6a440d3d34 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 11 Jan 2016 23:14:24 -0800 Subject: [PATCH 082/100] buildOrgUnitIdToNameMap only mapped top level org units --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 29427106..441eea5d 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1688,7 +1688,7 @@ def buildOrgUnitIdToNameMap(): cd = buildGAPIObject(u'directory') result = callGAPI(cd.orgunits(), u'list', customerId=GC_Values[GC_CUSTOMER_ID], - fields=u'organizationUnits(orgUnitPath,orgUnitId)') + fields=u'organizationUnits(orgUnitPath,orgUnitId)', type=u'all') GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME] = {} for orgUnit in result[u'organizationUnits']: GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][orgUnit[u'orgUnitId']] = orgUnit[u'orgUnitPath'] From 16d8cddf12a7baed4dbc0314b5930e4c644a839a Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 12 Jan 2016 07:27:31 -0800 Subject: [PATCH 083/100] In gam print admins, include id: with orgUnitId in CSV file --- src/gam.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 441eea5d..753d820d 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1665,7 +1665,6 @@ def doPrintAdmins(): continue if key not in admins_attrib[0]: admins_attrib[0][key] = key - admin_attrib[key] = value if key == u'assignedTo': assignedToUser = user_from_userid(value) if u'assignedToUser' not in admins_attrib[0]: @@ -1677,10 +1676,12 @@ def doPrintAdmins(): admins_attrib[0][u'role'] = u'role' admin_attrib[u'role'] = role elif key == u'orgUnitId': + value = u'id:%s' % value orgUnit = orgunit_from_orgunitid(value) if u'orgUnit' not in admins_attrib[0]: admins_attrib[0][u'orgUnit'] = u'orgUnit' admin_attrib[u'orgUnit'] = orgUnit + admin_attrib[key] = value admins_attrib.append(admin_attrib) output_csv(admins_attrib, admins_attrib[0], u'Admins', todrive) @@ -1696,7 +1697,7 @@ def buildOrgUnitIdToNameMap(): def orgunit_from_orgunitid(orgunitid): if not GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME]: buildOrgUnitIdToNameMap() - return GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][u'id:%s' % orgunitid] + return GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][orgunitid] def buildRoleIdToNameToIdMap(): cd = buildGAPIObject(u'directory') From f3b970ae14b592cc7ae7020618cec9c52aed4417 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 12 Jan 2016 15:34:25 -0800 Subject: [PATCH 084/100] getGDataOAuthToken (formerly tryOAuth) was wiping out additional_headers --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 753d820d..b5ad6633 100755 --- a/src/gam.py +++ b/src/gam.py @@ -638,7 +638,7 @@ def getGDataOAuthToken(gdataObject): systemErrorExit(4, e) except oauth2client.client.AccessTokenRefreshError, e: return handleOAuthTokenError(e, False) - gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} + gdataObject.additional_headers[u'Authorization'] = u'Bearer {0}'.format(credentials.access_token) return True def checkGDataError(e, service): From d07ab2d7e10a226fafae2b9359f16e43298d4566 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 12 Jan 2016 19:33:11 -0800 Subject: [PATCH 085/100] Clean up string quotes --- src/gam.py | 374 ++++++++++++++++++++++++++--------------------------- 1 file changed, 187 insertions(+), 187 deletions(-) diff --git a/src/gam.py b/src/gam.py index b5ad6633..92f2582f 100755 --- a/src/gam.py +++ b/src/gam.py @@ -257,7 +257,7 @@ MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service accou MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' -MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}' +MESSAGE_SERVICE_NOT_APPLICABLE = u'Service not applicable for this address: {0}' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service account.' OAUTH_TOKEN_ERRORS = [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.', u'invalid_grant: Not a valid email.', u'invalid_request: Invalid impersonation prn email address.'] @@ -309,18 +309,18 @@ class _DeHTMLParser(HTMLParser): def handle_data(self, data): text = data.strip() if len(text) > 0: - text = sub('[ \t\r\n]+', ' ', text) - self.__text.append(text + ' ') + text = sub(u'[ \t\r\n]+', u' ', text) + self.__text.append(text + u' ') def handle_starttag(self, tag, attrs): - if tag == 'p': - self.__text.append('\n\n') - elif tag == 'br': - self.__text.append('\n') + if tag == u'p': + self.__text.append(u'\n\n') + elif tag == u'br': + self.__text.append(u'\n') def handle_startendtag(self, tag, attrs): - if tag == 'br': - self.__text.append('\n\n') + if tag == u'br': + self.__text.append(u'\n\n') def text(self): return ''.join(self.__text).strip() @@ -329,7 +329,7 @@ class _DeHTMLParser(HTMLParser): def dehtml(text): try: parser = _DeHTMLParser() - parser.feed(text.encode('utf-8')) + parser.feed(text.encode(u'utf-8')) parser.close() return parser.text() except: @@ -608,7 +608,7 @@ def doGAMVersion(): print u'GAM {0} - {1}\n{2}\nPython {3}.{4}.{5} {6}-bit {7}\ngoogle-api-python-client {8}\n{9} {10}\nPath: {11}'.format(__version__, GAM_URL, __author__, sys.version_info[0], sys.version_info[1], sys.version_info[2], - struct.calcsize('P')*8, sys.version_info[3], + struct.calcsize(u'P')*8, sys.version_info[3], googleapiclient.__version__, platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) @@ -647,7 +647,7 @@ def checkGDataError(e, service): getGDataOAuthToken(service) return False if e[0][u'body'].startswith(u'Required field must not be blank:') or e[0][u'body'].startswith(u'These characters are not allowed:'): - return e[0]['body'] + return e[0][u'body'] if e.error_code == 600 and e[0][u'body'] == u'Quota exceeded for the current request' or e[0][u'reason'] == u'Bad Gateway': return False if e.error_code == 600 and e[0][u'reason'] == u'Token invalid - Invalid token: Token disabled, revoked, or expired.': @@ -800,9 +800,9 @@ def callGAPIpages(service, function, items, page_message=None, message_attribute show_message = show_message.replace(u'%%first_item%%', str(this_page[items][0][message_attribute])) show_message = show_message.replace(u'%%last_item%%', str(this_page[items][-1][message_attribute])) except (IndexError, KeyError): - show_message = show_message.replace(u'%%first_item%%', '') - show_message = show_message.replace(u'%%last_item%%', '') - sys.stderr.write('\r') + show_message = show_message.replace(u'%%first_item%%', u'') + show_message = show_message.replace(u'%%last_item%%', u'') + sys.stderr.write(u'\r') sys.stderr.flush() sys.stderr.write(show_message) if not pageToken: @@ -866,7 +866,7 @@ def getAPIversionHttpService(api): pass disc_filename = u'%s.json' % (api_version) disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) - if hasattr(sys, '_MEIPASS'): + if hasattr(sys, u'_MEIPASS'): pyinstaller_disc_file = os.path.join(sys._MEIPASS, disc_filename) else: pyinstaller_disc_file = None @@ -967,7 +967,7 @@ def showReport(): customerId = None date = filters = parameters = actorIpAddress = startTime = endTime = eventName = None to_drive = False - userKey = 'all' + userKey = u'all' i = 3 while i < len(sys.argv): if sys.argv[i].lower() == u'date': @@ -1073,9 +1073,9 @@ def showReport(): app = dict() for an_item in subitem: if an_item == u'client_name': - app['name'] = u'App: %s' % subitem[an_item] + app[u'name'] = u'App: %s' % subitem[an_item] elif an_item == u'num_users': - app['value'] = u'%s users' % subitem[an_item] + app[u'value'] = u'%s users' % subitem[an_item] elif an_item == u'client_id': app[u'client_id'] = subitem[an_item] auth_apps.append(app) @@ -1133,15 +1133,15 @@ def doDelegates(users): delegate_domain = delegate[delegate.find(u'@')+1:].lower() delegate_email = delegate else: - print 'ERROR: %s is not a valid argument for "gam delegate", expected to' % sys.argv[4] + print u'ERROR: %s is not a valid argument for "gam delegate", expected to' % sys.argv[4] sys.exit(2) count = len(users) i = 1 for delegator in users: if delegator.find(u'@') > 0: - delegator_domain = delegator[delegator.find('@')+1:].lower() + delegator_domain = delegator[delegator.find(u'@')+1:].lower() delegator_email = delegator - delegator = delegator[:delegator.find('@')] + delegator = delegator[:delegator.find(u'@')] else: delegator_domain = GC_Values[GC_DOMAIN].lower() delegator_email = u'%s@%s' % (delegator, delegator_domain) @@ -1168,7 +1168,7 @@ def doDelegates(users): pass if not found_alias_in_delegator_domain: delete_alias = True - use_delegate_address = u'%s@%s' % (''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz0123456789', 25)), delegator_domain) + use_delegate_address = u'%s@%s' % (u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz0123456789', 25)), delegator_domain) print u' Giving %s temporary alias %s for delegation' % (delegate_email, use_delegate_address) callGAPI(cd.users().aliases(), u'insert', userKey=delegate_email, body={u'alias': use_delegate_address}) time.sleep(5) @@ -1245,11 +1245,11 @@ def getDelegates(users): sys.exit(2) for user in users: if user.find(u'@') > 0: - emailsettings.domain = user[user.find('@')+1:] - user = user[:user.find('@')] + emailsettings.domain = user[user.find(u'@')+1:] + user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] - sys.stderr.write(u"Getting delegates for %s...\n" % (user + '@' + emailsettings.domain)) + sys.stderr.write(u"Getting delegates for %s...\n" % (user + u'@' + emailsettings.domain)) delegates = callGData(emailsettings, u'GetDelegates', soft_errors=True, delegator=user) try: for delegate in delegates: @@ -1396,7 +1396,7 @@ def doUpdateCourse(): elif sys.argv[i].lower() in [u'state', u'status']: body[u'courseState'] = sys.argv[i+1].upper() if body[u'courseState'] not in [u'ACTIVE', u'ARCHIVED', u'PROVISIONED', u'DECLINED']: - print 'ERROR: course state can be active or archived. Got %s' % body[u'courseState'] + print u'ERROR: course state can be active or archived. Got %s' % body[u'courseState'] sys.exit(2) i += 2 else: @@ -1827,10 +1827,10 @@ def doPrintDataTransfers(): status = None todrive = False while i < len(sys.argv): - if sys.argv[i].lower().replace(u'_', '') in [u'olduser', u'oldowner']: + if sys.argv[i].lower().replace(u'_', u'') in [u'olduser', u'oldowner']: oldOwnerUserId = convertToUserID(sys.argv[i+1]) i += 2 - elif sys.argv[i].lower().replace(u'_', '') in [u'newuser', u'newowner']: + elif sys.argv[i].lower().replace(u'_', u'') in [u'newuser', u'newowner']: newOwnerUserId = convertToUserID(sys.argv[i+1]) i += 2 elif sys.argv[i].lower() == u'status': @@ -1858,7 +1858,7 @@ def doPrintDataTransfers(): a_transfer[u'id'] = transfer[u'id'] if u'applicationTransferParams' in transfer[u'applicationDataTransfers'][i]: for param in transfer[u'applicationDataTransfers'][i][u'applicationTransferParams']: - a_transfer[param[u'key']] = ','.join(param[u'value']) + a_transfer[param[u'key']] = u','.join(param[u'value']) for title in a_transfer: if title not in transfers_attributes[0]: transfers_attributes[0][title] = title @@ -1912,16 +1912,16 @@ def doCreateCourse(): elif sys.argv[i].lower() in [u'state', u'status']: body[u'courseState'] = sys.argv[i+1].upper() if body[u'courseState'] not in [u'ACTIVE', u'ARCHIVED', u'PROVISIONED', u'DECLINED']: - print 'ERROR: course state can be active or archived. Got %s' % body[u'courseState'] + print u'ERROR: course state can be active or archived. Got %s' % body[u'courseState'] sys.exit(2) i += 2 else: print u'ERROR: %s is not a valid argument for "gam create course".' % sys.argv[i] sys.exit(2) if not u'ownerId' in body: - body['ownerId'] = u'me' + body[u'ownerId'] = u'me' if not u'name' in body: - body['name'] = u'Unknown Course' + body[u'name'] = u'Unknown Course' result = callGAPI(croom.courses(), u'create', body=body) print u'Created course %s' % result[u'id'] @@ -1996,7 +1996,7 @@ def doPrintCourses(): num_courses = len(croom_attributes[1:]) i = 1 for course in croom_attributes[1:]: - sys.stderr.write('Getting aliases for course %s (%s/%s)\n' % (course[u'id'], i, num_courses)) + sys.stderr.write(u'Getting aliases for course %s (%s/%s)\n' % (course[u'id'], i, num_courses)) course_aliases = callGAPIpages(croom.courses().aliases(), u'list', u'aliases', courseId=course[u'id']) my_aliases = [] for alias in course_aliases: @@ -2138,7 +2138,7 @@ def doPrintPrintJobs(): elif sys.argv[i].lower() == u'orderby': sortorder = sys.argv[i+1].lower().replace(u'_', u'') if sortorder not in PRINTJOB_ASCENDINGORDER_MAP: - print u'ERROR: orderby must be one of %s. Got %s' % (','.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder) + print u'ERROR: orderby must be one of %s. Got %s' % (u','.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder) sys.exit(2) sortorder = PRINTJOB_ASCENDINGORDER_MAP[sortorder] i += 2 @@ -2259,7 +2259,7 @@ def changeCalendarAttendees(users): print u'Got %s items' % len(events_page.get(u'items', [])) for event in events_page.get(u'items', []): if event[u'status'] == u'cancelled': - #print ' skipping cancelled event' + #print u' skipping cancelled event' continue try: event_summary = convertUTF8(event[u'summary']) @@ -2267,7 +2267,7 @@ def changeCalendarAttendees(users): event_summary = event[u'id'] try: if not allevents and event[u'organizer'][u'email'].lower() != user: - #print ' skipping not-my-event %s' % event_summary + #print u' skipping not-my-event %s' % event_summary continue except KeyError: pass # no email for organizer @@ -2295,7 +2295,7 @@ def changeCalendarAttendees(users): else: print u' not pulling the trigger.' #else: - # print ' no update needed for %s' % event_summary + # print u' no update needed for %s' % event_summary try: page_token = events_page[u'nextPageToken'] except KeyError: @@ -2376,7 +2376,7 @@ def addCalendar(users): for user in users: if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) - print u"Subscribing %s to %s calendar (%s of %s)" % (user, body['id'], i, count) + print u"Subscribing %s to %s calendar (%s of %s)" % (user, body[u'id'], i, count) cal = buildGAPIObject(u'calendar', user) callGAPI(cal.calendarList(), u'insert', body=body, colorRgbFormat=colorRgbFormat) i += 1 @@ -2474,8 +2474,8 @@ def doPrinterAddACL(): checkCloudPrintResult(result) who = scope if who == None: - who = 'public' - role = 'user' + who = u'public' + role = u'user' print u'Added %s %s' % (role, who) def doPrinterDelACL(): @@ -2497,36 +2497,36 @@ def doPrinterDelACL(): def encode_multipart(fields, files, boundary=None): def escape_quote(s): - return s.replace('"', '\\"') + return s.replace(u'"', u'\\"') def getFormDataLine(name, value, boundary): - return '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)), '', str(value) + return '--{0}'.format(boundary), u'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)), u'', str(value) if boundary is None: - boundary = ''.join(random.choice(string.digits + string.ascii_letters) for i in range(30)) + boundary = u''.join(random.choice(string.digits + string.ascii_letters) for i in range(30)) lines = [] for name, value in fields.items(): if name == u'tags': for tag in value: - lines.extend(getFormDataLine('tag', tag, boundary)) + lines.extend(getFormDataLine(u'tag', tag, boundary)) else: lines.extend(getFormDataLine(name, value, boundary)) for name, value in files.items(): - filename = value['filename'] - mimetype = value['mimetype'] + filename = value[u'filename'] + mimetype = value[u'mimetype'] lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format( escape_quote(name), escape_quote(filename)), 'Content-Type: {0}'.format(mimetype), '', - value['content'], + value[u'content'], )) lines.extend(( '--{0}--'.format(boundary), '', )) - body = '\r\n'.join(lines) + body = u'\r\n'.join(lines) headers = { 'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary), 'Content-Length': str(len(body)), @@ -2582,7 +2582,7 @@ def doPrintJobFetch(): elif sys.argv[i].lower() == u'orderby': sortorder = sys.argv[i+1].lower().replace(u'_', u'') if sortorder not in PRINTJOB_ASCENDINGORDER_MAP: - print 'ERROR: orderby must be one of %s. Got %s' % (','.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder) + print u'ERROR: orderby must be one of %s. Got %s' % (u','.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder) sys.exit(2) sortorder = PRINTJOB_ASCENDINGORDER_MAP[sortorder] i += 2 @@ -2600,7 +2600,7 @@ def doPrintJobFetch(): sys.exit(0) checkCloudPrintResult(result) valid_chars = u'-_.() abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - ssd = '''{ + ssd = u'''{ "state": {"type": "DONE"} }''' for job in result[u'jobs']: @@ -2615,7 +2615,7 @@ def doPrintJobFetch(): fileName = job[u'title'] fileName = u''.join(c if c in valid_chars else u'_' for c in fileName) fileName = u'%s-%s' % (fileName, jobid) - _, content = cp._http.request(uri=fileUrl, method='GET') + _, content = cp._http.request(uri=fileUrl, method=u'GET') if writeFile(fileName, content, continueOnError=True): # ticket = callGAPI(cp.jobs(), u'getticket', jobid=jobid, use_cjt=True) result = callGAPI(cp.jobs(), u'update', jobid=jobid, semantic_state_diff=ssd) @@ -2713,7 +2713,7 @@ def doPrinterRegister(): body, headers = encode_multipart(form_fields, form_files) #Get the printer first to make sure our OAuth access token is fresh callGAPI(cp.printers(), u'list') - _, result = cp._http.request(uri='https://www.google.com/cloudprint/register', method='POST', body=body, headers=headers) + _, result = cp._http.request(uri=u'https://www.google.com/cloudprint/register', method=u'POST', body=body, headers=headers) result = json.loads(result) checkCloudPrintResult(result) print u'Created printer %s' % result[u'printers'][0][u'id'] @@ -2722,7 +2722,7 @@ def doPrintJobResubmit(): cp = buildGAPIObject(u'cloudprint') jobid = sys.argv[2] printerid = sys.argv[4] - ssd = '''{ + ssd = u'''{ "state": {"type": "HELD"} }''' result = callGAPI(cp.jobs(), u'update', jobid=jobid, semantic_state_diff=ssd) @@ -2767,7 +2767,7 @@ def doPrintJobSubmit(): body, headers = encode_multipart(form_fields, form_files) #Get the printer first to make sure our OAuth access token is fresh callGAPI(cp.printers(), u'get', printerid=printer) - _, result = cp._http.request(uri='https://www.google.com/cloudprint/submit', method='POST', body=body, headers=headers) + _, result = cp._http.request(uri=u'https://www.google.com/cloudprint/submit', method=u'POST', body=body, headers=headers) checkCloudPrintResult(result) if type(result) is str: result = json.loads(result) @@ -2783,7 +2783,7 @@ def doDeletePrintJob(): def doCancelPrintJob(): cp = buildGAPIObject(u'cloudprint') job = sys.argv[2] - ssd = '{"state": {"type": "ABORTED", "user_action_cause": {"action_code": "CANCELLED"}}}' + ssd = u'{"state": {"type": "ABORTED", "user_action_cause": {"action_code": "CANCELLED"}}}' result = callGAPI(cp.jobs(), u'update', jobid=job, semantic_state_diff=ssd) checkCloudPrintResult(result) print u'Print Job %s cancelled' % job @@ -2829,7 +2829,7 @@ def doCalendarAddACL(calendarId=None, act_as=None, role=None, scope=None, entity else: body[u'role'] = sys.argv[4].lower() if body[u'role'] not in [u'freebusy', u'read', u'reader', u'editor', u'owner', u'none']: - print u'ERROR: Role must be freebusy, read, editor, owner or none. Not %s' % body['role'] + print u'ERROR: Role must be freebusy, read, editor, owner or none. Not %s' % body[u'role'] sys.exit(2) if body[u'role'] == u'freebusy': body[u'role'] = u'freeBusyReader' @@ -2879,7 +2879,7 @@ def doCalendarDelACL(): scope = u'domain' elif entity == u'default': scope = u'default' - entity = '' + entity = u'' doCalendarAddACL(calendarId=calendarId, role=u'none', scope=scope, entity=entity) def doCalendarWipeData(): @@ -2953,7 +2953,7 @@ def doCalendarAddEvent(): if sys.argv[i+1].lower() in [u'default', u'public', u'private']: body[u'visibility'] = sys.argv[i+1].lower() else: - print 'ERROR: visibility must be one of default, public or private, got %s' % sys.argv[i+1] + print u'ERROR: visibility must be one of default, public or private, got %s' % sys.argv[i+1] sys.exit(2) i += 2 elif sys.argv[i].lower() == u'tentative': @@ -2984,12 +2984,12 @@ def doCalendarAddEvent(): elif sys.argv[i].lower() == u'privateproperty': if u'extendedProperties' not in body: body[u'extendedProperties'] = {u'private': {}, u'shared': {}} - body[u'extendedProperties']['private'][sys.argv[i+1]] = sys.argv[i+2] + body[u'extendedProperties'][u'private'][sys.argv[i+1]] = sys.argv[i+2] i += 3 elif sys.argv[i].lower() == u'sharedproperty': if u'extendedProperties' not in body: body[u'extendedProperties'] = {u'private': {}, u'shared': {}} - body[u'extendedProperties']['shared'][sys.argv[i+1]] = sys.argv[i+2] + body[u'extendedProperties'][u'shared'][sys.argv[i+1]] = sys.argv[i+2] i += 3 elif sys.argv[i].lower() == u'colorindex': body[u'colorId'] = str(sys.argv[i+1]) @@ -3052,7 +3052,7 @@ def doPhoto(users): for user in users: if user[:4].lower() == u'uid:': user = user[4:] - elif user.find('@') == -1: + elif user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) filename = sys.argv[5].replace(u'#user#', user) filename = filename.replace(u'#email#', user) @@ -3111,10 +3111,10 @@ def deletePhoto(users): for user in users: if user[:4].lower() == u'uid:': user = user[4:] - elif user.find('@') == -1: + elif user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) print u"Deleting photo for %s (%s of %s)" % (user, i, count) - callGAPI(cd.users().photos(), 'delete', userKey=user) + callGAPI(cd.users().photos(), u'delete', userKey=user) i += 1 def showCalendars(users): @@ -3122,30 +3122,30 @@ def showCalendars(users): cal = buildGAPIObject(u'calendar', user) feed = callGAPI(cal.calendarList(), u'list') for usercal in feed[u'items']: - print u' Name: %s' % usercal['id'] - print convertUTF8(u' Summary: %s' % usercal['summary']) + print u' Name: %s' % usercal[u'id'] + print convertUTF8(u' Summary: %s' % usercal[u'summary']) try: - print convertUTF8(u' Description: %s' % usercal['description']) + print convertUTF8(u' Description: %s' % usercal[u'description']) except KeyError: print u' Description: ' - print u' Access Level: %s' % usercal['accessRole'] - print u' Timezone: %s' % usercal['timeZone'] + print u' Access Level: %s' % usercal[u'accessRole'] + print u' Timezone: %s' % usercal[u'timeZone'] try: - print convertUTF8(u' Location: %s' % usercal['location']) + print convertUTF8(u' Location: %s' % usercal[u'location']) except KeyError: pass try: - print u' Hidden: %s' % usercal['hidden'] + print u' Hidden: %s' % usercal[u'hidden'] except KeyError: print u' Hidden: False' try: - print u' Selected: %s' % usercal['selected'] + print u' Selected: %s' % usercal[u'selected'] except KeyError: print u' Selected: False' print u' Default Reminders:' try: for reminder in usercal[u'defaultReminders']: - print u' Type: %s Minutes: %s' % (reminder['method'], reminder['minutes']) + print u' Type: %s Minutes: %s' % (reminder[u'method'], reminder[u'minutes']) except KeyError: pass print u'' @@ -3153,7 +3153,7 @@ def showCalendars(users): def showCalSettings(users): for user in users: cal = buildGAPIObject(u'calendar', user) - feed = callGAPI(cal.settings(), 'list') + feed = callGAPI(cal.settings(), u'list') for setting in feed[u'items']: print u'%s: %s' % (setting[u'id'], setting[u'value']) @@ -3206,7 +3206,7 @@ def doDriveActivity(users): todrive = False i = 5 while i < len(sys.argv): - activity_object = sys.argv[i].lower().replace(u'_', '') + activity_object = sys.argv[i].lower().replace(u'_', u'') if activity_object == u'fileid': drive_fileId = sys.argv[i+1] drive_ancestorId = None @@ -3356,7 +3356,7 @@ def showDriveFiles(users): i = 5 labels = list() while i < len(sys.argv): - my_arg = sys.argv[i].lower().replace('_', '') + my_arg = sys.argv[i].lower().replace(u'_', u'') if my_arg == u'todrive': todrive = True i += 1 @@ -3414,7 +3414,7 @@ def showDriveFiles(users): elif my_arg in [u'originalfilename']: fields += u',originalFilename' i += 1 - elif my_arg in [u'quotaused', 'quotabytesused']: + elif my_arg in [u'quotaused', u'quotabytesused']: fields += u',quotaBytesUsed' i += 1 elif my_arg in [u'shared']: @@ -3427,9 +3427,9 @@ def showDriveFiles(users): print u'ERROR: %s is not a valid argument for "gam show filelist"' % my_arg sys.exit(2) if len(labels) > 0: - fields += ',labels(%s)' % ','.join(labels) + fields += u',labels(%s)' % ','.join(labels) if fields != u'*': - fields += ')' + fields += u')' for user in users: drive = buildGAPIObject(u'drive', user) if user.find(u'@') == -1: @@ -3521,7 +3521,7 @@ def printDriveFolderContents(feed, folderId, indent): for f_file in feed: for parent in f_file[u'parents']: if folderId == parent[u'id']: - print ' ' * indent, convertUTF8(f_file[u'title']) + print u' ' * indent, convertUTF8(f_file[u'title']) if f_file[u'mimeType'] == u'application/vnd.google-apps.folder': printDriveFolderContents(feed, f_file[u'id'], indent+1) @@ -3598,7 +3598,7 @@ def doUpdateDriveFile(users): elif sys.argv[i].lower() in [u'ocrlanguage',]: ocrLanguage = sys.argv[i+1] i += 2 - elif sys.argv[i].lower() in [u'restrict', 'restricted']: + elif sys.argv[i].lower() in [u'restrict', u'restricted']: if 'labels' not in body: body[u'labels'] = dict() if sys.argv[i+1].lower() in true_values: @@ -3707,9 +3707,9 @@ def doUpdateDriveFile(users): for fileId in fileIds: if operation == u'update': if media_body: - result = callGAPI(drive.files(), u'update', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields='id') + result = callGAPI(drive.files(), u'update', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields=u'id') else: - result = callGAPI(drive.files(), u'patch', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, body=body, fields='id,labels') + result = callGAPI(drive.files(), u'patch', fileId=fileId, convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, body=body, fields=u'id,labels') try: print u'Successfully updated %s drive file with content from %s' % (result[u'id'], local_filename) except UnboundLocalError: @@ -3744,7 +3744,7 @@ def createDriveFile(users): elif sys.argv[i].lower() in [u'ocrlanguage',]: ocrLanguage = sys.argv[i+1] i += 2 - elif sys.argv[i].lower() in [u'restrict', 'restricted']: + elif sys.argv[i].lower() in [u'restrict', u'restricted']: if u'labels' not in body: body[u'labels'] = dict() body[u'labels'][u'restricted'] = True @@ -3818,7 +3818,7 @@ def createDriveFile(users): body[u'parents'].append({u'id': a_parent}) if local_filepath: media_body = googleapiclient.http.MediaFileUpload(local_filepath, mimetype=mimetype, resumable=True) - result = callGAPI(drive.files(), u'insert', convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields='id') + result = callGAPI(drive.files(), u'insert', convert=convert, ocr=ocr, ocrLanguage=ocrLanguage, media_body=media_body, body=body, fields=u'id') try: print u'Successfully uploaded %s to Drive file ID %s' % (local_filename, result[u'id']) except UnboundLocalError: @@ -3834,16 +3834,16 @@ def downloadDriveFile(users): if sys.argv[i].lower() == u'id': fileIds = [sys.argv[i+1],] i += 2 - elif sys.argv[i].lower() == 'query': + elif sys.argv[i].lower() == u'query': query = sys.argv[i+1] i += 2 elif sys.argv[i].lower() == u'format': gdownload_format = sys.argv[i+1].lower() if gdownload_format not in [u'openoffice', u'ms', u'microsoft', u'micro$oft', u'pdf']: - print 'ERROR: format must be one of openoffice, microsoft or pdf. Got %s' % gdownload_format + print u'ERROR: format must be one of openoffice, microsoft or pdf. Got %s' % gdownload_format sys.exit(2) i += 2 - elif sys.argv[i].lower().replace('_', '') == u'targetfolder': + elif sys.argv[i].lower().replace(u'_', u'') == u'targetfolder': target_folder = sys.argv[i+1] if not os.path.isdir(target_folder): os.makedirs(target_folder) @@ -3877,10 +3877,10 @@ def downloadDriveFile(users): if query: fileIds = doDriveSearch(drive, query=query) else: - if fileIds[0][:8].lower() == 'https://' or fileIds[0][:7].lower() == 'http://': - fileIds[0] = fileIds[0][fileIds[0].find('/d/')+3:] - if fileIds[0].find('/') != -1: - fileIds[0] = fileIds[0][:fileIds[0].find('/')] + if fileIds[0][:8].lower() == u'https://' or fileIds[0][:7].lower() == u'http://': + fileIds[0] = fileIds[0][fileIds[0].find(u'/d/')+3:] + if fileIds[0].find(u'/') != -1: + fileIds[0] = fileIds[0][:fileIds[0].find(u'/')] if not fileIds: print u'No files to download for %s' % user i = 0 @@ -3915,13 +3915,13 @@ def downloadDriveFile(users): pass break else: - print convertUTF8(u'Skipping download of file {0}, Format {1} not available'.format(result[u'title'], ','.join(export_formats))) + print convertUTF8(u'Skipping download of file {0}, Format {1} not available'.format(result[u'title'], u','.join(export_formats))) continue else: print convertUTF8(u'Skipping download of file {0}, Format not downloadable') continue file_title = result[u'title'] - safe_file_title = ''.join(c for c in file_title if c in safe_filename_chars) + safe_file_title = u''.join(c for c in file_title if c in safe_filename_chars) filename = os.path.join(target_folder, safe_file_title) if extension and filename.lower()[:len(extension)] != extension: filename = u'%s%s' % (filename, extension) @@ -4049,7 +4049,7 @@ def transferDriveFiles(users): if source_parent[u'id'] not in all_source_file_ids and source_parent[u'id'] not in all_target_folder_ids: continue # means this parent isn't owned by source or target, shouldn't matter if source_parent[u'id'] not in transferred_files and source_parent[u'id'] != source_root: - #print 'skipping %s' % file_id + #print u'skipping %s' % file_id skipped_files = skip_file_for_now = True break if skip_file_for_now: @@ -4308,8 +4308,8 @@ def doLanguage(users): i = 1 for user in users: if user.find(u'@') > 0: - emailsettings.domain = user[user.find('@')+1:] - user = user[:user.find('@')] + emailsettings.domain = user[user.find(u'@')+1:] + user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain print u"Setting the language for %s to %s (%s of %s)" % (user+u'@'+emailsettings.domain, language, i, count) @@ -4579,7 +4579,7 @@ def showGmailProfile(users): sys.exit(2) profiles = [{}] for user in users: - print 'Getting Gmail profile for %s' % user + print u'Getting Gmail profile for %s' % user gmail = buildGAPIObject(u'gmail', user, soft_errors=True) if not gmail: continue @@ -4599,13 +4599,13 @@ def updateLabels(users): if sys.argv[i].lower() == u'name': body[u'name'] = sys.argv[i+1] i += 2 - elif sys.argv[i].lower().replace(u'_', '') == u'messagelistvisibility': + elif sys.argv[i].lower().replace(u'_', u'') == u'messagelistvisibility': body[u'messageListVisibility'] = sys.argv[i+1].lower() if body[u'messageListVisibility'] not in [u'hide', u'show']: - print 'ERROR: message_list_visibility should be show or hide, got %s' % sys.argv[i+1] + print u'ERROR: message_list_visibility should be show or hide, got %s' % sys.argv[i+1] sys.exit(2) i += 2 - elif sys.argv[i].lower().replace(u' ', '') == u'labellistvisibility': + elif sys.argv[i].lower().replace(u' ', u'') == u'labellistvisibility': if sys.argv[i+1].lower().replace(u'_', u'') == u'showifunread': body[u'labelListVisibility'] = u'labelShowIfUnread' elif sys.argv[i+1].lower().replace(u'_', u'') == u'show': @@ -4613,7 +4613,7 @@ def updateLabels(users): elif sys.argv[i+1].lower().replace(u'_', u'') == u'hide': body[u'labelListVisibility'] = u'labelHide' else: - print 'ERROR: label_list_visibility should be hide, show or show_if_unread, got %s' % sys.argv[i+1] + print u'ERROR: label_list_visibility should be hide, show or show_if_unread, got %s' % sys.argv[i+1] sys.exit(2) i += 2 else: @@ -4628,7 +4628,7 @@ def updateLabels(users): label_id = label[u'id'] break if not label_id: - print 'Error: user does not have a label named %s' % label_name + print u'Error: user does not have a label named %s' % label_name callGAPI(gmail.users().labels(), u'patch', soft_errors=True, userId=user, id=label_id, body=body) def renameLabels(users): @@ -4814,7 +4814,7 @@ def getForward(users): emailsettings = getEmailSettingsObject() for user in users: if user.find(u'@') > 0: - emailsettings.domain = user[user.find('@')+1:] + emailsettings.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] @@ -4876,7 +4876,7 @@ def doWebClips(users): i = 1 for user in users: if user.find(u'@') > 0: - emailsettings.domain = user[user.find('@')+1:] + emailsettings.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] else: emailsettings.domain = GC_Values[GC_DOMAIN] #make sure it's back at default domain @@ -5000,17 +5000,17 @@ def doCreateOrUpdateUserSchema(): i += 1 break else: - print 'ERROR: %s is not a valid argument for "gam create schema"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam create schema"' % sys.argv[i] sys.exit(2) else: - print 'ERROR: %s is not a valid argument for "gam create schema"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam create schema"' % sys.argv[i] sys.exit(2) if sys.argv[1].lower() == u'create': result = callGAPI(cd.schemas(), u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) - print 'Created user schema %s' % result[u'schemaName'] + print u'Created user schema %s' % result[u'schemaName'] elif sys.argv[1].lower() == u'update': result = callGAPI(cd.schemas(), u'update', customerId=GC_Values[GC_CUSTOMER_ID], body=body, schemaKey=schemaName) - print 'Updated user schema %s' % result[u'schemaName'] + print u'Updated user schema %s' % result[u'schemaName'] def doPrintUserSchemas(): cd = buildGAPIObject(u'directory') @@ -5021,13 +5021,13 @@ def doPrintUserSchemas(): print u'Schema: %s' % schema[u'schemaName'] for a_key in schema: if a_key not in [u'schemaName', u'fields', u'etag', u'kind']: - print '%s: %s' % (a_key, schema[a_key]) + print u'%s: %s' % (a_key, schema[a_key]) print for field in schema[u'fields']: print u' Field: %s' % field[u'fieldName'] for a_key in field: if a_key not in [u'fieldName', u'kind', u'etag']: - print ' %s: %s' % (a_key, field[a_key]) + print u' %s: %s' % (a_key, field[a_key]) print print @@ -5038,13 +5038,13 @@ def doGetUserSchema(): print u'Schema: %s' % schema[u'schemaName'] for a_key in schema: if a_key not in [u'schemaName', u'fields', u'etag', u'kind']: - print '%s: %s' % (a_key, schema[a_key]) + print u'%s: %s' % (a_key, schema[a_key]) print for field in schema[u'fields']: print u' Field: %s' % field[u'fieldName'] for a_key in field: if a_key not in [u'fieldName', u'kind', u'etag']: - print ' %s: %s' % (a_key, field[a_key]) + print u' %s: %s' % (a_key, field[a_key]) print def doCreateUser(): @@ -5155,7 +5155,7 @@ def doCreateUser(): i += 1 im[u'type'] = sys.argv[i].lower() if im[u'type'] not in [u'custom', u'home', u'other', u'work']: - print u'ERROR: type should be custom, home, other or work. Got %s' % im['type'] + print u'ERROR: type should be custom, home, other or work. Got %s' % im[u'type'] sys.exit(2) if im[u'type'] == u'custom': i += 1 @@ -5359,7 +5359,7 @@ def doCreateUser(): try: (schemaName, fieldName) = sys.argv[i].split(u'.') except ValueError: - print 'ERROR: %s is not a valid create user argument or custom schema name.' % sys.argv[i] + print u'ERROR: %s is not a valid create user argument or custom schema name.' % sys.argv[i] sys.exit(2) field_value = sys.argv[i+1] is_multivalue = False @@ -5387,7 +5387,7 @@ def doCreateUser(): body[u'password'] = gen_sha512_hash(body[u'password']) body[u'hashFunction'] = u'crypt' print u"Creating account for %s" % body[u'primaryEmail'] - callGAPI(cd.users(), 'insert', body=body, fields=u'primaryEmail') + callGAPI(cd.users(), u'insert', body=body, fields=u'primaryEmail') if do_admin: print u' Changing admin status for %s to %s' % (body[u'primaryEmail'], admin_body[u'status']) callGAPI(cd.users(), u'makeAdmin', userKey=body[u'primaryEmail'], body=admin_body) @@ -5549,7 +5549,7 @@ def doUpdateUser(users, i): do_update_user = True body[u'password'] = sys.argv[i+1] if body[u'password'].lower() == u'random': - body[u'password'] = ''.join(random.sample(u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789~`!@#$%^&*()-=_+:;"\'{}[]\\|', 50)) + body[u'password'] = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789~`!@#$%^&*()-=_+:;"\'{}[]\\|', 50)) i += 2 gotPassword = True elif sys.argv[i].lower() == u'admin': @@ -5608,7 +5608,7 @@ def doUpdateUser(users, i): elif sys.argv[i+1].lower() in false_values: body[u'changePasswordAtNextLogin'] = False i += 2 - elif sys.argv[i].lower() in ['org', u'ou']: + elif sys.argv[i].lower() in [u'org', u'ou']: do_update_user = True body[u'orgUnitPath'] = sys.argv[i+1] if body[u'orgUnitPath'][0] != u'/': @@ -6064,7 +6064,7 @@ def doUpdateGroup(): group = u'%s@%s' % (group, GC_Values[GC_DOMAIN]) if use_cd_api: try: - if cd_body[u'email'].find('@') == -1: + if cd_body[u'email'].find(u'@') == -1: cd_body[u'email'] = u'%s@%s' % (cd_body[u'email'], GC_Values[GC_DOMAIN]) except KeyError: pass @@ -6153,8 +6153,8 @@ def doUpdateCros(): i += 2 elif sys.argv[i].lower() == u'status': body[u'status'] = sys.argv[i + 1].upper() - #if body['status'] not in ['ACTIVE', 'DEPROVISIONED']: - # print 'ERROR: status must be active or deprovisioned, got %s' % body['status'] + #if body[u'status'] not in [u'ACTIVE', u'DEPROVISIONED']: + # print u'ERROR: status must be active or deprovisioned, got %s' % body[u'status'] # sys.exit(2) i += 2 elif sys.argv[i].lower() in [u'tag', u'asset', u'assetid']: @@ -6187,7 +6187,7 @@ def doUpdateMobile(): action_body[u'action'] = sys.argv[i+1].lower() if action_body[u'action'] == u'wipe': action_body[u'action'] = u'admin_remote_wipe' - elif action_body[u'action'].replace(u'_', '') in [u'accountwipe', u'wipeaccount']: + elif action_body[u'action'].replace(u'_', u'') in [u'accountwipe', u'wipeaccount']: action_body[u'action'] = u'admin_account_wipe' if action_body[u'action'] not in [u'admin_remote_wipe', u'admin_account_wipe', u'approve', u'block', u'cancel_remote_wipe_then_activate', u'cancel_remote_wipe_then_block']: print u'ERROR: action must be wipe, wipeaccount, approve, block, cancel_remote_wipe_then_activate or cancel_remote_wipe_then_block. Got %s' % action_body[u'action'] @@ -6217,7 +6217,7 @@ def doUpdateMobile(): def doDeleteMobile(): cd = buildGAPIObject(u'directory') resourceId = sys.argv[3] - callGAPI(cd.mobiledevices(), 'delete', resourceId=resourceId, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.mobiledevices(), u'delete', resourceId=resourceId, customerId=GC_Values[GC_CUSTOMER_ID]) def doUpdateOrg(): cd = buildGAPIObject(u'directory') @@ -6238,7 +6238,7 @@ def doUpdateOrg(): else: user_count = len(users) current_user = 1 - if orgUnitPath != u'/' and orgUnitPath[0] != '/': # we do want a / at the beginning for user updates + if orgUnitPath != u'/' and orgUnitPath[0] != u'/': # we do want a / at the beginning for user updates orgUnitPath = u'/%s' % orgUnitPath for user in users: sys.stderr.write(u' moving %s to %s (%s/%s)\n' % (user, orgUnitPath, current_user, user_count)) @@ -6260,7 +6260,7 @@ def doUpdateOrg(): elif sys.argv[i].lower() == u'parent': body[u'parentOrgUnitPath'] = sys.argv[i+1] if body[u'parentOrgUnitPath'][0] != u'/': - body[u'parentOrgUnitPath'] = '/'+body[u'parentOrgUnitPath'] + body[u'parentOrgUnitPath'] = u'/'+body[u'parentOrgUnitPath'] i += 2 elif sys.argv[i].lower() == u'noinherit': body[u'blockInheritance'] = True @@ -6478,7 +6478,7 @@ def doGetUserInfo(user_email=None): print u' %s <%s>' % (group[u'name'], group[u'email']) if getLicenses: print u'Licenses:' - lic = buildGAPIObject(api='licensing') + lic = buildGAPIObject(u'licensing') for sku in [u'Google-Apps', u'Google-Apps-For-Business', u'Google-Apps-Unlimited', u'Google-Apps-For-Postini', u'Google-Coordinate', u'Google-Drive-storage-20GB', u'Google-Drive-storage-50GB', u'Google-Drive-storage-200GB', u'Google-Drive-storage-400GB', u'Google-Drive-storage-1TB', u'Google-Drive-storage-2TB', @@ -6486,7 +6486,7 @@ def doGetUserInfo(user_email=None): u'Google-Vault-Former-Employee']: productId, skuId = getProductAndSKU(sku) try: - result = callGAPI(lic.licenseAssignments(), u'get', throw_reasons=['notFound'], userId=user_email, productId=productId, skuId=skuId) + result = callGAPI(lic.licenseAssignments(), u'get', throw_reasons=[u'notFound'], userId=user_email, productId=productId, skuId=skuId) except googleapiclient.errors.HttpError: continue print u' %s' % result[u'skuId'] @@ -6638,7 +6638,7 @@ def doUpdateNotification(): ids.append(sys.argv[i+1]) i += 2 else: - print 'ERROR: %s is not a valid argument for "gam update notification"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam update notification"' % sys.argv[i] sys.exit(2) if isUnread == None: print u'ERROR: notifications need to be marked as read or unread.' @@ -6670,7 +6670,7 @@ def doDeleteNotification(): ids.append(sys.argv[i+1]) i += 2 else: - print 'ERROR: %s is not a valid argument for "gam delete notification", expected id' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam delete notification", expected id' % sys.argv[i] sys.exit(2) if get_all: notifications = callGAPIpages(cd.notifications(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID], fields=u'items(notificationId),nextPageToken') @@ -6796,7 +6796,7 @@ def doGetNotifications(): if sys.argv[i].lower() == u'unreadonly': unread_only = True else: - print 'ERROR: %s is not a valid argument for "gam info notification", expected unreadonly' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam info notification", expected unreadonly' % sys.argv[i] sys.exit(2) i += 1 notifications = callGAPIpages(cd.notifications(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID]) @@ -6984,7 +6984,7 @@ def doDeprovUser(users): i = 1 try: for asp in asps[u'items']: - print u' deleting ASP %s of %s' % (i, len(asps['items'])) + print u' deleting ASP %s of %s' % (i, len(asps[u'items'])) callGAPI(cd.asps(), u'delete', userKey=user, codeId=asp[u'codeId']) i += 1 except KeyError: @@ -6999,7 +6999,7 @@ def doDeprovUser(users): i = 1 try: for token in tokens[u'items']: - print u' deleting token %s of %s' % (i, len(tokens['items'])) + print u' deleting token %s of %s' % (i, len(tokens[u'items'])) callGAPI(cd.tokens(), u'delete', userKey=user, clientId=token[u'clientId']) i += 1 except KeyError: @@ -7145,7 +7145,7 @@ def doGetInstanceInfo(): adm = buildGAPIObject(u'admin-settings') if len(sys.argv) > 4 and sys.argv[3].lower() == u'logo': target_file = sys.argv[4] - url = 'http://www.google.com/a/cpanel/%s/images/logo.gif' % (GC_Values[GC_DOMAIN]) + url = u'http://www.google.com/a/cpanel/%s/images/logo.gif' % (GC_Values[GC_DOMAIN]) geturl(url, target_file) sys.exit(0) print u'Google Apps Domain: %s' % (GC_Values[GC_DOMAIN]) @@ -7162,9 +7162,9 @@ def doGetInstanceInfo(): print u'Customer ID: %s' % customerId default_language = callGAPI(adm.defaultLanguage(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Default Language: %s' % default_language[u'entry'][u'apps$property'][0][u'value'] - org_name = callGAPI(adm.organizationName(), 'get', domainName=GC_Values[GC_DOMAIN]) + org_name = callGAPI(adm.organizationName(), u'get', domainName=GC_Values[GC_DOMAIN]) print convertUTF8(u'Organization Name: %s' % org_name[u'entry'][u'apps$property'][0][u'value']) - admin_email = callGAPI(adm.adminSecondaryEmail(), 'get', domainName=GC_Values[GC_DOMAIN]) + admin_email = callGAPI(adm.adminSecondaryEmail(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Admin Secondary Email: %s' % admin_email[u'entry'][u'apps$property'][0][u'value'] max_users = callGAPI(adm.maximumNumberOfUsers(), u'get', domainName=GC_Values[GC_DOMAIN]) print u'Maximum Users: %s' % max_users[u'entry'][u'apps$property'][0][u'value'] @@ -7374,21 +7374,21 @@ def output_csv(csv_list, titles, list_type, todrive): import webbrowser webbrowser.open(file_url) -def flatten_json(structure, key="", path="", flattened=None): +def flatten_json(structure, key=u'', path=u'', flattened=None): if flattened == None: flattened = {} if type(structure) not in(dict, list): - flattened[((path + ".") if path else "") + key] = structure + flattened[((path + u'.') if path else u'') + key] = structure elif isinstance(structure, list): for i, item in enumerate(structure): - flatten_json(item, "%d" % i, ".".join(filter(None, [path, key])), flattened) + flatten_json(item, u'%d' % i, u'.'.join(filter(None, [path, key])), flattened) else: for new_key, value in structure.items(): if new_key in [u'kind', u'etag']: continue if value == u'1970-01-01T00:00:00.000Z': value = u'Never' - flatten_json(value, new_key, ".".join(filter(None, [path, key])), flattened) + flatten_json(value, new_key, u'.'.join(filter(None, [path, key])), flattened) return flattened def doPrintUsers(): @@ -7471,7 +7471,7 @@ def doPrintUsers(): elif sys.argv[i].lower() == u'gal': user_fields.append(u'includeInGlobalAddressList') i += 1 - elif sys.argv[i].lower() in ['photo', 'photourl']: + elif sys.argv[i].lower() in [u'photo', u'photourl']: user_fields.append(u'thumbnailPhotoUrl') i += 1 elif sys.argv[i].lower() == u'id': @@ -7518,7 +7518,7 @@ def doPrintUsers(): email_parts = True i += 1 else: - print 'ERROR: %s is not a valid argument for "gam print users"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print users"' % sys.argv[i] sys.exit(2) if fields != None: user_fields = set(user_fields) @@ -7656,7 +7656,7 @@ def doPrintGroups(): settings = True i += 1 else: - print 'ERROR: %s is not a valid argument for "gam print groups"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print groups"' % sys.argv[i] sys.exit(2) sys.stderr.write(u"Retrieving All Groups for Google Apps account (may take some time on a large account)...\n") page_message = u'Got %%num_items%% groups: %%first_item%% - %%last_item%%\n' @@ -7801,7 +7801,7 @@ def doPrintOrgs(): titles.append(u'InheritanceBlocked') i += 1 else: - print 'ERROR: %s is not a valid argument for "gam print orgs"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print orgs"' % sys.argv[i] sys.exit(2) if fields: org_attributes[0][u'Path'] = u'Path' @@ -7838,7 +7838,7 @@ def doPrintOrgs(): if printparent: parent = org_vals[u'parentOrgUnitPath'] if parent == None: - parent = '' + parent = u'' orgUnit.update({u'Parent': parent}) if printinherit: try: @@ -7900,7 +7900,7 @@ def doPrintGroupMembers(): all_groups = [{u'email': sys.argv[i+1].lower()}] i += 2 else: - print 'ERROR: %s is not a valid argument for "gam print group-members"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print group-members"' % sys.argv[i] sys.exit(2) member_attributes = [{u'group': u'group'},] if not all_groups: @@ -7945,7 +7945,7 @@ def doPrintMobileDevices(): orderBy = sys.argv[i+1].lower() allowed_values = [u'deviceid', u'email', u'lastsync', u'model', u'name', u'os', u'status', u'type'] if orderBy.lower() not in allowed_values: - print u'ERROR: orderBy must be one of %s. Got %s' % (u', '.join(allowed_values), orderBy) + print u'ERROR: orderBy must be one of %s. Got %s' % (u', u'.join(allowed_values), orderBy) sys.exit(2) elif orderBy == u'lastsync': orderBy = u'lastSync' @@ -7956,7 +7956,7 @@ def doPrintMobileDevices(): sortOrder = sys.argv[i].upper() i += 1 else: - print 'ERROR: %s is not a valid argument for "gam print mobile"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print mobile"' % sys.argv[i] sys.exit(2) sys.stderr.write(u'Retrieving All Mobile Devices for organization (may take some time for large accounts)...\n') page_message = u'Got %%num_items%% mobile devices...\n' @@ -8016,7 +8016,7 @@ def doPrintCrosDevices(): orderBy = sys.argv[i+1].lower().replace(u'_', u'') allowed_values = [u'location', u'user', u'lastsync', u'notes', u'serialnumber', u'status', u'supportenddate'] if orderBy not in allowed_values: - print u'ERROR: orderBy must be one of %s. Got %s' % (u', '.join(allowed_values), orderBy) + print u'ERROR: orderBy must be one of %s. Got %s' % (u', u'.join(allowed_values), orderBy) sys.exit(2) elif orderBy == u'location': orderBy = u'annotatedLocation' @@ -8036,7 +8036,7 @@ def doPrintCrosDevices(): projection = my_arg.upper() i += 1 else: - print 'ERROR: %s is not a valid argument for "gam print cros"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print cros"' % sys.argv[i] sys.exit(2) if selectAttrib: projection = u'FULL' @@ -8082,7 +8082,7 @@ def doPrintCrosDevices(): else: new_row[attribMap[attrib]] = item[attrib] cros_attributes.append(new_row) - output_csv(cros_attributes, titles, 'CrOS', todrive) + output_csv(cros_attributes, titles, u'CrOS', todrive) def doPrintLicenses(return_list=False, skus=None): lic = buildGAPIObject(u'licensing') @@ -8096,10 +8096,10 @@ def doPrintLicenses(return_list=False, skus=None): todrive = True i += 1 elif sys.argv[i].lower() in [u'products', u'product']: - products = sys.argv[i+1].split(',') + products = sys.argv[i+1].split(u',') i += 2 elif sys.argv[i].lower() in [u'sku', u'skus']: - skus = sys.argv[i+1].split(',') + skus = sys.argv[i+1].split(u',') i += 2 else: print u'ERROR: %s is not a valid argument for "gam print licenses"' % sys.argv[i] @@ -8158,12 +8158,12 @@ def doPrintTokens(): token_attributes[0][title] = title for user in all_users: sys.stderr.write(u' getting tokens for %s\n' % user) - user_tokens = callGAPI(cd.tokens(), 'list', userKey=user) + user_tokens = callGAPI(cd.tokens(), u'list', userKey=user) try: for user_token in user_tokens[u'items']: this_token = dict() this_token[u'user'] = user - this_token[u'scopes'] = ' '.join(user_token[u'scopes']) + this_token[u'scopes'] = u' '.join(user_token[u'scopes']) for token_item in user_token: if token_item in [u'kind', u'etag', u'scopes']: continue @@ -8199,7 +8199,7 @@ def doPrintResourceCalendars(): fields.append(u'resourceType') i += 1 else: - print 'ERROR: %s is not a valid argument for "gam print resources"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam print resources"' % sys.argv[i] sys.exit(2) if fields: fields = u'nextPageToken,items(%s)' % u','.join(fields) @@ -8257,10 +8257,10 @@ def doCreateMonitor(): drafts_headers_only = True i += 1 else: - print 'ERROR: %s is not a valid argument for "gam create monitor"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam create monitor"' % sys.argv[i] sys.exit(2) audit = getAuditObject() - if source_user.find('@') > 0: + if source_user.find(u'@') > 0: audit.domain = source_user[source_user.find(u'@')+1:] source_user = source_user[:source_user.find(u'@')] callGData(audit, u'createEmailMonitor', source_user=source_user, destination_user=destination_user, end_date=end_date, begin_date=begin_date, @@ -8270,23 +8270,23 @@ def doCreateMonitor(): def doShowMonitors(): user = sys.argv[4].lower() audit = getAuditObject() - if user.find('@') > 0: + if user.find(u'@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] results = callGData(audit, u'getEmailMonitors', user=user) print sys.argv[4].lower()+u' has the following monitors:' print u'' for monitor in results: - print u' Destination: '+monitor['destUserName'] + print u' Destination: '+monitor[u'destUserName'] try: - print u' Begin: '+monitor['beginDate'] + print u' Begin: '+monitor[u'beginDate'] except KeyError: print u' Begin: immediately' - print u' End: '+monitor['endDate'] - print u' Monitor Incoming: '+monitor['outgoingEmailMonitorLevel'] - print u' Monitor Outgoing: '+monitor['incomingEmailMonitorLevel'] - print u' Monitor Chats: '+monitor['chatMonitorLevel'] - print u' Monitor Drafts: '+monitor['draftMonitorLevel'] + print u' End: '+monitor[u'endDate'] + print u' Monitor Incoming: '+monitor[u'outgoingEmailMonitorLevel'] + print u' Monitor Outgoing: '+monitor[u'incomingEmailMonitorLevel'] + print u' Monitor Chats: '+monitor[u'chatMonitorLevel'] + print u' Monitor Drafts: '+monitor[u'draftMonitorLevel'] print u'' def doDeleteMonitor(): @@ -8301,9 +8301,9 @@ def doDeleteMonitor(): def doRequestActivity(): user = sys.argv[4].lower() audit = getAuditObject() - if user.find('@') > 0: + if user.find(u'@') > 0: audit.domain = user[user.find(u'@')+1:] - user = user[:user.find('@')] + user = user[:user.find(u'@')] results = callGData(audit, u'createAccountInformationRequest', user=user) print u'Request successfully submitted:' print u' Request ID: '+results[u'requestId'] @@ -8317,7 +8317,7 @@ def doStatusActivityRequests(): try: user = sys.argv[4].lower() if user.find(u'@') > 0: - audit.domain = user[user.find('@')+1:] + audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] request_id = sys.argv[5].lower() results = callGData(audit, u'getAccountInformationRequestStatus', user=user, request_id=request_id) @@ -8392,28 +8392,28 @@ def doRequestExport(): include_deleted = True i += 1 else: - print 'ERROR: %s is not a valid argument for "gam export request"' % sys.argv[i] + print u'ERROR: %s is not a valid argument for "gam export request"' % sys.argv[i] sys.exit(2) audit = getAuditObject() - if user.find('@') > 0: + if user.find(u'@') > 0: audit.domain = user[user.find(u'@')+1:] user = user[:user.find(u'@')] results = callGData(audit, u'createMailboxExportRequest', user=user, begin_date=begin_date, end_date=end_date, include_deleted=include_deleted, search_query=search_query, headers_only=headers_only) print u'Export request successfully submitted:' - print u' Request ID: '+results['requestId'] - print u' User: '+results['userEmailAddress'] - print u' Status: '+results['status'] - print u' Request Date: '+results['requestDate'] - print u' Requested By: '+results['adminEmailAddress'] - print u' Include Deleted: '+results['includeDeleted'] - print u' Requested Parts: '+results['packageContent'] + print u' Request ID: '+results[u'requestId'] + print u' User: '+results[u'userEmailAddress'] + print u' Status: '+results[u'status'] + print u' Request Date: '+results[u'requestDate'] + print u' Requested By: '+results[u'adminEmailAddress'] + print u' Include Deleted: '+results[u'includeDeleted'] + print u' Requested Parts: '+results[u'packageContent'] try: - print u' Begin: '+results['beginDate'] + print u' Begin: '+results[u'beginDate'] except KeyError: print u' Begin: account creation date' try: - print u' End: '+results['endDate'] + print u' End: '+results[u'endDate'] except KeyError: print u' End: export request date' @@ -8509,7 +8509,7 @@ def doWatchExportRequest(): msg_txt += u" Include Deleted: %s\n" % results[u'includeDeleted'] try: msg_txt += u" Number Of Files: %s\n" % results[u'numberOfFiles'] - for i in range(int(results['numberOfFiles'])): + for i in range(int(results[u'numberOfFiles'])): msg_txt += u" Url%s: %s\n" % (i, results[u'fileUrl%s' % i]) except KeyError: pass @@ -8546,7 +8546,7 @@ def doDownloadExportRequest(): systemErrorExit(4, MESSAGE_REQUEST_NOT_COMPLETE.format(results[u'status'])) if int(results.get(u'numberOfFiles', u'0')) < 1: systemErrorExit(4, MESSAGE_REQUEST_COMPLETED_NO_FILES) - for i in range(0, int(results['numberOfFiles'])): + for i in range(0, int(results[u'numberOfFiles'])): url = results[u'fileUrl'+str(i)] filename = u'export-'+user+'-'+request_id+'-'+str(i)+u'.mbox.gpg' #don't download existing files. This does not check validity of existing local @@ -8716,13 +8716,13 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa if not silent: sys.stderr.write(u"done getting %s CrOS devices.\r\n" % len(users)) else: - print 'ERROR: %s is not a valid argument for "gam all"' % entity + print u'ERROR: %s is not a valid argument for "gam all"' % entity sys.exit(3) elif entity_type == u'cros': users = entity.replace(u',', u' ').split() entity = u'cros' else: - print 'ERROR: %s is not a valid argument for "gam"' % entity_type + print u'ERROR: %s is not a valid argument for "gam"' % entity_type sys.exit(2) full_users = list() if entity != u'cros' and not got_uids: @@ -8979,7 +8979,7 @@ def run_batch(items): total_items = len(items) current_item = 0 python_cmd = [sys.executable.lower(),] - if not getattr(sys, 'frozen', False): # we're not frozen + if not getattr(sys, u'frozen', False): # we're not frozen python_cmd.append(os.path.realpath(sys.argv[0])) num_worker_threads = min(total_items, GC_Values[GC_NUM_THREADS]) GM_Globals[GM_BATCH_QUEUE] = Queue.Queue(maxsize=num_worker_threads) # GM_Globals[GM_BATCH_QUEUE].put() gets blocked when trying to create more items than there are workers @@ -9093,8 +9093,8 @@ try: systemErrorExit(2, MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE.format(u'csv')) f = openFile(filename) input_file = csv.DictReader(f) - if sys.argv[3].lower() != 'gam': - print 'ERROR: "gam csv " should be followed by a full GAM command...' + if sys.argv[3].lower() != u'gam': + print u'ERROR: "gam csv " should be followed by a full GAM command...' sys.exit(3) GAM_argv, subFields = getSubFields(4, input_file.fieldnames) items = list() @@ -9113,7 +9113,7 @@ try: doCreateGroup() elif sys.argv[2].lower() in [u'nickname', u'alias']: doCreateAlias() - elif sys.argv[2].lower() in [u'org', 'ou']: + elif sys.argv[2].lower() in [u'org', u'ou']: doCreateOrg() elif sys.argv[2].lower() == u'resource': doCreateResourceCalendar() From 1746845651d58c4a1dca1a59c5e5ecadd8820d27 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 13 Jan 2016 08:07:44 -0800 Subject: [PATCH 086/100] Handle Unicode in doGetNotifications --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 92f2582f..0c0893a2 100755 --- a/src/gam.py +++ b/src/gam.py @@ -6812,7 +6812,7 @@ def doGetNotifications(): else: print u'Read Status: READ' print u'' - print dehtml(notification[u'body']) + print convertUTF8(dehtml(notification[u'body'])) print u'' print u'--------------' print u'' From 154099c3f424b116f7eb02adf7651ca50f07c3fc Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 13 Jan 2016 21:32:51 -0800 Subject: [PATCH 087/100] Back to single scopes list --- src/gam.py | 93 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/src/gam.py b/src/gam.py index 0c0893a2..6e30cc4d 100755 --- a/src/gam.py +++ b/src/gam.py @@ -91,7 +91,7 @@ GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json -GM_GAMSCOPES_BY_API = u'scop' +GM_GAMSCOPES_LIST = u'scop' # Current API user GM_CURRENT_API_USER = u'capu' # Current API scope @@ -118,7 +118,7 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, - GM_GAMSCOPES_BY_API: {}, + GM_GAMSCOPES_LIST: [], GM_CURRENT_API_USER: None, GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, @@ -250,7 +250,7 @@ MESSAGE_HIT_CONTROL_C_TO_UPDATE = u'\n\nHit CTRL+C to visit the GAM website and MESSAGE_INVALID_JSON = u'The file {0} has an invalid format.' MESSAGE_NO_DISCOVERY_INFORMATION = u'No online discovery doc and {0} does not exist locally' MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' -MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for {0}; please run gam oauth create' +MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for the {0}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT = u'Please authorize your Service account Client ID for the {0} scopes:\n\n{1}' @@ -458,7 +458,7 @@ def SetGlobalVariables(): GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY])) else falseValue def _getScopesAdminDomainFromGamScopesJson(): - GM_Globals[GM_GAMSCOPES_BY_API] = {} + GM_Globals[GM_GAMSCOPES_LIST] = [] json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=False) if json_string == None: return @@ -467,12 +467,9 @@ def SetGlobalVariables(): except ValueError: invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) scopes = json_data.get(u'scopes', None) - if (not scopes) or (not isinstance(scopes, dict)): + if not isinstance(scopes, list): invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) - for api, value in scopes.items(): - if not isinstance(value, list): - invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) - GM_Globals[GM_GAMSCOPES_BY_API][api] = list(set(value)) + GM_Globals[GM_GAMSCOPES_LIST] = list(set(scopes)) if not GC_Values[GC_ADMIN]: GC_Values[GC_ADMIN] = json_data.get(u'admin', GC_Defaults[GC_ADMIN]) if not GC_Values[GC_DOMAIN]: @@ -852,6 +849,25 @@ def getOAuth2ServiceDetails(): printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) invalidJSONExit(GC_Values[GC_OAUTH2SERVICE_JSON]) +def readDiscoveryFile(api_version): + disc_filename = u'%s.json' % (api_version) + disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) + if hasattr(sys, u'_MEIPASS'): + pyinstaller_disc_file = os.path.join(sys._MEIPASS, disc_filename) + else: + pyinstaller_disc_file = None + if os.path.isfile(disc_file): + json_string = readFile(disc_file) + elif pyinstaller_disc_file: + json_string = readFile(pyinstaller_disc_file) + else: + systemErrorExit(11, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) + try: + discovery = json.loads(json_string) + return (disc_file, discovery) + except ValueError: + invalidJSONExit(disc_file) + def getAPIversionHttpService(api): getOAuth2ServiceDetails() api, version, api_version = getAPIVersion(api) @@ -864,18 +880,7 @@ def getAPIversionHttpService(api): systemErrorExit(4, e) except googleapiclient.errors.UnknownApiNameOrVersion: pass - disc_filename = u'%s.json' % (api_version) - disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) - if hasattr(sys, u'_MEIPASS'): - pyinstaller_disc_file = os.path.join(sys._MEIPASS, disc_filename) - else: - pyinstaller_disc_file = None - if os.path.isfile(disc_file): - discovery = readFile(disc_file) - elif pyinstaller_disc_file: - discovery = readFile(pyinstaller_disc_file) - else: - systemErrorExit(11, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) + disc_file, discovery = readDiscoveryFile(api_version) try: service = googleapiclient.discovery.build_from_document(discovery, http=http) return (api_version, http, service) @@ -884,9 +889,9 @@ def getAPIversionHttpService(api): def buildGAPIObject(api, act_as=None, soft_errors=False): svcsub = act_as if act_as else GC_Values[GC_ADMIN] - api_version, http, service = getAPIversionHttpService(api) + _, http, service = getAPIversionHttpService(api) GM_Globals[GM_CURRENT_API_USER] = act_as - GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) + GM_Globals[GM_CURRENT_API_SCOPES] = list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(service._rootDesc[u'title'])) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], @@ -900,19 +905,17 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): return handleOAuthTokenError(e, soft_errors) return service -GDATA_API_INFO = { - u'admin-settings': u'Admin Settings API', - u'email-audit': u'Email Audit API', - u'email-settings': u'Email Settings API', - } - def initGDataObject(gdataObj, api): getOAuth2ServiceDetails() - api, _, api_version = getAPIVersion(api) + _, _, api_version = getAPIVersion(api) + disc_file, discovery = readDiscoveryFile(api_version) GM_Globals[GM_CURRENT_API_USER] = None - GM_Globals[GM_CURRENT_API_SCOPES] = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) + try: + GM_Globals[GM_CURRENT_API_SCOPES] = list(set(discovery[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) + except (ValueError, KeyError): + invalidJSONExit(disc_file) if not GM_Globals[GM_CURRENT_API_SCOPES]: - systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(GDATA_API_INFO[api])) + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(discovery.get(u'title', api_version))) getGDataOAuthToken(gdataObj) gdataObj.domain = GC_Values[GC_DOMAIN] #Identify GAM to Google's Servers @@ -8746,11 +8749,13 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa def OAuthInfo(): configRequired = False print u'API Access, Admin: {0}'.format(GC_Values[GC_ADMIN]) + i = 0 for api in sorted(API_VER_MAPPING.keys()): - print u' API: {0}'.format(api) - api_version, http, service = getAPIversionHttpService(api) + i += 1 + _, http, service = getAPIversionHttpService(api) api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] - requested_scopes = GM_Globals[GM_GAMSCOPES_BY_API].get(api_version, []) + requested_scopes = list(set(api_scopes.keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) + print u' %2d) %s (%d/%d scopes)' % (i, service._rootDesc[u'title'], len(requested_scopes), len(api_scopes)) if requested_scopes: for scope in requested_scopes: credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], @@ -8868,9 +8873,9 @@ def doRequestOAuth(): for _, api_version in sorted(api_titles.items()): api_index.append(api_version) i = len(api_index) - if GM_Globals[GM_GAMSCOPES_BY_API]: + if GM_Globals[GM_GAMSCOPES_LIST]: for api in all_apis: - all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES_BY_API][api] + all_apis[api][u'use_scopes'] = list(set(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) else: _select_default_scopes(all_apis) if not GC_Values[GC_ADMIN]: @@ -8902,20 +8907,18 @@ def doRequestOAuth(): for api in all_apis.keys(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: # continue - GM_Globals[GM_GAMSCOPES_BY_API] = {} - selected_scopes = [] + GM_Globals[GM_GAMSCOPES_LIST] = [] for api in all_apis.keys(): - GM_Globals[GM_GAMSCOPES_BY_API][api] = all_apis[api][u'use_scopes'] - selected_scopes += all_apis[api][u'use_scopes'] - selected_scopes = set(selected_scopes) - if not selected_scopes: + GM_Globals[GM_GAMSCOPES_LIST] += all_apis[api][u'use_scopes'] + GM_Globals[GM_GAMSCOPES_LIST] = list(set(GM_Globals[GM_GAMSCOPES_LIST])) + if not GM_Globals[GM_GAMSCOPES_LIST]: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue - writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps({u'scopes': GM_Globals[GM_GAMSCOPES_BY_API], + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps({u'scopes': GM_Globals[GM_GAMSCOPES_LIST], u'admin': GC_Values[GC_ADMIN], u'domain': GC_Values[GC_DOMAIN]})) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) - print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(selected_scopes), u','.join(selected_scopes)) + print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST]), u','.join(GM_Globals[GM_GAMSCOPES_LIST])) return elif selection == i+2: # cancel return From ee517c1800f63c6d0c2007bf47d37fc210961ec3 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 14 Jan 2016 05:53:28 -0800 Subject: [PATCH 088/100] Do better sort of API names in doRequestOAuth --- src/gam.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gam.py b/src/gam.py index 6e30cc4d..e2d2553b 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8755,7 +8755,7 @@ def OAuthInfo(): _, http, service = getAPIversionHttpService(api) api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] requested_scopes = list(set(api_scopes.keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) - print u' %2d) %s (%d/%d scopes)' % (i, service._rootDesc[u'title'], len(requested_scopes), len(api_scopes)) + print u' %2d) %s (%d/%d scopes)' % (i, service._rootDesc[u'title'].replace(u'Google ', u''), len(requested_scopes), len(api_scopes)) if requested_scopes: for scope in requested_scopes: credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], @@ -8862,13 +8862,13 @@ def doRequestOAuth(): else: print u'ERROR: please enter numbers only' - apis = API_VER_MAPPING.keys() all_apis = {} api_titles = {} - for api in apis: + for api in API_VER_MAPPING.keys(): api_version, _, service = getAPIversionHttpService(api) all_apis[api_version] = service._rootDesc - api_titles[api_version] = api_version + all_apis[api_version][u'title'] = all_apis[api_version][u'title'].replace(u'Google ', u'') + api_titles[all_apis[api_version][u'title']] = api_version api_index = [] for _, api_version in sorted(api_titles.items()): api_index.append(api_version) From 77ebba9c626c47ead569198d3df9a4a60334c01a Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 14 Jan 2016 06:14:51 -0800 Subject: [PATCH 089/100] Drop temporary environment variable GAM_ADMIN Admin email address comes from user via prompt, stored in gamscopes.json --- src/gam.py | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/gam.py b/src/gam.py index e2d2553b..a9dff64c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -92,6 +92,8 @@ GM_BATCH_QUEUE = u'batq' GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json GM_GAMSCOPES_LIST = u'scop' +# GAM admin user +GM_ADMIN = u'admin' # Current API user GM_CURRENT_API_USER = u'capu' # Current API scope @@ -119,6 +121,7 @@ GM_Globals = { GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, GM_GAMSCOPES_LIST: [], + GM_ADMIN: None, GM_CURRENT_API_USER: None, GM_CURRENT_API_SCOPES: [], GM_OAUTH2SERVICE_KEY: None, @@ -135,8 +138,6 @@ GM_Globals = { # # When retrieving lists of Google Drive activities from API, how many should be retrieved in each chunk GC_ACTIVITY_MAX_RESULTS = u'activity_max_results' -# GAM admin user -GC_ADMIN = u'admin' # Automatically generate gam batch command if number of users specified in gam users xxx command exceeds this number # Default: 0, don't automatically generate gam batch commands GC_AUTO_BATCH_MIN = u'auto_batch_min' @@ -179,7 +180,6 @@ GC_USER_MAX_RESULTS = u'user_max_results' GC_Defaults = { GC_ACTIVITY_MAX_RESULTS: 100, - GC_ADMIN: u'', GC_AUTO_BATCH_MIN: 0, GC_CACHE_DIR: u'', GC_CHARSET: u'utf-8', @@ -218,7 +218,6 @@ GC_VAR_LIMITS_KEY = u'lmit' GC_VAR_INFO = { GC_ACTIVITY_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_ACTIVITY_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 500)}, - GC_ADMIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GAM_ADMIN'}, GC_AUTO_BATCH_MIN: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_AUTOBATCH', GC_VAR_LIMITS_KEY: (0, None)}, GC_CACHE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMCACHEDIR'}, GC_CHARSET: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GAM_CHARSET'}, @@ -429,10 +428,10 @@ def writeFile(filename, data, mode=u'wb', continueOnError=False, displayError=Tr # Get global domain from global admin email address # def getDomainFromAdmin(): - if GC_Values[GC_ADMIN]: - loc = GC_Values[GC_ADMIN].find(u'@') + if GM_Globals[GM_ADMIN]: + loc = GM_Globals[GM_ADMIN].find(u'@') if loc > 0: - GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][loc+1:] + GC_Values[GC_DOMAIN] = GM_Globals[GM_ADMIN][loc+1:] # Set global variables # Check for GAM updates based on status of noupdatecheck.txt @@ -470,8 +469,8 @@ def SetGlobalVariables(): if not isinstance(scopes, list): invalidJSONExit(GC_Values[GC_GAMSCOPES_JSON]) GM_Globals[GM_GAMSCOPES_LIST] = list(set(scopes)) - if not GC_Values[GC_ADMIN]: - GC_Values[GC_ADMIN] = json_data.get(u'admin', GC_Defaults[GC_ADMIN]) + if not GM_Globals[GM_ADMIN]: + GM_Globals[GM_ADMIN] = json_data.get(u'admin', None) if not GC_Values[GC_DOMAIN]: GC_Values[GC_DOMAIN] = json_data.get(u'domain', GC_Defaults[GC_DOMAIN]) @@ -512,7 +511,6 @@ def SetGlobalVariables(): GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' _getOldEnvVar(GC_GAMSCOPES_JSON) _getOldEnvVar(GC_DOMAIN) - _getOldEnvVar(GC_ADMIN) _getOldEnvVar(GC_CUSTOMER_ID) _getOldEnvVar(GC_CHARSET) _getOldEnvVar(GC_NUM_THREADS) @@ -614,7 +612,7 @@ def handleOAuthTokenError(e, soft_errors): if e.message in OAUTH_TOKEN_ERRORS: if not GM_Globals[GM_CURRENT_API_USER]: sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], - u','.join(GM_Globals[GM_CURRENT_API_SCOPES]), GC_Values[GC_ADMIN]))) + u','.join(GM_Globals[GM_CURRENT_API_SCOPES]), GM_Globals[GM_ADMIN]))) systemErrorExit(12, MESSAGE_API_ACCESS_CONFIG) else: systemErrorExit(19, MESSAGE_SERVICE_NOT_APPLICABLE.format(GM_Globals[GM_CURRENT_API_USER])) @@ -626,7 +624,7 @@ def handleOAuthTokenError(e, soft_errors): def getGDataOAuthToken(gdataObject): credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) + scope=GM_Globals[GM_CURRENT_API_SCOPES], user_agent=GAM_INFO, sub=GM_Globals[GM_ADMIN]) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR]) try: @@ -888,7 +886,7 @@ def getAPIversionHttpService(api): invalidJSONExit(disc_file) def buildGAPIObject(api, act_as=None, soft_errors=False): - svcsub = act_as if act_as else GC_Values[GC_ADMIN] + svcsub = act_as if act_as else GM_Globals[GM_ADMIN] _, http, service = getAPIversionHttpService(api) GM_Globals[GM_CURRENT_API_USER] = act_as GM_Globals[GM_CURRENT_API_SCOPES] = list(set(service._rootDesc[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) @@ -6316,7 +6314,7 @@ def doGetUserInfo(user_email=None): user_email = sys.argv[3] i = 4 else: - user_email = GC_Values[GC_ADMIN] + user_email = GM_Globals[GM_ADMIN] if user_email[:4].lower() == u'uid:': user_email = user_email[4:] elif user_email.find(u'@') == -1: @@ -8748,7 +8746,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa def OAuthInfo(): configRequired = False - print u'API Access, Admin: {0}'.format(GC_Values[GC_ADMIN]) + print u'API Access, Admin: {0}'.format(GM_Globals[GM_ADMIN]) i = 0 for api in sorted(API_VER_MAPPING.keys()): i += 1 @@ -8760,7 +8758,7 @@ def OAuthInfo(): for scope in requested_scopes: credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) + scope=scope, user_agent=GAM_INFO, sub=GM_Globals[GM_ADMIN]) try: service._http = credentials.refresh(http) status = u'Authorized' @@ -8810,19 +8808,19 @@ def doRequestOAuth(): if json_string: try: json_data = json.loads(json_string) - GC_Values[GC_ADMIN] = json_data.get(u'id_token', {}).get(u'email', GC_Defaults[GC_ADMIN]) + GM_Globals[GM_ADMIN] = json_data.get(u'id_token', {}).get(u'email', None) if not GC_Values[GC_DOMAIN]: GC_Values[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', GC_Defaults[GC_DOMAIN]) except ValueError: pass - if GC_Values[GC_ADMIN]: + if GM_Globals[GM_ADMIN]: return print u'' while True: value = raw_input(u'Enter Admin email address: ').strip().lower() ema = EMAIL_PATTERN.match(value) if ema: - GC_Values[GC_ADMIN] = value + GM_Globals[GM_ADMIN] = value if not GC_Values[GC_DOMAIN]: GC_Values[GC_DOMAIN] = ema.group(2) return @@ -8878,7 +8876,7 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = list(set(all_apis[api][u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) else: _select_default_scopes(all_apis) - if not GC_Values[GC_ADMIN]: + if not GM_Globals[GM_ADMIN]: _getAdminDomain() while True: #os.system([u'clear', u'cls'][GM_Globals[GM_WINDOWS]]) @@ -8915,7 +8913,7 @@ def doRequestOAuth(): print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps({u'scopes': GM_Globals[GM_GAMSCOPES_LIST], - u'admin': GC_Values[GC_ADMIN], + u'admin': GM_Globals[GM_ADMIN], u'domain': GC_Values[GC_DOMAIN]})) print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) print MESSAGE_PLEASE_AUTHORIZE_SERVICE_ACCOUNT.format(len(GM_Globals[GM_GAMSCOPES_LIST]), u','.join(GM_Globals[GM_GAMSCOPES_LIST])) From 43adae4e70bf719f3f7f8544da472f948b8abae4 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 14 Jan 2016 07:38:51 -0800 Subject: [PATCH 090/100] Eliminate superflous exception --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index a9dff64c..e6c210c1 100755 --- a/src/gam.py +++ b/src/gam.py @@ -910,7 +910,7 @@ def initGDataObject(gdataObj, api): GM_Globals[GM_CURRENT_API_USER] = None try: GM_Globals[GM_CURRENT_API_SCOPES] = list(set(discovery[u'auth'][u'oauth2'][u'scopes'].keys()).intersection(GM_Globals[GM_GAMSCOPES_LIST])) - except (ValueError, KeyError): + except KeyError: invalidJSONExit(disc_file) if not GM_Globals[GM_CURRENT_API_SCOPES]: systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(discovery.get(u'title', api_version))) From b817bd04ec1b67bc232e89664f82f88ae6cd2409 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 14 Jan 2016 15:29:57 -0800 Subject: [PATCH 091/100] Handle "all users in domain" member in doPrintGroups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get id in members list, use that if there is no email. If neither email or id exist, give the “Not sure…” message --- src/gam.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/gam.py b/src/gam.py index e6c210c1..d8c69c3a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -7710,7 +7710,7 @@ def doPrintGroups(): sys.stderr.write(u' Getting %s for %s (%s of %s)\n' % (roles, group_vals[u'email'], count, total_groups)) page_message = u'Got %%num_items%% members: %%first_item%% - %%last_item%%\n' all_group_members = callGAPIpages(cd.members(), u'list', u'members', page_message=page_message, - message_attribute=u'email', groupKey=group_vals[u'email'], roles=roles, fields=u'nextPageToken,members(email,role)') + message_attribute=u'email', groupKey=group_vals[u'email'], roles=roles, fields=u'nextPageToken,members(email,id,role)') if members: all_true_members = list() if managers: @@ -7718,10 +7718,9 @@ def doPrintGroups(): if owners: all_owners = list() for member in all_group_members: - try: - member_email = member[u'email'] - except KeyError: - sys.stderr.write(u' Not sure to do with: %s' % member) + member_email = member.get(u'email', member.get(u'id', None)) + if not member_email: + sys.stderr.write(u' Not sure what to do with: %s' % member) continue try: if members and member[u'role'] == u'MEMBER': From 4a168d16a3af60dddec483f70b2e1f31b6418b50 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 15 Jan 2016 14:28:19 -0800 Subject: [PATCH 092/100] Get environment variables, signal files via table; Fix update groups to allow IDs --- src/gam.py | 113 +++++++++++++++++++++++++++-------------------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/src/gam.py b/src/gam.py index d8c69c3a..d8d707cb 100755 --- a/src/gam.py +++ b/src/gam.py @@ -159,6 +159,8 @@ GC_DOMAIN = u'domain' GC_DRIVE_DIR = u'drive_dir' # When retrieving lists of Drive files/folders from API, how many should be retrieved in each chunk GC_DRIVE_MAX_RESULTS = u'drive_max_results' +# Path to extra_args.txt +GC_EXTRA_ARGS = u'extra_args' # Path to gamscopes.json GC_GAMSCOPES_JSON = u'gamscopes_json' # If no_browser is False, output_csv won't open a browser when todrive is set @@ -190,6 +192,7 @@ GC_Defaults = { GC_DOMAIN: u'', GC_DRIVE_DIR: u'', GC_DRIVE_MAX_RESULTS: 1000, + GC_EXTRA_ARGS: u'', GC_GAMSCOPES_JSON: FN_GAMSCOPES_JSON, GC_NO_BROWSER: FALSE, GC_NO_CACHE: FALSE, @@ -215,6 +218,7 @@ GC_TYPE_STRING = u'stri' GC_VAR_TYPE_KEY = u'type' GC_VAR_ENVVAR_KEY = u'enva' GC_VAR_LIMITS_KEY = u'lmit' +GC_VAR_SFFT_KEY = u'sfft' GC_VAR_INFO = { GC_ACTIVITY_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_ACTIVITY_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 500)}, @@ -223,16 +227,17 @@ GC_VAR_INFO = { GC_CHARSET: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GAM_CHARSET'}, GC_CONFIG_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMUSERCONFIGDIR'}, GC_CUSTOMER_ID: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'CUSTOMER_ID'}, - GC_DEBUG_LEVEL: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'debug.gam', GC_VAR_LIMITS_KEY: (0, None)}, + GC_DEBUG_LEVEL: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'debug.gam', GC_VAR_LIMITS_KEY: (0, None), GC_VAR_SFFT_KEY: (0, 4)}, GC_DEVICE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_DEVICE_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 1000)}, GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING, GC_VAR_ENVVAR_KEY: u'GA_DOMAIN'}, GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMDRIVEDIR'}, GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_DRIVE_MAX_RESULTS', GC_VAR_LIMITS_KEY: (1, 1000)}, + GC_EXTRA_ARGS: {GC_VAR_TYPE_KEY: GC_TYPE_FILE, GC_VAR_ENVVAR_KEY: FN_EXTRA_ARGS_TXT, GC_VAR_SFFT_KEY: (u'', FN_EXTRA_ARGS_TXT)}, GC_GAMSCOPES_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE, GC_VAR_ENVVAR_KEY: u'GAMSCOPESFILE'}, - GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'nobrowser.txt'}, - GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'nocache.txt'}, - GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'noupdatecheck.txt'}, - GC_NO_VERIFY_SSL: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'noverifyssl.txt'}, + GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'nobrowser.txt', GC_VAR_SFFT_KEY: (False, True)}, + GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'nocache.txt', GC_VAR_SFFT_KEY: (False, True)}, + GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'noupdatecheck.txt', GC_VAR_SFFT_KEY: (False, True)}, + GC_NO_VERIFY_SSL: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN, GC_VAR_ENVVAR_KEY: u'noverifyssl.txt', GC_VAR_SFFT_KEY: (False, True)}, GC_NUM_THREADS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_ENVVAR_KEY: u'GAM_THREADS', GC_VAR_LIMITS_KEY: (1, None)}, GC_OAUTH2SERVICE_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE, GC_VAR_ENVVAR_KEY: u'OAUTHSERVICEFILE'}, GC_SITE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY, GC_VAR_ENVVAR_KEY: u'GAMSITECONFIGDIR'}, @@ -438,23 +443,23 @@ def getDomainFromAdmin(): # def SetGlobalVariables(): - def _getOldEnvVar(itemName): - value = os.environ.get(GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY], GC_Defaults[itemName]) - if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_INTEGER: - try: - number = int(value) - minVal, maxVal = GC_VAR_INFO[itemName][GC_VAR_LIMITS_KEY] - if number < minVal: - number = minVal - elif maxVal and (number > maxVal): - number = maxVal - except ValueError: - number = GC_Defaults[itemName] - value = number - GC_Defaults[itemName] = value - - def _getOldSignalFile(itemName, trueValue=True, falseValue=False): - GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY])) else falseValue + def _getDefault(itemName, itemEntry): + if GC_VAR_SFFT_KEY in itemEntry: + GC_Defaults[itemName] = itemEntry[GC_VAR_SFFT_KEY][os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], itemEntry[GC_VAR_ENVVAR_KEY]))] + else: + value = os.environ.get(itemEntry[GC_VAR_ENVVAR_KEY], GC_Defaults[itemName]) + if itemEntry[GC_VAR_TYPE_KEY] == GC_TYPE_INTEGER: + try: + number = int(value) + minVal, maxVal = itemEntry[GC_VAR_LIMITS_KEY] + if number < minVal: + number = minVal + elif maxVal and (number > maxVal): + number = maxVal + except ValueError: + number = GC_Defaults[itemName] + value = str(number) + GC_Defaults[itemName] = value def _getScopesAdminDomainFromGamScopesJson(): GM_Globals[GM_GAMSCOPES_LIST] = [] @@ -501,29 +506,14 @@ def SetGlobalVariables(): GC_Defaults[GC_CACHE_DIR] = os.path.join(GM_Globals[GM_GAM_PATH], u'gamcache') GC_Defaults[GC_DRIVE_DIR] = GM_Globals[GM_GAM_PATH] GC_Defaults[GC_SITE_DIR] = GM_Globals[GM_GAM_PATH] - - _getOldEnvVar(GC_CONFIG_DIR) - _getOldEnvVar(GC_SITE_DIR) - _getOldEnvVar(GC_CACHE_DIR) - _getOldEnvVar(GC_DRIVE_DIR) - _getOldEnvVar(GC_OAUTH2SERVICE_JSON) + for itemName, itemEntry in GC_VAR_INFO.items(): + if itemEntry[GC_VAR_TYPE_KEY] == GC_TYPE_DIRECTORY: + _getDefault(itemName, itemEntry) + for itemName, itemEntry in GC_VAR_INFO.items(): + if itemEntry[GC_VAR_TYPE_KEY] != GC_TYPE_DIRECTORY: + _getDefault(itemName, itemEntry) if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' - _getOldEnvVar(GC_GAMSCOPES_JSON) - _getOldEnvVar(GC_DOMAIN) - _getOldEnvVar(GC_CUSTOMER_ID) - _getOldEnvVar(GC_CHARSET) - _getOldEnvVar(GC_NUM_THREADS) - _getOldEnvVar(GC_AUTO_BATCH_MIN) - _getOldEnvVar(GC_ACTIVITY_MAX_RESULTS) - _getOldEnvVar(GC_DEVICE_MAX_RESULTS) - _getOldEnvVar(GC_DRIVE_MAX_RESULTS) - _getOldEnvVar(GC_USER_MAX_RESULTS) - _getOldSignalFile(GC_DEBUG_LEVEL, trueValue=4, falseValue=0) - _getOldSignalFile(GC_NO_VERIFY_SSL) - _getOldSignalFile(GC_NO_BROWSER) - _getOldSignalFile(GC_NO_CACHE) - _getOldSignalFile(GC_NO_UPDATE_CHECK) # Assign directories first for itemName in GC_VAR_INFO: if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_DIRECTORY: @@ -538,13 +528,13 @@ def SetGlobalVariables(): if not GC_Values[GC_NO_UPDATE_CHECK]: doGAMCheckForUpdates() # Globals derived from config file values - GM_Globals[GM_EXTRA_ARGS_DICT] = {u'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} httplib2.debuglevel = GC_Values[GC_DEBUG_LEVEL] - if os.path.isfile(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)): + GM_Globals[GM_EXTRA_ARGS_DICT] = {u'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} + if GC_Values[GC_EXTRA_ARGS]: import ConfigParser ea_config = ConfigParser.ConfigParser() ea_config.optionxform = str - ea_config.read(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)) + ea_config.read(GC_Values[GC_EXTRA_ARGS]) GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) GM_Globals[GM_OAUTH2SERVICE_KEY] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL] = None @@ -5950,9 +5940,15 @@ def doUpdateGroup(): users_email = getUsersToModify(entity_type=sys.argv[i], entity=sys.argv[i+1]) else: users_email = [sys.argv[i],] + body = {u'role': role} for user_email in users_email: - if user_email != u'*' and user_email.find(u'@') == -1: - user_email = u'%s@%s' % (user_email, GC_Values[GC_DOMAIN]) + if user_email[:4].lower() == u'uid:': + user_email = user_email[4:] + body[u'id'] = user_email + else: + if user_email.find(u'@') == -1: + user_email = u'%s@%s' % (user_email, GC_Values[GC_DOMAIN]) + body[u'email'] = user_email sys.stderr.write(u' %sing %s %s...' % (sys.argv[4].lower(), role.lower(), user_email)) try: if sys.argv[4].lower() == u'add': @@ -5960,14 +5956,17 @@ def doUpdateGroup(): body[u'email'] = user_email result = callGAPI(cd.members(), u'insert', soft_errors=True, groupKey=group, body=body) elif sys.argv[4].lower() == u'update': - result = callGAPI(cd.members(), u'update', soft_errors=True, groupKey=group, memberKey=user_email, body={u'email': user_email, u'role': role}) - try: - if str(result[u'email']).lower() != user_email.lower(): - print u'added %s (primary address) to group' % result[u'email'] + result = callGAPI(cd.members(), u'update', soft_errors=True, groupKey=group, memberKey=user_email, body=body) + if result: + addr = result.get(u'email', None) + if addr: + addr = addr.lower() + if addr != user_email.lower(): + print u'added %s (primary address) to group' % addr + else: + print u'added %s to group' % addr else: - print u'added %s to group' % result[u'email'] - except TypeError: - pass + print u'added %s to group' % result[u'id'] except googleapiclient.errors.HttpError: pass elif sys.argv[4].lower() == u'sync': @@ -6000,7 +5999,9 @@ def doUpdateGroup(): else: user_emails = [sys.argv[i],] for user_email in user_emails: - if user_email != u'*' and user_email.find(u'@') == -1: + if user_email[:4].lower() == u'uid:': + user_email = user_email[4:] + elif user_email.find(u'@') == -1: user_email = u'%s@%s' % (user_email, GC_Values[GC_DOMAIN]) sys.stderr.write(u' removing %s\n' % user_email) result = callGAPI(cd.members(), u'delete', soft_errors=True, groupKey=group, memberKey=user_email) From f9bd5506c79d860cc3da14a872f72954c5c5463a Mon Sep 17 00:00:00 2001 From: jeremi Date: Fri, 22 Jan 2016 10:25:59 +0800 Subject: [PATCH 093/100] Setting up vacation was not working anymore The underlying function enable as a UpdateVacation is exception a boolean and not a string. --- src/gam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index bb7bbebd..3130e8b7 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4884,9 +4884,9 @@ def doWebClips(users): def doVacation(users): subject = message = u'' if sys.argv[4].lower() in true_values: - enable = u'true' + enable = True elif sys.argv[4].lower() in false_values: - enable = u'false' + enable = False else: print u'ERROR: value for "gam vacation" must be true or false, got %s' % sys.argv[4] sys.exit(2) From 9c368b7d1003276d23d0b29076260f66c31a1a4a Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 22 Jan 2016 07:25:01 -0800 Subject: [PATCH 094/100] Fix handling of nonexistent extra-args.txt --- src/gam.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gam.py b/src/gam.py index d8d707cb..bee07f46 100755 --- a/src/gam.py +++ b/src/gam.py @@ -484,6 +484,8 @@ def SetGlobalVariables(): def _getCfgFile(itemName): value = os.path.expanduser(GC_Defaults[itemName]) + if (not value) and (itemName == GC_EXTRA_ARGS): + return value if not os.path.isabs(value): value = os.path.expanduser(os.path.join(GC_Values[GC_CONFIG_DIR], value)) return value @@ -499,6 +501,8 @@ def SetGlobalVariables(): for itemName in GC_VAR_INFO: if GC_VAR_INFO[itemName][GC_VAR_TYPE_KEY] == GC_TYPE_FILE: fileName = GC_Values[itemName] + if (not fileName) and (itemName == GC_EXTRA_ARGS): + continue if not os.path.isfile(fileName): sys.stderr.write(u'{0}{1}={2}, Not Found\n'.format(WARNING_PREFIX, GC_VAR_INFO[itemName][GC_VAR_ENVVAR_KEY], fileName)) From 2704a8b69506d7b0c57edc78ec211436caef32a6 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 27 Jan 2016 09:25:21 -0500 Subject: [PATCH 095/100] groups and mobile reports gam report groups gam report mobile (always blank?) --- src/gam.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index bee07f46..27450310 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1079,7 +1079,13 @@ def showReport(): for app in auth_apps: # put apps at bottom cust_attributes.append(app) output_csv(csv_list=cust_attributes, titles=titles, list_type=u'Customer Report - %s' % try_date, todrive=to_drive) - elif report in [u'doc', u'docs', u'calendar', u'calendars', u'login', u'logins', u'admin', u'drive', u'token', u'tokens']: + elif report in [u'doc', u'docs', u'drive', + u'calendar', u'calendars', + u'login', u'logins', + u'admin', + u'token', u'tokens', + u'group', u'groups', + u'mobile']: if report in [u'doc', u'docs']: report = u'drive' elif report in [u'calendars']: @@ -1088,6 +1094,8 @@ def showReport(): report = u'login' elif report == u'tokens': report = u'token' + elif report == u'group': + report = u'groups' page_message = u'Got %%num_items%% items\n' activities = callGAPIpages(rep.activities(), u'list', u'items', page_message=page_message, applicationName=report, userKey=userKey, customerId=customerId, actorIpAddress=actorIpAddress, From 10fcf566b8d5297836903a53ff0f30e13a15d4ed Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 27 Jan 2016 09:32:48 -0500 Subject: [PATCH 096/100] fix todrive for "gam print adminroles" --- src/gam.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index 27450310..1b6015df 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1619,6 +1619,15 @@ def doPrintAdminRoles(): roles = callGAPIpages(cd.roles(), u'list', u'items', customer=GC_Values[GC_CUSTOMER_ID]) roles_attrib = [{}] + todrive = False + i = 3 + while i < len(sys.argv): + if sys.argv[i].lower() == u'todrive': + todrive = True + i += 1 + else: + print u'ERROR: %s is not a valid argument for "gam print adminroles".' % sys.argv[i] + sys.exit(2) for role in roles: role_attrib = {} for key, value in role.items(): @@ -1630,7 +1639,7 @@ def doPrintAdminRoles(): roles_attrib[0][key] = key role_attrib[key] = value roles_attrib.append(role_attrib) - output_csv(roles_attrib, roles_attrib[0], u'Admin Roles', False) + output_csv(roles_attrib, roles_attrib[0], u'Admin Roles', todrive) def doPrintAdmins(): cd = buildGAPIObject(u'directory') From 0a41b4ec68ff14559f0892344d41855993d37e8f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 27 Jan 2016 10:41:17 -0800 Subject: [PATCH 097/100] Optionally get admin email address from command line in doRequestOauth --- src/gam.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/gam.py b/src/gam.py index 1b6015df..2a0462c8 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8814,38 +8814,28 @@ def doDeleteOAuth(): sys.stderr.write(u'{0}{1}\n'.format(WARNING_PREFIX, e)) EMAIL_PATTERN = re.compile(r'^(.+)@(.+\..+)$') -EMAIL_FORMAT_REQUIRED = u'@.' +EMAIL_FORMAT_REQUIRED = u'@' UBER_SCOPES = {u'gmail-v1': [u'https://mail.google.com/'],} def doRequestOAuth(): + def _setAdminDomain(value): + ema = EMAIL_PATTERN.match(value) + if ema: + GM_Globals[GM_ADMIN] = value + if not GC_Values[GC_DOMAIN]: + GC_Values[GC_DOMAIN] = ema.group(2) + return True + print u'{0}Admin email address must be: {1}'.format(ERROR_PREFIX, EMAIL_FORMAT_REQUIRED) + return False + def _getAdminDomain(): - srcFile = os.path.expanduser(os.environ.get(u'OAUTHFILE', u'oauth2.txt')) - if not os.path.isabs(srcFile): - srcFile = os.path.expanduser(os.path.join(GC_Values[GC_CONFIG_DIR], srcFile)) - if os.path.isfile(srcFile): - json_string = readFile(srcFile, continueOnError=True, displayError=False) - if json_string: - try: - json_data = json.loads(json_string) - GM_Globals[GM_ADMIN] = json_data.get(u'id_token', {}).get(u'email', None) - if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = json_data.get(u'id_token', {}).get(u'hd', GC_Defaults[GC_DOMAIN]) - except ValueError: - pass - if GM_Globals[GM_ADMIN]: - return print u'' while True: value = raw_input(u'Enter Admin email address: ').strip().lower() - ema = EMAIL_PATTERN.match(value) - if ema: - GM_Globals[GM_ADMIN] = value - if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = ema.group(2) + if _setAdminDomain(value): return - print u'{0}Enter full email address: {1}'.format(ERROR_PREFIX, EMAIL_FORMAT_REQUIRED) def _select_default_scopes(apis): for api_name, api in apis.items(): @@ -8881,6 +8871,12 @@ def doRequestOAuth(): else: print u'ERROR: please enter numbers only' + if len(sys.argv) > 3: + GM_Globals[GM_ADMIN] = u'' + if not _setAdminDomain(sys.argv[3].lower()): + _getAdminDomain() + elif not GM_Globals[GM_ADMIN]: + _getAdminDomain() all_apis = {} api_titles = {} for api in API_VER_MAPPING.keys(): From 6519a5b007b510f9ca72b3949e5f4a79684203d5 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Mon, 29 Feb 2016 09:03:14 -0500 Subject: [PATCH 098/100] First shot at an issue template goal is to reduce # of "how do I" issues which should go to discussion forum as well as bug reports which don't include full details. --- .github/ISSUE_TEMPLATE | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE new file mode 100644 index 00000000..cf29e312 --- /dev/null +++ b/.github/ISSUE_TEMPLATE @@ -0,0 +1,15 @@ +*The issue tracker is for reporting product deficiencies. How do I questions should be posted to [the discussion forum](https://groups.google.com/group/google-apps-manager). When in doubt, start at [the discussion forum](https://groups.google.com/group/google-apps-manager) and return here to file an issue only when instructed to do so.* + +### Please confirm the following: +- I have upgraded to the [latest GAM release](http://git.io/gamreleases) and I still have this issue. +- I am typing the command as described in the [GAM Wiki](https://github.com/jay0lee/GAM/wiki) +- I am using a Google Apps for Work, Unlimited or EDU account (Google Apps Standard / Free is not supported) + +### Steps to reproduce the issue: +1. ... +1. ... +1. ... + +### Expected outcome (what are you trying to do?) + +### Actual outcome (what errors or bad behavior do you see instead?) \ No newline at end of file From a334645910923862ecc50480206d0b12d7d8dc84 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Mon, 29 Feb 2016 09:07:09 -0500 Subject: [PATCH 099/100] ISSUE_TEMPLATE is shown in plaintext on new issues --- .github/ISSUE_TEMPLATE | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index cf29e312..2a199f6f 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -1,15 +1,15 @@ -*The issue tracker is for reporting product deficiencies. How do I questions should be posted to [the discussion forum](https://groups.google.com/group/google-apps-manager). When in doubt, start at [the discussion forum](https://groups.google.com/group/google-apps-manager) and return here to file an issue only when instructed to do so.* +The issue tracker is for reporting product deficiencies. How do I questions should be posted to the discussion forum at https://groups.google.com/group/google-apps-manager. When in doubt, start at the discussion forum and return here only when instructed to do so. -### Please confirm the following: -- I have upgraded to the [latest GAM release](http://git.io/gamreleases) and I still have this issue. -- I am typing the command as described in the [GAM Wiki](https://github.com/jay0lee/GAM/wiki) -- I am using a Google Apps for Work, Unlimited or EDU account (Google Apps Standard / Free is not supported) +Please confirm the following: +* I have upgraded to the latest GAM release from http://git.io/gamreleases and I still have this issue. +* I am typing the command as described in the GAM Wiki at https://github.com/jay0lee/GAM/wiki +* I am using a Google Apps for Work, Unlimited or EDU account (Google Apps Standard / Free is not supported) -### Steps to reproduce the issue: -1. ... -1. ... -1. ... +Full steps to reproduce the issue: +1. +2. +3. -### Expected outcome (what are you trying to do?) +Expected outcome (what are you trying to do?): -### Actual outcome (what errors or bad behavior do you see instead?) \ No newline at end of file +Actual outcome (what errors or bad behavior do you see instead?): \ No newline at end of file From 0034704b3ffc22cb3fbc7c2e80f14def3695f5e2 Mon Sep 17 00:00:00 2001 From: Vincent Tertre Date: Wed, 30 Mar 2016 17:07:05 +0200 Subject: [PATCH 100/100] Handle query for drive file update --- src/gam.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 2a0462c8..7d3dfe14 100755 --- a/src/gam.py +++ b/src/gam.py @@ -3575,7 +3575,7 @@ def deleteEmptyDriveFolders(users): print convertUTF8(u' not deleting folder %s because it contains at least 1 item (%s)' % (folder[u'title'], children[u'items'][0][u'id'])) def doUpdateDriveFile(users): - convert = ocr = ocrLanguage = parent_query = local_filepath = media_body = fileIds = drivefilename = None + convert = ocr = ocrLanguage = parent_query = local_filepath = media_body = fileIds = query = drivefilename = None operation = u'update' i = 5 body = {} @@ -3595,6 +3595,9 @@ def doUpdateDriveFile(users): elif sys.argv[i].lower() == u'id': fileIds = [sys.argv[i+1],] i += 2 + elif sys.argv[i].lower() == u'query': + query = sys.argv[i+1] + i += 2 elif sys.argv[i].lower() == u'drivefilename': drivefilename = sys.argv[i+1] i += 2 @@ -3698,11 +3701,11 @@ def doUpdateDriveFile(users): else: print u'ERROR: %s is not a valid argument for "gam update drivefile"' % sys.argv[i] sys.exit(2) - if not fileIds and not drivefilename: - print u'ERROR: you need to specify either id or query in order to determine the file(s) to update' + if not fileIds and not drivefilename and not query: + print u'ERROR: you need to specify either id, query or drivefilename in order to determine the file(s) to update' sys.exit(2) - elif fileIds and drivefilename: - print u'ERROR: you cannot specify both an id and a query.' + elif (fileIds and drivefilename) or (fileIds and query) or (drivefilename and query): + print u'ERROR: you cannot specify multiple file identifiers. Choose one of id, drivefilename, query.' sys.exit(2) for user in users: drive = buildGAPIObject(u'drive', user) @@ -3712,7 +3715,9 @@ def doUpdateDriveFile(users): body[u'parents'] = list() for a_parent in more_parents: body[u'parents'].append({u'id': a_parent}) - if drivefilename: + if query: + fileIds = doDriveSearch(drive, query=query) + elif drivefilename: fileIds = doDriveSearch(drive, query=u'"me" in owners and title = "%s"' % drivefilename) if local_filepath: media_body = googleapiclient.http.MediaFileUpload(local_filepath, mimetype=mimetype, resumable=True)