From 5ae1f3c44143bb2ea3dcb7664dabf9c2d23832fc Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 11:50:17 -0800 Subject: [PATCH 01/17] Cleanup --- src/gam.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/gam.py b/src/gam.py index 5da9d282..2f80fd98 100755 --- a/src/gam.py +++ b/src/gam.py @@ -560,7 +560,7 @@ def doGAMVersion(): platform.platform(), platform.machine(), GM_Globals[GM_GAM_PATH]) -def tryOAuth(gdataObject, scope): +def tryOAuth(gdataObject, scope, soft_errors=False): scope = [scope, u'email'] credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], @@ -5330,7 +5330,6 @@ def doCreateUser(): def doCreateGroup(): cd = buildGAPIObject(u'directory') - use_gs_api = False body = dict() body[u'email'] = sys.argv[3] if body[u'email'].find(u'@') == -1: @@ -5993,7 +5992,6 @@ def doUpdateGroup(): print u'ERROR: %s is not a valid argument for "gam update group"' % sys.argv[i] sys.exit(2) gs_body[attrib] = value - use_gs_api = True i += 2 if group[:4].lower() == u'uid:': # group settings API won't take uid so we make sure cd API is used so that we can grab real email. use_cd_api = True @@ -6251,12 +6249,7 @@ def doGetUserInfo(user_email=None): user_email = sys.argv[3] i = 4 else: - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - user_email = credentials.id_token[u'email'] + user_email = GC_Values[GC_ADMIN] if user_email[:4].lower() == u'uid:': user_email = user_email[4:] elif user_email.find(u'@') == -1: From ab0bec7a7b3727be67baed4ec6ba5802f03f1096 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 12:54:35 -0800 Subject: [PATCH 02/17] Merge remote-tracking branch 'jay0lee/master' # Conflicts: # src/gam.py --- src/email-audit-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/email-settings-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/gam.py | 35 +++++++++++++++++++---------------- src/gam.spec | 2 ++ 4 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 src/email-audit-v1.json create mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json new file mode 100644 index 00000000..b617e1f5 --- /dev/null +++ b/src/email-audit-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-audit:v1", + "name": "email-audit", + "version": "v1", + "revision": "20130823", + "title": "Email Audit API", + "description": "Lets you peform Google Apps Email Audits", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-audit", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/compliance/audit/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/compliance/audit/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json new file mode 100644 index 00000000..d2ab10be --- /dev/null +++ b/src/email-settings-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-settings:v1", + "name": "email-settings", + "version": "v1", + "revision": "20130823", + "title": "Email Settings API", + "description": "Lets you manage Google Apps Email Settings", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-settings", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/emailsettings/2.0/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/gam.py b/src/gam.py index 2f80fd98..b00f1d73 100755 --- a/src/gam.py +++ b/src/gam.py @@ -73,10 +73,6 @@ FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' -GDATA_EMAIL_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/' -GDATA_ADMIN_SETTINGS_SCOPE = u'https://apps-apis.google.com/a/feeds/domain/' -GDATA_EMAIL_AUDIT_SCOPE = u'https://apps-apis.google.com/a/feeds/compliance/audit/' - # # Global variables # @@ -561,7 +557,7 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope, soft_errors=False): - scope = [scope, u'email'] + scope.append(u'email') credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file @@ -772,6 +768,8 @@ API_VER_MAPPING = { u'oauth2': u'v2', u'reports': u'reports_v1', u'siteVerification': u'v1', + u'email-settings': u'v1', + u'email-audit': u'v1' } def getAPIVer(api): @@ -782,7 +780,7 @@ def getAPIScope(service): granted_scopes = api_scopes # TODO fix to lookup from file return [val for val in api_scopes if val in granted_scopes] + [u'email'] -def getServiceFromDiscoveryDocument(api, version, http): +def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) disc_file = os.path.join(GC_Values[GC_SITE_DIR], disc_filename) if hasattr(sys, '_MEIPASS'): @@ -858,18 +856,24 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service + service = getServiceFromDiscoveryDocument(u'admin-settings', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), - GDATA_ADMIN_SETTINGS_SCOPE) + scope) def getAuditObject(): import gdata.apps.audit.service + service = getServiceFromDiscoveryDocument(u'email-audit', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.audit.service.AuditService(), - GDATA_EMAIL_AUDIT_SCOPE) + scope) def getEmailSettingsObject(): import gdata.apps.emailsettings.service + service = getServiceFromDiscoveryDocument(u'email-settings', u'v1') + scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - GDATA_EMAIL_SETTINGS_SCOPE) + scope) def geturl(url, dst): import urllib2 @@ -8713,6 +8717,7 @@ def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() apis.remove(u'oauth2') + all_apis = {} for api in apis: version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: @@ -8723,13 +8728,11 @@ def doRequestOAuth(): service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) - print u'%s: %s' % (service._rootDesc['title'], service._rootDesc['description']) - for scope in service._rootDesc[u'auth'][u'oauth2'][u'scopes'].items(): - scope_value = scope[0] - scope_description = scope[1][u'description'] - print u' %s\n %s' % (scope_value, scope_description) - print - print + all_apis[api] = service._rootDesc + i = 0 + for api in all_apis.values(): + print u'[*] %s) %s' % (i, api[u'title']) + i += 1 def batch_worker(): while True: diff --git a/src/gam.spec b/src/gam.spec index 57abac1f..83620ea1 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,6 +11,8 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] +a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] +a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From eca89ca5e9a86c15debc62935704d6e2673b1e77 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 15:17:43 -0800 Subject: [PATCH 03/17] Fix typo --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index e0c4ede0..d87f85f3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -576,7 +576,7 @@ def tryOAuth(gdataObject, scope, soft_errors=False): sys.exit(4) gdataObject.additional_headers = {u'Authorization': u'Bearer %s' % credentials.access_token} if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][GC_Values[GC_ADMIN].find(u'@'):].lower() + GC_Values[GC_DOMAIN] = GC_Values[GC_ADMIN][GC_Values[GC_ADMIN].find(u'@')+1:].lower() if not GC_Values[GC_CUSTOMER_ID]: GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER gdataObject.domain = GC_Values[GC_DOMAIN] From abde922b496df09555fe4b9e1f8b70acba4a09f8 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 15:38:39 -0800 Subject: [PATCH 04/17] Only build object if necessary --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index d87f85f3..44c94d25 100755 --- a/src/gam.py +++ b/src/gam.py @@ -7089,10 +7089,10 @@ def doGetInstanceInfo(): geturl(url, target_file) sys.exit(0) print u'Google Apps Domain: %s' % (GC_Values[GC_DOMAIN]) - cd = buildGAPIObject(u'directory') if GC_Values[GC_CUSTOMER_ID] != MY_CUSTOMER: customerId = GC_Values[GC_CUSTOMER_ID] else: + cd = buildGAPIObject(u'directory') result = callGAPI(cd.users(), u'list', fields=u'users(customerId)', customer=GC_Values[GC_CUSTOMER_ID], maxResults=1) try: From bf6c2ef2661b52850982fd43de0b3cf618f8252c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 30 Dec 2015 23:39:04 -0800 Subject: [PATCH 05/17] buildGAPIObject reworked This gets everything working but does not address the issue of matching the admins actual scope list. You'd better define GA_DOMAIN as it used to come out of oauth2.txt if it wasn't defined. I had to add Audit API and Site Verification API to the service account list of APIs and downloaded a new oauth2service.json. --- src/email-audit-v1.json | 34 -------------- src/email-settings-v1.json | 34 -------------- src/gam.py | 91 ++++++++++++++++++++++++-------------- src/gam.spec | 2 - 4 files changed, 57 insertions(+), 104 deletions(-) delete mode 100644 src/email-audit-v1.json delete mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json deleted file mode 100644 index b617e1f5..00000000 --- a/src/email-audit-v1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "kind": "discovery#restDescription", - "discoveryVersion": "v1", - "id": "email-audit:v1", - "name": "email-audit", - "version": "v1", - "revision": "20130823", - "title": "Email Audit API", - "description": "Lets you peform Google Apps Email Audits", - "ownerDomain": "google.com", - "ownerName": "Google", - "icons": { - "x16": "http://www.google.com/images/icons/product/search-16.gif", - "x32": "http://www.google.com/images/icons/product/search-32.gif" - }, - "documentationLink": "https://developers.google.com/admin-sdk/email-audit", - "protocol": "rest", - "baseUrl": "https://apps-apis.google.com/", - "rootUrl": "https://apps-apis.google.com/", - "servicePath": "/a/feeds/compliance/audit/", - "auth": { - "oauth2": { - "scopes": { - "https://apps-apis.google.com/a/feeds/compliance/audit/": { - "description": "Manage email audits" - } - } - } - }, - "schemas": { - }, - "resources": { - } -} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json deleted file mode 100644 index d2ab10be..00000000 --- a/src/email-settings-v1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "kind": "discovery#restDescription", - "discoveryVersion": "v1", - "id": "email-settings:v1", - "name": "email-settings", - "version": "v1", - "revision": "20130823", - "title": "Email Settings API", - "description": "Lets you manage Google Apps Email Settings", - "ownerDomain": "google.com", - "ownerName": "Google", - "icons": { - "x16": "http://www.google.com/images/icons/product/search-16.gif", - "x32": "http://www.google.com/images/icons/product/search-32.gif" - }, - "documentationLink": "https://developers.google.com/admin-sdk/email-settings", - "protocol": "rest", - "baseUrl": "https://apps-apis.google.com/", - "rootUrl": "https://apps-apis.google.com/", - "servicePath": "/a/feeds/emailsettings/2.0/", - "auth": { - "oauth2": { - "scopes": { - "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { - "description": "Manage email audits" - } - } - } - }, - "schemas": { - }, - "resources": { - } -} diff --git a/src/gam.py b/src/gam.py index 44c94d25..dd00497a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -104,6 +104,8 @@ GM_MAP_ROLE_ID_TO_NAME = u'ri2n' GM_MAP_ROLE_NAME_TO_ID = u'rn2i' # Dictionary mapping User ID to Name GM_MAP_USER_ID_TO_NAME = u'ui2n' +# Current API scope +GM_API_SCOPE = u'scop' # GM_Globals = { GM_SYSEXITRC: 0, @@ -120,6 +122,7 @@ GM_Globals = { GM_MAP_ROLE_ID_TO_NAME: None, GM_MAP_ROLE_NAME_TO_ID: None, GM_MAP_USER_ID_TO_NAME: None, + GM_API_SCOPE: None, } # # Global variables defined by environment variables/signal files @@ -557,12 +560,12 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope, soft_errors=False): - scope.append(u'email') +# scope.append(u'email') #TODO: What is this for? credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) # TODO lookup admin user from file + scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) + cache=GC_Values[GC_CACHE_DIR]) try: credentials.refresh(http) except oauth2client.client.AccessTokenRefreshError, e: @@ -586,7 +589,7 @@ def checkGDataError(e, service): # First check for errors that need special handling if e[0].get(u'reason', u'') in [u'Token invalid - Invalid token: Stateless token expired', u'Token invalid - Invalid token: Token not found']: keep_domain = service.domain - tryOAuth(service) + tryOAuth(service, GM_Globals[GM_API_SCOPE]) service.domain = keep_domain return False if e[0][u'body'].startswith(u'Required field must not be blank:') or e[0][u'body'].startswith(u'These characters are not allowed:'): @@ -762,19 +765,45 @@ API_VER_MAPPING = { u'datatransfer': u'datatransfer_v1', u'directory': u'directory_v1', u'drive': u'v2', + u'email-audit': u'v1', + u'email-settings': u'v1', u'gmail': u'v1', u'groupssettings': u'v1', u'licensing': u'v1', u'oauth2': u'v2', u'reports': u'reports_v1', u'siteVerification': u'v1', - u'email-settings': u'v1', - u'email-audit': u'v1' } def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') +SERVICE_API_SCOPE_MAPPING = { + u'admin-settings': [u'https://apps-apis.google.com/a/feeds/domain/',], + u'appsactivity': [u'https://www.googleapis.com/auth/activity', u'https://www.googleapis.com/auth/drive'], + u'calendar': [u'https://www.googleapis.com/auth/calendar',], + u'classroom': [u'https://www.googleapis.com/auth/classroom.courses', u'https://www.googleapis.com/auth/classroom.profile.emails', u'https://www.googleapis.com/auth/classroom.profile.photos',], + u'cloudprint': [u'https://www.googleapis.com/auth/cloudprint',], + u'datatransfer': [u'https://www.googleapis.com/auth/admin.datatransfer',], + u'drive': [u'https://www.googleapis.com/auth/drive',], + u'directory': [u'https://www.googleapis.com/auth/admin.datatransfer', u'https://www.googleapis.com/auth/admin.directory.customer', u'https://www.googleapis.com/auth/admin.directory.device.chromeos', + u'https://www.googleapis.com/auth/admin.directory.device.mobile', u'https://www.googleapis.com/auth/admin.directory.device.mobile.action', + u'https://www.googleapis.com/auth/admin.directory.domain', u'https://www.googleapis.com/auth/admin.directory.group', u'https://www.googleapis.com/auth/admin.directory.group.member', + u'https://www.googleapis.com/auth/admin.directory.notifications', u'https://www.googleapis.com/auth/admin.directory.orgunit', u'https://www.googleapis.com/auth/admin.directory.resource.calendar', + u'https://www.googleapis.com/auth/admin.directory.rolemanagement', u'https://www.googleapis.com/auth/admin.directory.user', u'https://www.googleapis.com/auth/admin.directory.user.alias', + u'https://www.googleapis.com/auth/admin.directory.user.security', u'https://www.googleapis.com/auth/admin.directory.userschema',], + u'email-audit': [u'https://apps-apis.google.com/a/feeds/compliance/audit/',], + u'email-settings': [u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/',], + u'gmail': [u'https://mail.google.com/',], + u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], + u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], + u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], + u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], + } + +def getServiceAPIScope(api): + return SERVICE_API_SCOPE_MAPPING.get(api, []) + def getAPIScope(service): api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] granted_scopes = api_scopes # TODO fix to lookup from file @@ -795,9 +824,7 @@ def getServiceFromDiscoveryDocument(api, version, http=None): systemErrorExit(4, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) return googleapiclient.discovery.build_from_document(discovery, base=u'https://www.googleapis.com', http=http) -def buildGAPIObject(api, act_as=None, soft_errors=False): - if not act_as: - act_as = GC_Values[GC_ADMIN] # TODO lookup admin user from file +def getOAuth2ServiceDetails(): if not GM_Globals[GM_OAUTH2SERVICE_KEY]: json_string = readFile(GC_Values[GC_OAUTH2SERVICE_JSON], continueOnError=True, displayError=True) if not json_string: @@ -814,39 +841,41 @@ def buildGAPIObject(api, act_as=None, soft_errors=False): printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) systemErrorExit(17, MESSAGE_OAUTH2SERVICE_JSON_INVALID.format(GC_Values[GC_OAUTH2SERVICE_JSON])) + +def buildGAPIObject(api, act_as=None, soft_errors=False): + sub = act_as if act_as else GC_Values[GC_ADMIN] + getOAuth2ServiceDetails() + GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api) + credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], + GM_Globals[GM_OAUTH2SERVICE_KEY], + scope=GM_Globals[GM_API_SCOPE], user_agent=GAM_INFO, sub=sub) + http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GC_Values[GC_CACHE_DIR])) version = getAPIVer(api) if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) try: - service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + return googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: - service = getServiceFromDiscoveryDocument(api, version, http) + return getServiceFromDiscoveryDocument(api, version, http) except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) - scope = getAPIScope(service) - credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], - GM_Globals[GM_OAUTH2SERVICE_KEY], - scope=scope, user_agent=GAM_INFO, sub=act_as) - try: - service._http = credentials.authorize(http) - service._http.request.credentials.refresh(http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.', u'access_denied: Requested client not authorized.']: - systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(scope))) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], u','.join(GM_Globals[GM_API_SCOPE]))) sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) if soft_errors: return False sys.exit(4) - return service def commonAppsObjInit(appsObj, scope): - if not tryOAuth(appsObj, scope): + getOAuth2ServiceDetails() + GM_Globals[GM_API_SCOPE] = scope + if not tryOAuth(appsObj, GM_Globals[GM_API_SCOPE]): doRequestOAuth() - tryOAuth(appsObj, scope) + tryOAuth(appsObj, GM_Globals[GM_API_SCOPE]) #Identify GAM to Google's Servers appsObj.source = GAM_INFO #Show debugging output if debug.gam exists @@ -856,24 +885,18 @@ def commonAppsObjInit(appsObj, scope): def getAdminSettingsObject(): import gdata.apps.adminsettings.service - service = getServiceFromDiscoveryDocument(u'admin-settings', u'v1') - scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.adminsettings.service.AdminSettingsService(), - scope) + getServiceAPIScope(u'admin-settings')) def getAuditObject(): import gdata.apps.audit.service - service = getServiceFromDiscoveryDocument(u'email-audit', u'v1') - scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.audit.service.AuditService(), - scope) + getServiceAPIScope(u'email-audit')) def getEmailSettingsObject(): import gdata.apps.emailsettings.service - service = getServiceFromDiscoveryDocument(u'email-settings', u'v1') - scope = service._rootDesc[u'auth'][u'oauth2']['scopes'].keys() return commonAppsObjInit(gdata.apps.emailsettings.service.EmailSettingsService(), - scope) + getServiceAPIScope(u'email-settings')) def geturl(url, dst): import urllib2 @@ -8747,7 +8770,7 @@ def doRequestOAuth(): if api in [u'directory', u'reports', u'datatransfer']: api = u'admin' http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) + cache=GC_Values[GC_CACHE_DIR]) try: service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: diff --git a/src/gam.spec b/src/gam.spec index 83620ea1..57abac1f 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,8 +11,6 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] -a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] -a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From b037333d2bfe62f82e742a2007ec3321a92bfa8c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 01:01:05 -0800 Subject: [PATCH 06/17] Clean up OAuthInfo --- src/gam.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/gam.py b/src/gam.py index dd00497a..2a986a1c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -797,6 +797,7 @@ SERVICE_API_SCOPE_MAPPING = { u'gmail': [u'https://mail.google.com/',], u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], + u'oauth2': [u'https://www.googleapis.com/auth/plus.login', u'https://www.googleapis.com/auth/plus.me',], u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], } @@ -8707,20 +8708,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): - if len(sys.argv) > 3: - access_token = sys.argv[3] - else: - storage = oauth2client.file.Storage(GC_Values[GC_OAUTH2_TXT]) - credentials = storage.get() - if credentials is None or credentials.invalid: - doRequestOAuth() - credentials = storage.get() - credentials.user_agent = GAM_INFO - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) - if credentials.access_token_expired: - credentials.refresh(http) - access_token = credentials.access_token - print u"\nOAuth File: %s" % GC_Values[GC_OAUTH2_TXT] + access_token = sys.argv[3] oa2 = buildGAPIObject(u'oauth2') token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) print u"Client ID: %s" % token_info[u'issued_to'] From 56732ea3e8aaa1555f9887ffe03deefa3074828d Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 08:18:28 -0800 Subject: [PATCH 07/17] Commit Jay's changes --- src/email-audit-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/email-settings-v1.json | 34 ++++++++++++++++++++++++++++++++++ src/gam.py | 19 +------------------ src/gam.spec | 2 ++ 4 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 src/email-audit-v1.json create mode 100644 src/email-settings-v1.json diff --git a/src/email-audit-v1.json b/src/email-audit-v1.json new file mode 100644 index 00000000..b617e1f5 --- /dev/null +++ b/src/email-audit-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-audit:v1", + "name": "email-audit", + "version": "v1", + "revision": "20130823", + "title": "Email Audit API", + "description": "Lets you peform Google Apps Email Audits", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-audit", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/compliance/audit/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/compliance/audit/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json new file mode 100644 index 00000000..d2ab10be --- /dev/null +++ b/src/email-settings-v1.json @@ -0,0 +1,34 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "email-settings:v1", + "name": "email-settings", + "version": "v1", + "revision": "20130823", + "title": "Email Settings API", + "description": "Lets you manage Google Apps Email Settings", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/admin-sdk/email-settings", + "protocol": "rest", + "baseUrl": "https://apps-apis.google.com/", + "rootUrl": "https://apps-apis.google.com/", + "servicePath": "/a/feeds/emailsettings/2.0/", + "auth": { + "oauth2": { + "scopes": { + "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { + "description": "Manage email audits" + } + } + } + }, + "schemas": { + }, + "resources": { + } +} diff --git a/src/gam.py b/src/gam.py index ab3d468b..8a19a4e4 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8707,23 +8707,6 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): -<<<<<<< HEAD - access_token = sys.argv[3] - oa2 = buildGAPIObject(u'oauth2') - token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) - print u"Client ID: %s" % token_info[u'issued_to'] - try: - print u"Secret: %s" % credentials.client_secret - except UnboundLocalError: - pass - print u'Scopes:' - for scope in token_info[u'scope'].split(u' '): - print u' %s' % scope - try: - print u'Google Apps Admin: %s' % token_info[u'email'] - except KeyError: - print u'Google Apps Admin: Unknown' -======= # TODO eventually would be good if this did something to test admin-selected scopes pass @@ -8732,7 +8715,6 @@ UBER_SCOPES = { u'drive-v2': [u'https://www.googleapis.com/auth/drive'], u'appsactivity-v1': [u'https://www.googleapis.com/auth/activity'] } ->>>>>>> jay0lee/master def select_default_scopes(all_apis): for api_name, api in all_apis.items(): @@ -8764,6 +8746,7 @@ def select_default_scopes(all_apis): def doRequestOAuth(): admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() + apis.remove(u'oauth2') all_apis = {} for api in apis: version = getAPIVer(api) diff --git a/src/gam.spec b/src/gam.spec index 57abac1f..83620ea1 100644 --- a/src/gam.spec +++ b/src/gam.spec @@ -11,6 +11,8 @@ for d in a.datas: a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')] +a.datas += [('email-settings-v1.json', 'email-settings-v1.json', 'DATA')] +a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, From 2bf8f9164e7c3bbe05a42f2a8e363ed88bc7e283 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 08:59:38 -0800 Subject: [PATCH 08/17] Fix typo, drop unneeded API from table --- src/email-settings-v1.json | 2 +- src/gam.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/email-settings-v1.json b/src/email-settings-v1.json index d2ab10be..19ed87f2 100644 --- a/src/email-settings-v1.json +++ b/src/email-settings-v1.json @@ -22,7 +22,7 @@ "oauth2": { "scopes": { "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": { - "description": "Manage email audits" + "description": "Manage email settings" } } } diff --git a/src/gam.py b/src/gam.py index 8a19a4e4..c16450f1 100755 --- a/src/gam.py +++ b/src/gam.py @@ -796,7 +796,6 @@ SERVICE_API_SCOPE_MAPPING = { u'gmail': [u'https://mail.google.com/',], u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], - u'oauth2': [u'https://www.googleapis.com/auth/plus.login', u'https://www.googleapis.com/auth/plus.me',], u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], } From f6dd0ccd12b8e5368bee9e0d05fc971149ea3518 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 17:30:39 -0800 Subject: [PATCH 09/17] Cleanup doRequestOAuth admin_email isn't used; If it's going to be used I'd say: ` if GC-Values{GC_ADMIN): admin_email = GC_Values[GC_ADMIN] else: admin_email = raw_input(u'Please enter your admin email address: ') ` 'oauth2' isn't in API_VER_MAPPING so the remove fails. If it might go back in but you don't want it here, say: ` if u'oauth2' in apis: apis.remove(u'oauth2') ` --- src/gam.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index 54b94fbe..b34f0c0e 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8742,9 +8742,7 @@ def select_default_scopes(apis): return apis def doRequestOAuth(): - admin_email = raw_input(u'Please enter your admin email address: ') apis = API_VER_MAPPING.keys() - apis.remove(u'oauth2') all_apis = {} for api in apis: version = getAPIVer(api) @@ -8845,7 +8843,7 @@ def doRequestOAuth(): break os.system([u'clear', u'cls'][os.name == u'nt']) os.system([u'clear', u'cls'][os.name == u'nt']) - print u'Please authorize your client id for the %s scopes:' % (len(selected_scopes)) + print u'Please authorize your service account client id for the %s scopes:' % (len(selected_scopes)) print print u','.join(selected_scopes) From 3182ce031cd90fa84185a83bb9f65bcf1b12c03c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 18:00:29 -0800 Subject: [PATCH 10/17] Leave email scope in tryOAuth --- src/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gam.py b/src/gam.py index b34f0c0e..3237ca99 100755 --- a/src/gam.py +++ b/src/gam.py @@ -560,7 +560,7 @@ def doGAMVersion(): GM_Globals[GM_GAM_PATH]) def tryOAuth(gdataObject, scope, soft_errors=False): -# scope.append(u'email') #TODO: What is this for? + scope.append(u'email') credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=scope, user_agent=GAM_INFO, sub=GC_Values[GC_ADMIN]) From e0c52c8660c5abdbbca7b6f64c25bfd1e0cae37c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 20:20:38 -0800 Subject: [PATCH 11/17] First cut, dynamic scopes Environment variable GAMSCOPESFILE points to scopes file. Scopes file gamscopes.json --- src/gam.py | 63 ++++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/gam.py b/src/gam.py index 3237ca99..f22ddf42 100755 --- a/src/gam.py +++ b/src/gam.py @@ -68,6 +68,7 @@ ERROR_PREFIX = ERROR+u': ' WARNING = u'WARNING' WARNING_PREFIX = WARNING+u': ' FN_EXTRA_ARGS_TXT = u'extra-args.txt' +FN_GAMSCOPES_JSON = u'gamscopes.json' FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' @@ -90,6 +91,8 @@ GM_SYS_ENCODING = u'syen' GM_BATCH_QUEUE = u'batq' # Extra arguments to pass to GAPI functions GM_EXTRA_ARGS_DICT = u'exad' +# Scopes retrieved from gamscopes.json +GM_GAMSCOPES = u'scop' # Values retrieved from oauth2service.json GM_OAUTH2SERVICE_KEY = u'oauk' GM_OAUTH2SERVICE_ACCOUNT_EMAIL = u'oaae' @@ -105,7 +108,7 @@ GM_MAP_ROLE_NAME_TO_ID = u'rn2i' # Dictionary mapping User ID to Name GM_MAP_USER_ID_TO_NAME = u'ui2n' # Current API scope -GM_API_SCOPE = u'scop' +GM_API_SCOPE = u'csco' # GM_Globals = { GM_SYSEXITRC: 0, @@ -114,6 +117,7 @@ GM_Globals = { GM_SYS_ENCODING: sys.getfilesystemencoding() if os.name == u'nt' else u'utf-8', GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, + GM_GAMSCOPES: {}, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID: None, @@ -152,6 +156,8 @@ GC_DOMAIN = u'domain' GC_DRIVE_DIR = u'drive_dir' # When retrieving lists of Drive files/folders from API, how many should be retrieved in each chunk GC_DRIVE_MAX_RESULTS = u'drive_max_results' +# Path to gamscopes.json +GC_GAMSCOPES_JSON = u'gamscopes_json' # If no_browser is False, output_csv won't open a browser when todrive is set GC_NO_BROWSER = u'no_browser' # Disable GAM API caching @@ -190,6 +196,7 @@ GC_Defaults = { GC_DOMAIN: u'', GC_DRIVE_DIR: u'', GC_DRIVE_MAX_RESULTS: 1000, + GC_GAMSCOPES_JSON: FN_GAMSCOPES_JSON, GC_NO_BROWSER: FALSE, GC_NO_CACHE: FALSE, GC_NO_UPDATE_CHECK: FALSE, @@ -231,6 +238,7 @@ GC_VAR_INFO = { GC_DOMAIN: {GC_VAR_TYPE_KEY: GC_TYPE_STRING}, GC_DRIVE_DIR: {GC_VAR_TYPE_KEY: GC_TYPE_DIRECTORY}, GC_DRIVE_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 1000)}, + GC_GAMSCOPES_JSON: {GC_VAR_TYPE_KEY: GC_TYPE_FILE}, GC_NO_BROWSER: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_CACHE: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, GC_NO_UPDATE_CHECK: {GC_VAR_TYPE_KEY: GC_TYPE_BOOLEAN}, @@ -465,6 +473,7 @@ def SetGlobalVariables(): _getOldEnvVar(GC_OAUTH2SERVICE_JSON, u'OAUTHSERVICEFILE') if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' + _getOldEnvVar(GC_GAMSCOPES_JSON, u'GAMSCOPESFILE') _getOldEnvVar(GC_DOMAIN, u'GA_DOMAIN') _getOldEnvVar(GC_ADMIN, u'GAM_ADMIN') _getOldEnvVar(GC_CUSTOMER_ID, u'CUSTOMER_ID') @@ -507,6 +516,13 @@ def SetGlobalVariables(): GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None + while True: + json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) + if not json_string: + doRequestOAuth() + continue + GM_Globals[GM_GAMSCOPES] = json.loads(json_string) + break return True def doGAMCheckForUpdates(forceCheck=False): @@ -777,36 +793,10 @@ API_VER_MAPPING = { def getAPIVer(api): return API_VER_MAPPING.get(api, u'v1') -SERVICE_API_SCOPE_MAPPING = { - u'admin-settings': [u'https://apps-apis.google.com/a/feeds/domain/',], - u'appsactivity': [u'https://www.googleapis.com/auth/activity', u'https://www.googleapis.com/auth/drive'], - u'calendar': [u'https://www.googleapis.com/auth/calendar',], - u'classroom': [u'https://www.googleapis.com/auth/classroom.courses', u'https://www.googleapis.com/auth/classroom.profile.emails', u'https://www.googleapis.com/auth/classroom.profile.photos',], - u'cloudprint': [u'https://www.googleapis.com/auth/cloudprint',], - u'datatransfer': [u'https://www.googleapis.com/auth/admin.datatransfer',], - u'drive': [u'https://www.googleapis.com/auth/drive',], - u'directory': [u'https://www.googleapis.com/auth/admin.datatransfer', u'https://www.googleapis.com/auth/admin.directory.customer', u'https://www.googleapis.com/auth/admin.directory.device.chromeos', - u'https://www.googleapis.com/auth/admin.directory.device.mobile', u'https://www.googleapis.com/auth/admin.directory.device.mobile.action', - u'https://www.googleapis.com/auth/admin.directory.domain', u'https://www.googleapis.com/auth/admin.directory.group', u'https://www.googleapis.com/auth/admin.directory.group.member', - u'https://www.googleapis.com/auth/admin.directory.notifications', u'https://www.googleapis.com/auth/admin.directory.orgunit', u'https://www.googleapis.com/auth/admin.directory.resource.calendar', - u'https://www.googleapis.com/auth/admin.directory.rolemanagement', u'https://www.googleapis.com/auth/admin.directory.user', u'https://www.googleapis.com/auth/admin.directory.user.alias', - u'https://www.googleapis.com/auth/admin.directory.user.security', u'https://www.googleapis.com/auth/admin.directory.userschema',], - u'email-audit': [u'https://apps-apis.google.com/a/feeds/compliance/audit/',], - u'email-settings': [u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/',], - u'gmail': [u'https://mail.google.com/',], - u'groupsettings': [u'https://www.googleapis.com/auth/apps.groups.settings',], - u'licensing': [u'https://www.googleapis.com/auth/apps.licensing',], - u'reports': [u'https://www.googleapis.com/auth/admin.reports.audit.readonly', u'https://www.googleapis.com/auth/admin.reports.usage.readonly',], - u'siteVerification': ['https://www.googleapis.com/auth/siteverification',], - } - -def getServiceAPIScope(api): - return SERVICE_API_SCOPE_MAPPING.get(api, []) - -def getAPIScope(service): - api_scopes = service._rootDesc[u'auth'][u'oauth2'][u'scopes'] - granted_scopes = api_scopes # TODO fix to lookup from file - return [val for val in api_scopes if val in granted_scopes] + [u'email'] +def getServiceAPIScope(api, version=None): + if not version: + version = getAPIVer(api) + return GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -843,15 +833,15 @@ def getOAuth2ServiceDetails(): def buildGAPIObject(api, act_as=None, soft_errors=False): sub = act_as if act_as else GC_Values[GC_ADMIN] getOAuth2ServiceDetails() - GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api) + version = getAPIVer(api) + if api in [u'directory', u'reports', u'datatransfer']: + api = u'admin' + GM_Globals[GM_API_SCOPE] = getServiceAPIScope(api, version) credentials = oauth2client.client.SignedJwtAssertionCredentials(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_EMAIL], GM_Globals[GM_OAUTH2SERVICE_KEY], scope=GM_Globals[GM_API_SCOPE], user_agent=GAM_INFO, sub=sub) http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=GC_Values[GC_CACHE_DIR])) - version = getAPIVer(api) - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' try: return googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) except googleapiclient.errors.UnknownApiNameOrVersion: @@ -8785,12 +8775,15 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: selected_scopes = [u'email'] + json_scopes = {} for api in all_apis.keys(): selected_scopes += all_apis[api][u'use_scopes'] + json_scopes[api] = all_apis[api][u'use_scopes'] selected_scopes = list(set(selected_scopes)) # unique only if len(selected_scopes) < 2: print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue + writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) break elif selection >= 0 and selection < len(all_apis.keys()): api = all_apis.keys()[selection] From 0470680a4d00cc0202ce2c5d57a2becad7d74fa1 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 20:43:33 -0800 Subject: [PATCH 12/17] Add some error checking for gamscopes.json --- src/gam.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index f22ddf42..8ed94a9a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -266,6 +266,7 @@ MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files we MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service Account.' +MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' def convertUTF8(data): @@ -521,7 +522,10 @@ def SetGlobalVariables(): if not json_string: doRequestOAuth() continue - GM_Globals[GM_GAMSCOPES] = json.loads(json_string) + json_data = json.loads(json_string) + if not isinstance(json_data, dict) or (u'scopes' not in json_data) or (not isinstance(json_data[u'scopes'], dict)): + systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) + GM_Globals[GM_GAMSCOPES] = json_data[u'scopes'] break return True @@ -8775,10 +8779,10 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: selected_scopes = [u'email'] - json_scopes = {} + json_scopes = {u'scopes': {}} for api in all_apis.keys(): selected_scopes += all_apis[api][u'use_scopes'] - json_scopes[api] = all_apis[api][u'use_scopes'] + json_scopes[u'scopes'][api] = all_apis[api][u'use_scopes'] selected_scopes = list(set(selected_scopes)) # unique only if len(selected_scopes) < 2: print u'YOU MUST SELECT AT LEAST ONE SCOPE' From ea6f49f7be743e668acb84a1396d855f656a98e1 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 22:54:07 -0800 Subject: [PATCH 13/17] Have OAuthInfo print API scope table --- src/gam.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 8ed94a9a..475feb5a 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8699,8 +8699,11 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa return full_users def OAuthInfo(): - # TODO eventually would be good if this did something to test admin-selected scopes - pass + print u'API Scopes' + for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): + print u' API: {0}'.format(api) + for scope in GM_Globals[GM_GAMSCOPES][api]: + print u' {0}'.format(scope) UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], From a0ac6265e95e6c83c360f6a0182d0523a1b47e8b Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 23:18:44 -0800 Subject: [PATCH 14/17] If SetGlobalVariables calls doRequestOAuth, don't call again if command is oauth create --- src/gam.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/gam.py b/src/gam.py index 475feb5a..18c37add 100755 --- a/src/gam.py +++ b/src/gam.py @@ -73,7 +73,6 @@ FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' FN_OAUTH2SERVICE_JSON = u'oauth2service.json' MY_CUSTOMER = u'my_customer' UNKNOWN = u'Unknown' - # # Global variables # @@ -93,6 +92,8 @@ GM_BATCH_QUEUE = u'batq' GM_EXTRA_ARGS_DICT = u'exad' # Scopes retrieved from gamscopes.json GM_GAMSCOPES = u'scop' +# gamscopes.json created +GM_GAMSCOPES_CREATED = u'gscr' # Values retrieved from oauth2service.json GM_OAUTH2SERVICE_KEY = u'oauk' GM_OAUTH2SERVICE_ACCOUNT_EMAIL = u'oaae' @@ -118,6 +119,7 @@ GM_Globals = { GM_BATCH_QUEUE: None, GM_EXTRA_ARGS_DICT: {u'prettyPrint': False}, GM_GAMSCOPES: {}, + GM_GAMSCOPES_CREATED: False, GM_OAUTH2SERVICE_KEY: None, GM_OAUTH2SERVICE_ACCOUNT_EMAIL: None, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID: None, @@ -517,6 +519,7 @@ def SetGlobalVariables(): GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None + GM_Globals[GM_GAMSCOPES_CREATED] = False while True: json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) if not json_string: @@ -8791,6 +8794,7 @@ def doRequestOAuth(): print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) + GM_Globals[GM_GAMSCOPES_CREATED] = True break elif selection >= 0 and selection < len(all_apis.keys()): api = all_apis.keys()[selection] @@ -9216,7 +9220,8 @@ try: sys.exit(0) elif sys.argv[1].lower() in [u'oauth', u'oauth2']: if sys.argv[2].lower() in [u'request', u'create']: - doRequestOAuth() + if not GM_Globals[GM_GAMSCOPES_CREATED]: + doRequestOAuth() elif sys.argv[2].lower() == u'info': OAuthInfo() else: From 571a9dcb3e839359173ce212b0665aa40282ee81 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 31 Dec 2015 23:37:15 -0800 Subject: [PATCH 15/17] Handle no scopes for API This should probably call doRequestOAuth rather than bailing out --- src/gam.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 18c37add..f5e6d993 100755 --- a/src/gam.py +++ b/src/gam.py @@ -255,21 +255,22 @@ GC_VAR_INFO = { GC_USER_MAX_RESULTS: {GC_VAR_TYPE_KEY: GC_TYPE_INTEGER, GC_VAR_LIMITS_KEY: (1, 500)}, } -MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' MESSAGE_BATCH_CSV_DASH_DEBUG_INCOMPATIBLE = u'"gam {0} - ..." is not compatible with debugging. Disable debugging by deleting debug.gam and try again.' +MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' +MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV headers of "{1}".' MESSAGE_HIT_CONTROL_C_TO_UPDATE = u'\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.' MESSAGE_NO_DISCOVERY_INFORMATION = u'No online discovery doc and {0} does not exist locally' MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' +MESSAGE_NO_SCOPES_FOR_API = u'There are no scopes authorized for API {0}-{1}; please run gam oauth create' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' +MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service Account.' -MESSAGE_GAMSCOPES_JSON_INVALID = u'The file {0} is missing the required key (scopes) or has an invalid format.' -MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' def convertUTF8(data): import collections @@ -519,7 +520,7 @@ def SetGlobalVariables(): GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: GC_Values[GC_CACHE_DIR] = None - GM_Globals[GM_GAMSCOPES_CREATED] = False + GM_Globals[GM_GAMSCOPES_CREATED] = False while True: json_string = readFile(GC_Values[GC_GAMSCOPES_JSON], continueOnError=True, displayError=True) if not json_string: @@ -803,7 +804,10 @@ def getAPIVer(api): def getServiceAPIScope(api, version=None): if not version: version = getAPIVer(api) - return GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) + scopes = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) + if scopes: + return scopes + systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) def getServiceFromDiscoveryDocument(api, version, http=None): disc_filename = u'%s-%s.json' % (api, version) @@ -8706,7 +8710,7 @@ def OAuthInfo(): for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): print u' API: {0}'.format(api) for scope in GM_Globals[GM_GAMSCOPES][api]: - print u' {0}'.format(scope) + print u' {0}'.format(scope) UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], From 1ec164a25a3a7d2d724962057251a84bc33ab199 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 06:50:16 -0800 Subject: [PATCH 16/17] Bring gam oauth delete back, give creation message --- src/gam.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/gam.py b/src/gam.py index f5e6d993..6c83b4c3 100755 --- a/src/gam.py +++ b/src/gam.py @@ -8712,6 +8712,17 @@ def OAuthInfo(): for scope in GM_Globals[GM_GAMSCOPES][api]: print u' {0}'.format(scope) +def doDeleteOAuth(): + sys.stdout.write(u'Scopes file: {0}, will be Deleted in 3...'.format(GC_Values[GC_GAMSCOPES_JSON])) + time.sleep(1) + sys.stdout.write(u'2...') + time.sleep(1) + sys.stdout.write(u'1...') + time.sleep(1) + sys.stdout.write(u'boom!\n') + os.remove(GC_Values[GC_GAMSCOPES_JSON]) + sys.stdout.write(u'Scopes file: {0}, Deleted\n'.format(GC_Values[GC_GAMSCOPES_JSON])) + UBER_SCOPES = { u'gmail-v1': [u'https://mail.google.com/'], u'drive-v2': [u'https://www.googleapis.com/auth/drive'], @@ -8798,6 +8809,7 @@ def doRequestOAuth(): print u'YOU MUST SELECT AT LEAST ONE SCOPE' continue writeFile(GC_Values[GC_GAMSCOPES_JSON], json.dumps(json_scopes)) + print u'Scopes file: {0}, Created'.format(GC_Values[GC_GAMSCOPES_JSON]) GM_Globals[GM_GAMSCOPES_CREATED] = True break elif selection >= 0 and selection < len(all_apis.keys()): @@ -9228,6 +9240,8 @@ try: doRequestOAuth() elif sys.argv[2].lower() == u'info': OAuthInfo() + elif sys.argv[2].lower() in [u'delete', u'revoke']: + doDeleteOAuth() else: print u'ERROR: %s is not a valid argument for "gam oauth"' % sys.argv[2] sys.exit(2) From bcb17cd0a580a3481ab81309e0f2d392fe2eadee Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 1 Jan 2016 07:40:22 -0800 Subject: [PATCH 17/17] in doRequestOauth, initialize API use_scopes from gamscopes.json if available --- src/gam.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 6c83b4c3..1a09cbc6 100755 --- a/src/gam.py +++ b/src/gam.py @@ -527,9 +527,9 @@ def SetGlobalVariables(): doRequestOAuth() continue json_data = json.loads(json_string) - if not isinstance(json_data, dict) or (u'scopes' not in json_data) or (not isinstance(json_data[u'scopes'], dict)): + if not isinstance(json_data, dict): systemErrorExit(17, MESSAGE_GAMSCOPES_JSON_INVALID.format(GC_Values[GC_GAMSCOPES_JSON])) - GM_Globals[GM_GAMSCOPES] = json_data[u'scopes'] + GM_Globals[GM_GAMSCOPES] = json_data break return True @@ -804,7 +804,8 @@ def getAPIVer(api): def getServiceAPIScope(api, version=None): if not version: version = getAPIVer(api) - scopes = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), []) + apiData = GM_Globals[GM_GAMSCOPES].get(u'{0}-{1}'.format(api, version), {}) + scopes = apiData.get(u'use_scopes', []) if scopes: return scopes systemErrorExit(15, MESSAGE_NO_SCOPES_FOR_API.format(api, version)) @@ -8709,17 +8710,21 @@ def OAuthInfo(): print u'API Scopes' for api in sorted(GM_Globals[GM_GAMSCOPES].keys()): print u' API: {0}'.format(api) - for scope in GM_Globals[GM_GAMSCOPES][api]: + for scope in GM_Globals[GM_GAMSCOPES][api][u'use_scopes']: print u' {0}'.format(scope) def doDeleteOAuth(): sys.stdout.write(u'Scopes file: {0}, will be Deleted in 3...'.format(GC_Values[GC_GAMSCOPES_JSON])) + sys.stdout.flush() time.sleep(1) sys.stdout.write(u'2...') + sys.stdout.flush() time.sleep(1) sys.stdout.write(u'1...') + sys.stdout.flush() time.sleep(1) sys.stdout.write(u'boom!\n') + sys.stdout.flush() os.remove(GC_Values[GC_GAMSCOPES_JSON]) sys.stdout.write(u'Scopes file: {0}, Deleted\n'.format(GC_Values[GC_GAMSCOPES_JSON])) @@ -8775,6 +8780,9 @@ def doRequestOAuth(): all_apis[api_name][u'index'] = i i += 1 all_apis = select_default_scopes(all_apis) + if GM_Globals[GM_GAMSCOPES]: + for api in GM_Globals[GM_GAMSCOPES]: + all_apis[api][u'use_scopes'] = GM_Globals[GM_GAMSCOPES][api][u'use_scopes'] os.system([u'clear', u'cls'][os.name == u'nt']) while True: print u'Select the APIs to use with GAM.' @@ -8800,10 +8808,10 @@ def doRequestOAuth(): all_apis[api][u'use_scopes'] = [] elif selection == i+3: selected_scopes = [u'email'] - json_scopes = {u'scopes': {}} + json_scopes = {} for api in all_apis.keys(): selected_scopes += all_apis[api][u'use_scopes'] - json_scopes[u'scopes'][api] = all_apis[api][u'use_scopes'] + json_scopes[api] = {u'use_scopes': all_apis[api][u'use_scopes']} selected_scopes = list(set(selected_scopes)) # unique only if len(selected_scopes) < 2: print u'YOU MUST SELECT AT LEAST ONE SCOPE'