From 3edfce202f38123738e6fc551627e4dcf901744f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 16 Nov 2015 07:13:39 -0800 Subject: [PATCH 1/8] Cleanup - use named constants Use named constants instead of repeated literals Define messages at top, would make language translation easier Clean up result handling in doDownloadActivity/ExportRequest Handle missing Python SSL module in one place The following two files need to be updated to 3.61 https://gam-update.appspot.com/latest-version-announcement.txt https://gam-update.appspot.com/latest-version.txt --- src/gam.py | 189 +++++++++++++++++++++++++++++------------------------ 1 file changed, 104 insertions(+), 85 deletions(-) diff --git a/src/gam.py b/src/gam.py index 43a16037..c4bd35de 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # # GAM # @@ -27,8 +28,7 @@ __author__ = u'Jay Lee ' __version__ = u'3.62' __license__ = u'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)' -import sys, os, time, datetime, random, socket, csv, platform, re, calendar, base64, string -import subprocess +import sys, os, time, datetime, random, socket, csv, platform, re, calendar, base64, string, subprocess import json import httplib2 @@ -42,16 +42,21 @@ import oauth2client.tools import mimetypes import ntpath -is_frozen = getattr(sys, 'frozen', '') - GAM_URL = u'http://git.io/gam' GAM_INFO = u'GAM {0} - {1} / {2} / Python {3}.{4}.{5} {6} / {7} {8} /'.format(__version__, GAM_URL, __author__, sys.version_info[0], sys.version_info[1], sys.version_info[2], sys.version_info[3], platform.platform(), platform.machine()) -GAM_RELEASES = u'http://git.io/gamreleases' +GAM_RELEASES = u'https://github.com/jay0lee/GAM/releases' +GAM_WIKI = u'https://github.com/jay0lee/GAM/wiki' +GAM_WIKI_CREATE_CLIENT_SECRETS = GAM_WIKI+u'/CreatingClientSecretsFile#creating-your-own-oauth2servicejson' +GAM_APPSPOT = u'https://gam-update.appspot.com' +GAM_APPSPOT_LATEST_VERSION = GAM_APPSPOT+u'/latest-version.txt?v='+__version__ +GAM_APPSPOT_LATEST_VERSION_ANNOUNCEMENT = GAM_APPSPOT+u'/latest-version-announcement.txt?v='+__version__ +TRUE = u'true' +FALSE = u'false' extra_args = {u'prettyPrint': False} true_values = [u'on', u'yes', u'enabled', u'true', u'1'] false_values = [u'off', u'no', u'disabled', u'false', u'0'] @@ -59,6 +64,17 @@ usergroup_types = [u'user', u'users', u'group', u'ou', u'org', u'ou_and_children', u'ou_and_child', u'query', u'license', u'licenses', u'licence', u'licences', u'file', u'all', u'cros'] +ERROR = u'ERROR' +ERROR_PREFIX = ERROR+u': ' +WARNING = u'WARNING' +WARNING_PREFIX = WARNING+u': ' +FN_CLIENT_SECRETS_JSON = u'client_secrets.json' +FN_EXTRA_ARGS_TXT = u'extra-args.txt' +FN_LAST_UPDATE_CHECK_TXT = u'lastupdatecheck.txt' +FN_OAUTH2SERVICE_JSON = u'oauth2service.json' +FN_OAUTH2_TXT = u'oauth2.txt' +MY_CUSTOMER = u'my_customer' +UNKNOWN_DOMAIN = u'Unknown' customerId = None domain = None @@ -70,6 +86,20 @@ gamUserConfigDir = None gamDriveDir = None gamCacheDir = None +MESSAGE_CLIENT_API_ACCESS_DENIED = u'Access Denied. Please make sure the Client Name:\n\n{0}\n\nis authorized for the API Scope(s):\n\n{1}\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage API client access' +MESSAGE_GAM_EXITING_FOR_UPDATE = u'GAM is now exiting so that you can overwrite this old version with the latest release' +MESSAGE_GAM_OUT_OF_MEMORY = u'GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' +MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = u'Header "{0}" not found in CSV headers of "{1}".' +MESSAGE_HIT_CONTROL_C_TO_UPDATE = u'\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.' +MESSAGE_NO_DISCOVERY_INFORMATION = u'No online discovery doc and {0} does not exist locally' +MESSAGE_NO_PYTHON_SSL = u'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' +MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = u'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' +MESSAGE_REQUEST_COMPLETED_NO_FILES = u'Request completed but no results/files were returned, try requesting again' +MESSAGE_REQUEST_NOT_COMPLETE = u'Request needs to be completed before downloading, current status is: {0}' +MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = u'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' +MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON = u'Please follow the instructions at this site to setup a Service Account.' +MESSAGE_OAUTH2SERVICE_JSON_INVALID = u'The file {0} is missing required keys (client_email, client_id or private_key).' + def convertUTF8(data): import collections if isinstance(data, str): @@ -162,6 +192,20 @@ gam.exe update group announcements add member jsmith ''' +# +# Error handling +# +def systemErrorExit(sysRC, message): + if message: + sys.stderr.write(u'\n{0}{1}\n'.format(ERROR_PREFIX, message)) + sys.exit(sysRC) + +def noPythonSSLExit(): + systemErrorExit(8, MESSAGE_NO_PYTHON_SSL) + +def printLine(message): + sys.stdout.write(message+u'\n') + def setGamDirs(): global gamPath, gamSiteConfigDir, gamUserConfigDir, gamDriveDir, gamCacheDir gamPath = os.path.dirname(os.path.realpath(__file__)) @@ -191,39 +235,38 @@ def doGAMCheckForUpdates(): current_version = float(__version__) except ValueError: return - if os.path.isfile(os.path.join(gamUserConfigDir, u'lastupdatecheck.txt')): - f = open(os.path.join(gamUserConfigDir, u'lastupdatecheck.txt'), 'r') + if os.path.isfile(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT)): + f = open(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), 'r') last_check_time = int(f.readline()) f.close() else: last_check_time = 0 now_time = calendar.timegm(time.gmtime()) - one_week_ago_time = now_time - 604800 - if last_check_time > one_week_ago_time: + if last_check_time > now_time-604800: return try: - c = urllib2.urlopen(u'https://gam-update.appspot.com/latest-version.txt?v=%s' % __version__) + c = urllib2.urlopen(GAM_APPSPOT_LATEST_VERSION) try: latest_version = float(c.read()) except ValueError: return if latest_version <= current_version: - f = open(os.path.join(gamUserConfigDir, u'lastupdatecheck.txt'), 'w') + f = open(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), 'w') f.write(str(now_time)) f.close() return - a = urllib2.urlopen(u'https://gam-update.appspot.com/latest-version-announcement.txt?v=%s' % __version__) + a = urllib2.urlopen(GAM_APPSPOT_LATEST_VERSION_ANNOUNCEMENT) announcement = a.read() sys.stderr.write(announcement) try: - print u"\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won't bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won't ever check for updates." + printLine(MESSAGE_HIT_CONTROL_C_TO_UPDATE) time.sleep(15) except KeyboardInterrupt: import webbrowser - webbrowser.open(u'https://github.com/jay0lee/GAM/releases') - print u'GAM is now exiting so that you can overwrite this old version with the latest release' + webbrowser.open(GAM_RELEASES) + printLine(MESSAGE_GAM_EXITING_FOR_UPDATE) sys.exit(0) - f = open(os.path.join(gamUserConfigDir, u'lastupdatecheck.txt'), 'w') + f = open(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), 'w') f.write(str(now_time)) f.close() except urllib2.HTTPError: @@ -288,7 +331,7 @@ def checkErrorCode(e, service): def tryOAuth(gdataObject): global domain global customerId - oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', 'oauth2.txt')) + oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', FN_OAUTH2_TXT)) storage = oauth2client.file.Storage(oauth2file) credentials = storage.get() if credentials is None or credentials.invalid: @@ -358,8 +401,7 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re print u'ERROR: %s' % e.content if soft_errors: return - else: - sys.exit(5) + sys.exit(5) http_status = error[u'error'][u'code'] message = error[u'error'][u'errors'][0][u'message'] try: @@ -383,17 +425,14 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re if n != 1: sys.stderr.write(u' - Giving up.\n') return - else: - sys.exit(int(http_status)) + sys.exit(int(http_status)) except oauth2client.client.AccessTokenRefreshError, e: sys.stderr.write(u'Error: Authentication Token Error - %s' % e) sys.exit(403) except httplib2.CertificateValidationUnsupported: - print u'\nError: You don\'t have the Python ssl module installed so we can\'t verify SSL Certificates.\n\nYou can fix this by installing the Python SSL module or you can live on dangerously and turn SSL validation off by creating a file called noverifyssl.txt in the same location as gam.exe / gam.py' - sys.exit(8) + noPythonSSLExit() except TypeError, e: - print u'Error: %s' % e - sys.exit(4) + systemErrorExit(4, e) def restart_line(): sys.stderr.write('\r') @@ -492,13 +531,12 @@ def getServiceFromDiscoveryDocument(api, version, http): with open(pyinstaller_disc_file, 'rb') as f: discovery = f.read() else: - print u'No online discovery doc and {0} does not exist locally'.format(disc_file) - raise + systemErrorExit(4, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) return googleapiclient.discovery.build_from_document(discovery, base=u'https://www.googleapis.com', http=http) def buildGAPIObject(api): global domain, customerId - storage = oauth2client.file.Storage(os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', 'oauth2.txt'))) + storage = oauth2client.file.Storage(os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', FN_OAUTH2_TXT))) credentials = storage.get() if credentials is None or credentials.invalid: doRequestOAuth() @@ -511,11 +549,11 @@ def buildGAPIObject(api): if os.path.isfile(os.path.join(gamUserConfigDir, u'debug.gam')): httplib2.debuglevel = 4 extra_args[u'prettyPrint'] = True - if os.path.isfile(os.path.join(gamUserConfigDir, u'extra-args.txt')): + if os.path.isfile(os.path.join(gamUserConfigDir, FN_EXTRA_ARGS_TXT)): import ConfigParser config = ConfigParser.ConfigParser() config.optionxform = str - config.read(os.path.join(gamUserConfigDir, u'extra-args.txt')) + config.read(os.path.join(gamUserConfigDir, FN_EXTRA_ARGS_TXT)) extra_args.update(dict(config.items(u'extra-args'))) http = credentials.authorize(http) version = getAPIVer(api) @@ -526,8 +564,7 @@ def buildGAPIObject(api): except googleapiclient.errors.UnknownApiNameOrVersion: service = getServiceFromDiscoveryDocument(api, version, http) except httplib2.CertificateValidationUnsupported: - print u'Error: You don\'t have the Python ssl module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file called noverifyssl.txt in the same location as gam.exe / gam.py' - sys.exit(8) + noPythonSSLExit() try: domain = os.environ[u'GA_DOMAIN'] _, customerId_result = service._http.request(u'https://www.googleapis.com/admin/directory/v1/users?domain=%s&maxResults=1&fields=users(customerId)' % domain) @@ -537,8 +574,8 @@ def buildGAPIObject(api): try: domain = credentials.id_token[u'hd'] except (TypeError, KeyError): - domain = u'Unknown' - customerId = u'my_customer' + domain = UNKNOWN_DOMAIN + customerId = MY_CUSTOMER return service def buildGAPIServiceObject(api, act_as=None, soft_errors=False): @@ -548,9 +585,8 @@ def buildGAPIServiceObject(api, act_as=None, soft_errors=False): try: json_string = open(oauth2servicefilejson).read() except IOError, e: - print u'Error: %s' % e - print u'' - print u'Please follow the instructions at:\n\nhttps://github.com/jay0lee/GAM/wiki/CreatingClientSecretsFile#creating-your-own-oauth2servicejson\n\nto setup a Service Account' + printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) + printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) sys.exit(6) json_data = json.loads(json_string) try: @@ -577,11 +613,11 @@ def buildGAPIServiceObject(api, act_as=None, soft_errors=False): if os.path.isfile(os.path.join(gamUserConfigDir, u'debug.gam')): httplib2.debuglevel = 4 extra_args[u'prettyPrint'] = True - if os.path.isfile(os.path.join(gamUserConfigDir, u'extra-args.txt')): + if os.path.isfile(os.path.join(gamUserConfigDir, FN_EXTRA_ARGS_TXT)): import ConfigParser config = ConfigParser.ConfigParser() config.optionxform = str - config.read(os.path.join(gamUserConfigDir, u'extra-args.txt')) + config.read(os.path.join(gamUserConfigDir, FN_EXTRA_ARGS_TXT)) extra_args.update(dict(config.items(u'extra-args'))) http = credentials.authorize(http) version = getAPIVer(api) @@ -591,13 +627,11 @@ def buildGAPIServiceObject(api, act_as=None, soft_errors=False): return getServiceFromDiscoveryDocument(api, version, http) except oauth2client.client.AccessTokenRefreshError, e: if e.message in [u'access_denied', u'unauthorized_client: Unauthorized client or scope in request.']: - print u'Error: Access Denied. Please make sure the Client Name:\n\n%s\n\nis authorized for the API Scope(s):\n\n%s\n\nThis can be configured in your Control Panel under:\n\nSecurity -->\nAdvanced Settings -->\nManage third party OAuth Client access' % (SERVICE_ACCOUNT_CLIENT_ID, ','.join(scope)) - sys.exit(5) - else: - print u'Error: %s' % e - if soft_errors: - return False - sys.exit(4) + systemErrorExit(5, MESSAGE_CLIENT_API_ACCESS_DENIED.format(SERVICE_ACCOUNT_CLIENT_ID, u','.join(scope))) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) + if soft_errors: + return False + sys.exit(4) def buildDiscoveryObject(api): import uritemplate @@ -677,7 +711,7 @@ def showReport(): report = sys.argv[2].lower() global customerId rep = buildGAPIObject(u'reports') - if customerId == u'my_customer': + if customerId == MY_CUSTOMER: customerId = None date = filters = parameters = actorIpAddress = startTime = endTime = eventName = None to_drive = False @@ -1126,7 +1160,7 @@ def doCreateDomainAlias(): body = {} body[u'domainAliasName'] = sys.argv[3] body[u'parentDomainName'] = sys.argv[4] - result = callGAPI(service=cd.domainAliases(), function=u'insert', customer=customerId, body=body) + callGAPI(service=cd.domainAliases(), function=u'insert', customer=customerId, body=body) def doUpdateDomain(): cd = buildGAPIObject(u'directory') @@ -1140,7 +1174,7 @@ def doUpdateDomain(): else: print u'ERROR: %s is not a valid argument for "gam update domain"' % sys.argv[i] sys.exit(2) - result = callGAPI(service=cd.customers(), function=u'update', customerKey=customerId, body=body) + callGAPI(service=cd.customers(), function=u'update', customerKey=customerId, body=body) print u'%s is now the primary domain.' % domain_name def doGetDomainInfo(): @@ -3525,8 +3559,7 @@ def transferDriveFiles(users): source_about = callGAPI(service=source_drive.about(), function=u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId, permissionId') source_drive_size = int(source_about[u'quotaBytesUsed']) if target_drive_free < source_drive_size: - print u'Error: Cowardly refusing to perform migration due to lack of target drive space. Source size: %smb Target Free: %smb' % (source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024) - sys.exit(4) + systemErrorExit(4, MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE.format(source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024)) print u'Source drive size: %smb Target drive free: %smb' % (source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024) target_drive_free = target_drive_free - source_drive_size # prep target_drive_free for next user source_root = source_about[u'rootFolderId'] @@ -5839,7 +5872,7 @@ def doGetUserInfo(user_email=None): try: user_email = sys.argv[3] except IndexError: - oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE'), 'oauth2.txt') + oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE'), FN_OAUTH2_TXT) storage = oauth2client.file.Storage(oauth2file) credentials = storage.get() if credentials is None or credentials.invalid: @@ -6699,7 +6732,7 @@ def doGetInstanceInfo(): sys.exit(0) print u'Google Apps Domain: %s' % domain cd = buildGAPIObject(u'directory') - if customerId != u'my_customer': + if customerId != MY_CUSTOMER: customer_id = customerId else: result = callGAPI(service=cd.users(), function=u'list', fields=u'users(customerId)', customer=customerId, maxResults=1) @@ -6899,7 +6932,7 @@ def output_csv(csv_list, titles, list_type, todrive): cell_count = rows * columns convert = True if cell_count > 500000 or columns > 256: - print u'Warning: results are to large for Google Spreadsheets. Uploading as a regular CSV file.' + print u'{0}{1}'.format(WARNING_PREFIX, MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET) convert = False drive = buildGAPIObject(u'drive') string_data = string_file.getvalue() @@ -7903,15 +7936,9 @@ def doDownloadActivityRequest(): user = user[:user.find(u'@')] results = callGData(service=audit, function=u'getAccountInformationRequestStatus', user=user, request_id=request_id) if results[u'status'] != u'COMPLETED': - print u'Request needs to be completed before downloading, current status is: '+results[u'status'] - sys.exit(4) - try: - if int(results[u'numberOfFiles']) < 1: - print u'ERROR: Request completed but no results were returned, try requesting again' - sys.exit(4) - except KeyError: - print u'ERROR: Request completed but no files were returned, try requesting again' - sys.exit(4) + systemErrorExit(4, MESSAGE_REQUEST_NOT_COMPLETE.format(results[u'status'])) + if int(results.get(u'numberOfFiles', u'0')) < 1: + systemErrorExit(4, MESSAGE_REQUEST_COMPLETED_NO_FILES) for i in range(0, int(results[u'numberOfFiles'])): url = results[u'fileUrl'+str(i)] filename = u'activity-'+user+'-'+request_id+'-'+unicode(i)+u'.txt.gpg' @@ -8091,15 +8118,9 @@ def doDownloadExportRequest(): user = user[:user.find(u'@')] results = callGData(service=audit, function=u'getMailboxExportRequestStatus', user=user, request_id=request_id) if results[u'status'] != u'COMPLETED': - print u'Request needs to be completed before downloading, current status is: '+results[u'status'] - sys.exit(4) - try: - if int(results[u'numberOfFiles']) < 1: - print u'ERROR: Request completed but no results were returned, try requesting again' - sys.exit(4) - except KeyError: - print u'ERROR: Request completed but no files were returned, try requesting again' - sys.exit(4) + systemErrorExit(4, MESSAGE_REQUEST_NOT_COMPLETE.format(results[u'status'])) + if int(results.get(u'numberOfFiles', u'0')) < 1: + systemErrorExit(4, MESSAGE_REQUEST_COMPLETED_NO_FILES) for i in range(0, int(results['numberOfFiles'])): url = results[u'fileUrl'+str(i)] filename = u'export-'+user+'-'+request_id+'-'+str(i)+u'.mbox.gpg' @@ -8290,7 +8311,7 @@ def OAuthInfo(): try: access_token = sys.argv[3] except IndexError: - oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', 'oauth2.txt')) + oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', FN_OAUTH2_TXT)) storage = oauth2client.file.Storage(oauth2file) credentials = storage.get() if credentials is None or credentials.invalid: @@ -8323,7 +8344,7 @@ def OAuthInfo(): print u'Google Apps Admin: Unknown' def doDeleteOAuth(): - oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', 'oauth2.txt')) + oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', FN_OAUTH2_TXT)) storage = oauth2client.file.Storage(oauth2file) credentials = storage.get() try: @@ -8380,12 +8401,12 @@ possible_scopes = [u'https://www.googleapis.com/auth/admin.directory.group', u'https://www.googleapis.com/auth/admin.directory.userschema', # Customer User Schema u'https://www.googleapis.com/auth/classroom.rosters https://www.googleapis.com/auth/classroom.courses https://www.googleapis.com/auth/classroom.profile.emails https://www.googleapis.com/auth/classroom.profile.photos', # Classroom API u'https://www.googleapis.com/auth/cloudprint', # CloudPrint API - u'https://www.googleapis.com/auth/admin.datatransfer', # Data Transfer API + u'https://www.googleapis.com/auth/admin.datatransfer', # Data Transfer API u'https://www.googleapis.com/auth/admin.directory.customer', # Customer API - u'https://www.googleapis.com/auth/admin.directory.domain'] # Domain API + u'https://www.googleapis.com/auth/admin.directory.domain'] # Domain API def doRequestOAuth(incremental_auth=False): - CLIENT_SECRETS = os.path.join(gamUserConfigDir, os.environ.get(u'CLIENTSECRETSFILE', 'client_secrets.json')) + CLIENT_SECRETS = os.path.join(gamUserConfigDir, os.environ.get(u'CLIENTSECRETSFILE', FN_CLIENT_SECRETS_JSON)) MISSING_CLIENT_SECRETS_MESSAGE = u""" WARNING: Please configure OAuth 2.0 @@ -8508,7 +8529,7 @@ access or an 'a' to grant action-only access. FLOW = oauth2client.client.flow_from_clientsecrets(CLIENT_SECRETS, scope=scopes, message=MISSING_CLIENT_SECRETS_MESSAGE) - oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', 'oauth2.txt')) + oauth2file = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHFILE', FN_OAUTH2_TXT)) storage = oauth2client.file.Storage(oauth2file) credentials = storage.get() flags = cmd_flags() @@ -8525,8 +8546,7 @@ access or an 'a' to grant action-only access. try: credentials = oauth2client.tools.run_flow(flow=FLOW, storage=storage, flags=flags, http=http) except httplib2.CertificateValidationUnsupported: - print u'\nError: You don\'t have the Python ssl module installed so we can\'t verify SSL Certificates.\n\nYou can fix this by installing the Python SSL module or you can live on dangerously and turn SSL validation off by creating a file called noverifyssl.txt in the same location as gam.exe / gam.py' - sys.exit(8) + noPythonSSLExit() def batch_worker(): while True: @@ -8588,7 +8608,7 @@ try: items.append(argv) run_batch(items) sys.exit(0) - elif sys.argv[1].lower() == 'csv': + elif sys.argv[1].lower() == u'csv': csv_filename = sys.argv[2] if csv_filename == u'-': import StringIO @@ -8610,8 +8630,7 @@ try: elif arg[1:] in row: argv.append(row[arg[1:]]) else: - print 'ERROR: header "%s" not found in CSV headers of "%s", giving up.' % (arg[1:], ','.join(row.keys())) - sys.exit(0) + systemErrorExit(2, MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(arg[1:], ','.join(row.keys()))) items.append(argv) run_batch(items) sys.exit(0) @@ -9114,8 +9133,8 @@ except IndexError: except KeyboardInterrupt: sys.exit(50) except socket.error, e: - print u'\nError: %s' % e + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) sys.exit(3) except MemoryError: - print u'Error: GAM has run out of memory. If this is a large Google Apps instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.' + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, MESSAGE_GAM_OUT_OF_MEMORY)) sys.exit(99) From 7a9bda9b1b714b0c4daac8733974b68f9c800ea1 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 19 Nov 2015 16:46:16 -0800 Subject: [PATCH 2/8] Guard vacation info with convertUTF8, fix typo in doGetNotifications --- src/gam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index c4bd35de..b32ac6de 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4520,7 +4520,7 @@ def getVacation(users): emailsettings.domain = domain vacationsettings = callGData(service=emailsettings, function=u'GetVacation', soft_errors=True, username=user) try: - print u'''User %s + print convertUTF8(u'''User %s Enabled: %s Contacts Only: %s Domain Only: %s @@ -4529,7 +4529,7 @@ def getVacation(users): Start Date: %s End Date: %s ''' % (user+u'@'+emailsettings.domain, vacationsettings[u'enable'], vacationsettings[u'contactsOnly'], vacationsettings[u'domainOnly'], vacationsettings[u'subject'], - vacationsettings[u'message'], vacationsettings[u'startDate'], vacationsettings[u'endDate']) + vacationsettings[u'message'], vacationsettings[u'startDate'], vacationsettings[u'endDate'])) except TypeError: pass @@ -6364,7 +6364,7 @@ def doGetNotifications(): if sys.argv[i].lower() == u'unreadonly': unread_only = True else: - print 'ERROR: %s is not a valid argument for "gam delete notification", expected unreadonly' % sys.argv[i] + print 'ERROR: %s is not a valid argument for "gam info notification", expected unreadonly' % sys.argv[i] sys.exit(2) i += 1 notifications = callGAPIpages(service=cd.notifications(), function=u'list', customer=customerId) From b822608b15eb5e4769c10a2209a14d3b9f10243f Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sat, 21 Nov 2015 09:50:11 -0800 Subject: [PATCH 3/8] Define/use file processing routines; fix show filelist allfields Define and use new file handling primitives to simply code and isolate error handling. Error message cleanup. Google added a new object (userPermission) to the filesResource object which broke showDriveFiles because it has an item named 'id' which wiped out the file id. This fix makes compound column names for all dictionary objects except for labels. --- src/gam.py | 262 +++++++++++++++++++++++++++-------------------------- 1 file changed, 136 insertions(+), 126 deletions(-) diff --git a/src/gam.py b/src/gam.py index b32ac6de..cbfd009c 100755 --- a/src/gam.py +++ b/src/gam.py @@ -28,7 +28,7 @@ __author__ = u'Jay Lee ' __version__ = u'3.62' __license__ = u'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)' -import sys, os, time, datetime, random, socket, csv, platform, re, calendar, base64, string, subprocess +import sys, os, time, datetime, random, socket, csv, platform, re, calendar, base64, string, StringIO, subprocess import json import httplib2 @@ -206,6 +206,59 @@ def noPythonSSLExit(): def printLine(message): sys.stdout.write(message+u'\n') +# +# Open a file +# +def openFile(filename, mode='rb'): + try: + if filename != u'-': + return open(filename, mode) + if mode.startswith(u'r'): + return StringIO.StringIO(unicode(sys.stdin.read())) + return sys.stdout + except IOError as e: + systemErrorExit(6, e) +# +# Close a file +# +def closeFile(f): + try: + f.close() + return True + except IOError as e: + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) + return False +# +# Read a file +# +def readFile(filename, mode='rb', continueOnError=False, displayError=True): + try: + if filename != u'-': + with open(filename, mode) as f: + return f.read() + else: + return unicode(sys.stdin.read()) + except IOError as e: + if continueOnError: + if displayError: + sys.stderr.write(u'{0}{1}\n'.format(WARNING_PREFIX, e)) + return None + systemErrorExit(6, e) +# +# Write a file +# +def writeFile(filename, data, mode='wb', continueOnError=False, displayError=True): + try: + with open(filename, mode) as f: + f.write(data) + return True + except IOError as e: + if continueOnError: + if displayError: + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e)) + return False + systemErrorExit(6, e) + def setGamDirs(): global gamPath, gamSiteConfigDir, gamUserConfigDir, gamDriveDir, gamCacheDir gamPath = os.path.dirname(os.path.realpath(__file__)) @@ -216,6 +269,8 @@ def setGamDirs(): else: gamCacheDir = os.environ.get(u'GAMCACHEDIR', os.path.join(gamPath, u'gamcache')) gamDriveDir = os.environ.get(u'GAMDRIVEDIR', gamPath) + if not os.path.isfile(os.path.join(gamUserConfigDir, u'noupdatecheck.txt')): + doGAMCheckForUpdates() def doGAMVersion(): import struct @@ -227,33 +282,29 @@ def doGAMVersion(): platform.platform(), platform.machine(), gamPath) -def doGAMCheckForUpdates(): +def doGAMCheckForUpdates(forceCheck=False): import urllib2 - if os.path.isfile(os.path.join(gamUserConfigDir, u'noupdatecheck.txt')): - return try: current_version = float(__version__) except ValueError: return - if os.path.isfile(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT)): - f = open(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), 'r') - last_check_time = int(f.readline()) - f.close() - else: - last_check_time = 0 now_time = calendar.timegm(time.gmtime()) - if last_check_time > now_time-604800: - return + if not forceCheck: + last_check_time = readFile(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), continueOnError=True, displayError=forceCheck) + if last_check_time == None: + last_check_time = 0 + if last_check_time > now_time-604800: + return try: c = urllib2.urlopen(GAM_APPSPOT_LATEST_VERSION) try: latest_version = float(c.read()) except ValueError: return + if forceCheck or (latest_version > current_version): + print u'Version: Check, Current: {0:.2f}, Latest: {1:.2f}'.format(current_version, latest_version) if latest_version <= current_version: - f = open(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), 'w') - f.write(str(now_time)) - f.close() + writeFile(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), str(now_time), continueOnError=True, displayError=forceCheck) return a = urllib2.urlopen(GAM_APPSPOT_LATEST_VERSION_ANNOUNCEMENT) announcement = a.read() @@ -266,29 +317,23 @@ def doGAMCheckForUpdates(): webbrowser.open(GAM_RELEASES) printLine(MESSAGE_GAM_EXITING_FOR_UPDATE) sys.exit(0) - f = open(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), 'w') - f.write(str(now_time)) - f.close() - except urllib2.HTTPError: - return - except urllib2.URLError: + writeFile(os.path.join(gamUserConfigDir, FN_LAST_UPDATE_CHECK_TXT), str(now_time), continueOnError=True, displayError=forceCheck) + except (urllib2.HTTPError, urllib2.URLError): return -def checkErrorCode(e, service): - +def checkGDataError(e, service): # First check for errors that need special handling - if e[0].get('reason', '') in [u'Token invalid - Invalid token: Stateless token expired', u'Token invalid - Invalid token: Token not found']: + if e[0].get(u'reason', u'') in [u'Token invalid - Invalid token: Stateless token expired', u'Token invalid - Invalid token: Token not found']: keep_domain = service.domain tryOAuth(service) service.domain = keep_domain return False - if e[0]['body'][:34] in [u'Required field must not be blank: ', u'These characters are not allowed: ']: + if e[0][u'body'].startswith(u'Required field must not be blank:') or e[0][u'body'].startswith(u'These characters are not allowed:'): return e[0]['body'] if e.error_code == 600 and e[0][u'body'] == u'Quota exceeded for the current request' or e[0][u'reason'] == u'Bad Gateway': return False if e.error_code == 600 and e[0][u'reason'] == u'Token invalid - Invalid token: Token disabled, revoked, or expired.': return u'403 - Token disabled, revoked, or expired. Please delete and re-create oauth.txt' - # We got a "normal" error, define the mapping below error_code_map = { 1000: False, @@ -325,8 +370,7 @@ def checkErrorCode(e, service): 1800: u'Group Cannot Contain Cycle', 1801: u'Invalid value %s' % getattr(e, u'invalidInput', u''), } - - return u'%s - %s' % (e.error_code, error_code_map.get(e.error_code, u'Unknown Error: %s' % (str(e)))) + return u'{0} - {1}'.format(e.error_code, error_code_map.get(e.error_code, u'Unknown Error: {0}'.format(str(e)))) def tryOAuth(gdataObject): global domain @@ -356,7 +400,7 @@ def callGData(service, function, soft_errors=False, throw_errors=[], **kwargs): try: return method(**kwargs) except gdata.apps.service.AppsForYourDomainException, e: - terminating_error = checkErrorCode(e, service) + terminating_error = checkGDataError(e, service) if e.error_code in throw_errors: raise if not terminating_error and n != retries: @@ -369,13 +413,12 @@ def callGData(service, function, soft_errors=False, throw_errors=[], **kwargs): if n > 3: sys.stderr.write(u'attempt %s/%s\n' % (n+1, retries)) continue - sys.stderr.write(u'Error: %s' % terminating_error) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, terminating_error)) if soft_errors: if n != 1: sys.stderr.write(u' - Giving up.\n') return - else: - sys.exit(int(e.error_code)) + sys.exit(int(e.error_code)) def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_reasons=[], retry_reasons=[], **kwargs): method = getattr(service, function) @@ -398,7 +441,7 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re time.sleep(1) continue if not silent_errors: - print u'ERROR: %s' % e.content + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e.content)) if soft_errors: return sys.exit(5) @@ -420,14 +463,14 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re if n > 3: sys.stderr.write(u'attempt %s/%s\n' % (n+1, retries)) continue - sys.stderr.write(u'Error %s: %s - %s\n\n' % (http_status, message, reason)) + sys.stderr.write(u'{0}{1}: {2} - {3}\n'.format(ERROR_PREFIX, http_status, message, reason)) if soft_errors: if n != 1: sys.stderr.write(u' - Giving up.\n') return sys.exit(int(http_status)) except oauth2client.client.AccessTokenRefreshError, e: - sys.stderr.write(u'Error: Authentication Token Error - %s' % e) + sys.stderr.write(u'{0}Authentication Token Error: {1}\n'.format(ERROR_PREFIX, e)) sys.exit(403) except httplib2.CertificateValidationUnsupported: noPythonSSLExit() @@ -525,11 +568,9 @@ def getServiceFromDiscoveryDocument(api, version, http): else: pyinstaller_disc_file = None if os.path.isfile(disc_file): - with open(disc_file, 'rb') as f: - discovery = f.read() + discovery = readFile(disc_file) elif pyinstaller_disc_file: - with open(pyinstaller_disc_file, 'rb') as f: - discovery = f.read() + discovery = readFile(pyinstaller_disc_file) else: systemErrorExit(4, MESSAGE_NO_DISCOVERY_INFORMATION.format(disc_file)) return googleapiclient.discovery.build_from_document(discovery, base=u'https://www.googleapis.com', http=http) @@ -582,9 +623,8 @@ def buildGAPIServiceObject(api, act_as=None, soft_errors=False): oauth2servicefile = os.path.join(gamUserConfigDir, os.environ.get(u'OAUTHSERVICEFILE', 'oauth2service')) oauth2servicefilejson = u'%s.json' % oauth2servicefile oauth2servicefilep12 = u'%s.p12' % oauth2servicefile - try: - json_string = open(oauth2servicefilejson).read() - except IOError, e: + json_string = readFile(oauth2servicefilejson, continueOnError=True, displayError=True) + if not json_string: printLine(MESSAGE_WIKI_INSTRUCTIONS_OAUTH2SERVICE_JSON) printLine(GAM_WIKI_CREATE_CLIENT_SECRETS) sys.exit(6) @@ -592,9 +632,7 @@ def buildGAPIServiceObject(api, act_as=None, soft_errors=False): try: SERVICE_ACCOUNT_EMAIL = json_data[u'web'][u'client_email'] SERVICE_ACCOUNT_CLIENT_ID = json_data[u'web'][u'client_id'] - f = file(oauth2servicefilep12, 'rb') - key = f.read() - f.close() + key = readFile(oauth2servicefilep12) except KeyError: # new format with config and data in the .json file... SERVICE_ACCOUNT_EMAIL = json_data[u'client_email'] @@ -652,7 +690,7 @@ def buildDiscoveryObject(api): try: return json.loads(content) except ValueError: - sys.stderr.write(u'Failed to parse as JSON: ' + content+u'\n') + sys.stderr.write(u'{0}Failed to parse as JSON: {1}\n'.format(ERROR_PREFIX, content)) raise googleapiclient.errors.InvalidJsonError() def commonAppsObjInit(appsObj): @@ -685,7 +723,7 @@ def getResCalObject(): def geturl(url, dst): import urllib2 u = urllib2.urlopen(url) - f = open(dst, 'wb') + f = openFile(dst, 'wb') meta = u.info() try: file_size = int(meta.getheaders(u'Content-Length')[0]) @@ -705,7 +743,7 @@ def geturl(url, dst): status = r"%10d [unknown size]" % (file_size_dl) status = status + chr(8)*(len(status)+1) print status, - f.close() + closeFile(f) def showReport(): report = sys.argv[2].lower() @@ -1181,8 +1219,7 @@ def doGetDomainInfo(): if (len(sys.argv) < 4) or (sys.argv[3] == u'logo'): doGetInstanceInfo() return - else: - domainName = sys.argv[3] + domainName = sys.argv[3] cd = buildGAPIObject(u'directory') result = callGAPI(service=cd.domains(), function=u'get', customer=customerId, domainName=domainName) if u'creationTime' in result: @@ -2184,13 +2221,11 @@ def doPrintJobFetch(): fileName = u''.join(c if c in valid_chars else u'_' for c in fileName) fileName = u'%s-%s' % (fileName, jobid) _, content = cp._http.request(uri=fileUrl, method='GET') - f = open(fileName, 'wb') - f.write(content) - f.close() - #ticket = callGAPI(service=cp.jobs(), function=u'getticket', jobid=jobid, use_cjt=True) - result = callGAPI(service=cp.jobs(), function=u'update', jobid=jobid, semantic_state_diff=ssd) - checkCloudPrintResult(result) - print u'Printed job %s to %s' % (jobid, fileName) + if writeFile(fileName, content, continueOnError=True): +# ticket = callGAPI(service=cp.jobs(), function=u'getticket', jobid=jobid, use_cjt=True) + result = callGAPI(service=cp.jobs(), function=u'update', jobid=jobid, semantic_state_diff=ssd) + checkCloudPrintResult(result) + print u'Printed job %s to %s' % (jobid, fileName) def doDelPrinter(): cp = buildGAPIObject(u'cloudprint') @@ -2331,9 +2366,7 @@ def doPrintJobSubmit(): mimetype = mimetypes.guess_type(filepath)[0] if mimetype == None: mimetype = u'application/octet-stream' - f = open(filepath, 'rb') - filecontent = f.read() - f.close() + filecontent = readFile(filepath) form_files[u'content'] = {u'filename': content, u'content': filecontent, u'mimetype': mimetype} #result = callGAPI(service=cp.printers(), function=u'submit', body=body) body, headers = encode_multipart(form_fields, form_files) @@ -2641,9 +2674,8 @@ def doPhoto(users): continue else: try: - f = open(filename, 'rb') - image_data = f.read() - f.close() + with open(filename, 'rb') as f: + image_data = f.read() except IOError, e: print u' couldn\'t open %s: %s' % (filename, e.strerror) continue @@ -2675,9 +2707,7 @@ def getPhoto(users): except KeyError: print u' no photo for %s' % user continue - photo_file = open(filename, 'wb') - photo_file.write(photo_data) - photo_file.close() + writeFile(filename, photo_data, continueOnError=True) def deletePhoto(users): cd = buildGAPIObject(u'directory') @@ -3025,13 +3055,21 @@ def showDriveFiles(users): elif attrib_type is unicode or attrib_type is bool: a_file[attrib] = f_file[attrib] elif attrib_type is dict: - for dict_attrib in f_file[attrib]: - if dict_attrib in [u'kind', u'etags', u'etag']: - continue - if dict_attrib not in titles: - titles.append(dict_attrib) - files_attr[0][dict_attrib] = dict_attrib - a_file[dict_attrib] = f_file[attrib][dict_attrib] + if attrib == u'labels': + for dict_attrib in f_file[attrib]: + if dict_attrib not in titles: + titles.append(dict_attrib) + files_attr[0][dict_attrib] = dict_attrib + a_file[dict_attrib] = f_file[attrib][dict_attrib] + else: + for dict_attrib in f_file[attrib]: + if dict_attrib in [u'kind', u'etags', u'etag']: + continue + x_attrib = u'{0}.{1}'.format(attrib, dict_attrib) + if x_attrib not in titles: + titles.append(x_attrib) + files_attr[0][x_attrib] = x_attrib + a_file[x_attrib] = f_file[attrib][dict_attrib] else: print attrib_type files_attr.append(a_file) @@ -3492,9 +3530,7 @@ def downloadDriveFile(users): filename = new_filename print convertUTF8(my_line % filename) _, content = drive._http.request(download_url) - f = open(filename, 'wb') - f.write(content) - f.close() + writeFile(filename, content, continueOnError=True) def showDriveFileInfo(users): for user in users: @@ -3516,7 +3552,7 @@ def showDriveFileInfo(users): if setti == u'kind': continue print convertUTF8(u' %s: %s' % (setti, settin[setti])) - print '' + print u'' elif setting_type == u"": print u'%s:' % setting for settin in feed[setting]: @@ -4384,9 +4420,7 @@ def getForward(users): def doSignature(users): import cgi if sys.argv[4].lower() == u'file': - fp = open(sys.argv[5], 'rb') - signature = cgi.escape(fp.read().replace(u'\\n', u' ').replace(u'"', u"'")) - fp.close() + signature = cgi.escape(readFile(sys.argv[5]).replace(u'\\n', u' ').replace(u'"', u"'")) else: signature = cgi.escape(sys.argv[4]).replace(u'\\n', u' ').replace(u'"', u"'") xmlsig = u''' @@ -4476,9 +4510,7 @@ def doVacation(users): end_date = sys.argv[i+1] i += 2 elif sys.argv[i].lower() == u'file': - fp = open(sys.argv[i+1], 'rb') - message = fp.read() - fp.close() + message = readFile(sys.argv[i+1]) i += 2 else: print u'ERROR: %s is not a valid argument for "gam vacation"' % sys.argv[i] @@ -4916,9 +4948,6 @@ def doCreateUser(): except KeyError: body[u'externalIds'] = [externalid,] i += 1 -# else: -# showUsage() -# sys.exit(2) else: if u'customSchemas' not in body: body[u'customSchemas'] = {} @@ -5963,7 +5992,7 @@ def doGetUserInfo(user_email=None): for address in user[u'addresses']: for key in address: print convertUTF8(u' %s: %s' % (key, address[key])) - print '' + print u'' if u'organizations' in user: print u'Organizations:' for org in user[u'organizations']: @@ -6265,9 +6294,7 @@ def doSiteVerifyShow(): webserver_file_record = callGAPI(service=verif.webResource(), function=u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'FILE'}) webserver_file_token = webserver_file_record[u'token'] print u'Saving web server verification file to: %s' % webserver_file_token - f = open(webserver_file_token, 'wb') - f.write(u'google-site-verification: %s' % webserver_file_token) - f.close() + writeFile(webserver_file_token, u'google-site-verification: {0}'.format(webserver_file_token), continueOnError=True) print u'Verification File URL: http://%s/%s' % (a_domain, webserver_file_token) print webserver_meta_record = callGAPI(service=verif.webResource(), function=u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'META'}) @@ -6472,7 +6499,7 @@ def doGenBackupCodes(users): callGAPI(service=cd.verificationCodes(), function=u'generate', userKey=user) codes = callGAPI(service=cd.verificationCodes(), function=u'list', userKey=user) print u'Backup verification codes for %s' % user - print '' + print u'' try: i = 0 while True: @@ -6537,7 +6564,7 @@ def doGetTokens(users): print u' %s: %s' % (item, token[item]) except UnicodeEncodeError: print u' %s: %s' % (item, token[item][:-1]) - print '' + print u'' except KeyError: print u' no tokens for %s' % user print u'' @@ -6593,15 +6620,9 @@ def doUpdateInstance(): admin_secondary_email = sys.argv[4] callGData(service=adminObj, function=u'UpdateAdminSecondaryEmail', adminSecondaryEmail=admin_secondary_email) elif command == u'logo': - logo_file = sys.argv[4] - try: - fp = open(logo_file, 'rb') - logo_image = fp.read() - fp.close() - except IOError: - print u'Error: can\'t open file %s' % logo_file - sys.exit(11) - callGData(service=adminObj, function=u'UpdateDomainLogo', logoImage=logo_image) + logoFile = sys.argv[4] + logoImage = readFile(logoFile) + callGData(service=adminObj, function=u'UpdateDomainLogo', logoImage=logoImage) elif command == u'mx_verify': result = callGData(service=adminObj, function=u'UpdateMXVerificationStatus') print u'Verification Method: %s' % result[u'verificationMethod'] @@ -6645,15 +6666,9 @@ def doUpdateInstance(): sys.exit(2) callGData(service=adminObj, function=u'UpdateSSOSettings', enableSSO=enableSSO, samlSignonUri=samlSignonUri, samlLogoutUri=samlLogoutUri, changePasswordUri=changePasswordUri, ssoWhitelist=ssoWhitelist, useDomainSpecificIssuer=useDomainSpecificIssuer) elif command == u'sso_key': - key_file = sys.argv[4] - try: - fp = open(key_file, 'rb') - key_data = fp.read() - fp.close() - except IOError: - print u'Error: can\'t open file %s' % logo_file - sys.exit(11) - callGData(service=adminObj, function=u'UpdateSSOKey', signingKey=key_data) + keyFile = sys.argv[4] + keyData = readFile(keyFile) + callGData(service=adminObj, function=u'UpdateSSOKey', signingKey=keyData) elif command == u'user_migrations': value = sys.argv[4].lower() if value not in [u'true', u'false']: @@ -6729,7 +6744,7 @@ def doGetInstanceInfo(): target_file = sys.argv[4] url = 'http://www.google.com/a/cpanel/%s/images/logo.gif' % (domain) geturl(url, target_file) - sys.exit(0) + return print u'Google Apps Domain: %s' % domain cd = buildGAPIObject(u'directory') if customerId != MY_CUSTOMER: @@ -6920,7 +6935,6 @@ def doDeleteOrg(): def output_csv(csv_list, titles, list_type, todrive): csv.register_dialect(u'nixstdout', lineterminator=u'\n') if todrive: - import StringIO string_file = StringIO.StringIO() writer = csv.DictWriter(string_file, fieldnames=titles, dialect=u'nixstdout', quoting=csv.QUOTE_MINIMAL) else: @@ -7908,7 +7922,7 @@ def doStatusActivityRequests(): print u' Url%s: %s' % (i, results[u'fileUrl%s' % i]) except KeyError: pass - print '' + print u'' except IndexError: results = callGData(service=audit, function=u'getAllAccountInformationRequestsStatus') print u'Current Activity Requests:' @@ -8350,8 +8364,7 @@ def doDeleteOAuth(): try: credentials.revoke_uri = oauth2client.GOOGLE_REVOKE_URI except AttributeError: - print u'Error: Authorization doesn\'t exist' - sys.exit(1) + systemErrorExit(1, u'Authorization doesn\'t exist') disable_ssl_certificate_validation = False if os.path.isfile(os.path.join(gamUserConfigDir, u'noverifyssl.txt')): disable_ssl_certificate_validation = True @@ -8368,7 +8381,7 @@ def doDeleteOAuth(): try: credentials.revoke(http) except oauth2client.client.TokenRevokeError, e: - print u'Error: %s' % e + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e.message)) os.remove(oauth2file) class cmd_flags(object): @@ -8555,6 +8568,8 @@ def batch_worker(): q.task_done() def run_batch(items): + import Queue, threading + global q total_items = len(items) current_item = 0 python_cmd = [sys.executable.lower(),] @@ -8564,8 +8579,7 @@ def run_batch(items): num_worker_threads = int(os.environ.get(u'GAM_THREADS', '5')) except TypeError: num_worker_threads = 5 - import Queue, threading - global q + num_worker_threads = min(total_items, num_worker_threads) q = Queue.Queue(maxsize=num_worker_threads) # q.put() gets blocked when trying to create more items than there are workers print u'starting %s worker threads...' % num_worker_threads for i in range(num_worker_threads): @@ -8591,10 +8605,9 @@ try: if os.name == u'nt': sys.argv = win32_unicode_argv() # cleanup sys.argv on Windows setGamDirs() - doGAMCheckForUpdates() if sys.argv[1].lower() == u'batch': import shlex - f = file(sys.argv[2], 'rb') + f = openFile(sys.argv[2]) items = list() for line in f: argv = shlex.split(line) @@ -8606,16 +8619,12 @@ try: if argv[0] == u'gam': argv = argv[1:] items.append(argv) + closeFile(f) run_batch(items) sys.exit(0) elif sys.argv[1].lower() == u'csv': csv_filename = sys.argv[2] - if csv_filename == u'-': - import StringIO - input_string = unicode(sys.stdin.read()) - f = StringIO.StringIO(input_string) - else: - f = file(csv_filename, 'rb') + f = openFile(csv_filename) input_file = csv.DictReader(f) if sys.argv[3].lower() != 'gam': print 'ERROR: "gam csv " should be followed by a full GAM command...' @@ -8632,6 +8641,7 @@ try: else: systemErrorExit(2, MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(arg[1:], ','.join(row.keys()))) items.append(argv) + closeFile(f) run_batch(items) sys.exit(0) elif sys.argv[1].lower() == u'version': From ab6f8fa7bf8bb4dfe736ac75bc444039d71d4ede Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Sun, 22 Nov 2015 10:57:00 -0800 Subject: [PATCH 4/8] Update doVacation --- src/gam.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/gam.py b/src/gam.py index cbfd009c..446ce7f9 100755 --- a/src/gam.py +++ b/src/gam.py @@ -209,7 +209,7 @@ def printLine(message): # # Open a file # -def openFile(filename, mode='rb'): +def openFile(filename, mode=u'rb'): try: if filename != u'-': return open(filename, mode) @@ -231,7 +231,7 @@ def closeFile(f): # # Read a file # -def readFile(filename, mode='rb', continueOnError=False, displayError=True): +def readFile(filename, mode=u'rb', continueOnError=False, displayError=True): try: if filename != u'-': with open(filename, mode) as f: @@ -247,7 +247,7 @@ def readFile(filename, mode='rb', continueOnError=False, displayError=True): # # Write a file # -def writeFile(filename, data, mode='wb', continueOnError=False, displayError=True): +def writeFile(filename, data, mode=u'wb', continueOnError=False, displayError=True): try: with open(filename, mode) as f: f.write(data) @@ -723,7 +723,7 @@ def getResCalObject(): def geturl(url, dst): import urllib2 u = urllib2.urlopen(url) - f = openFile(dst, 'wb') + f = openFile(dst, u'wb') meta = u.info() try: file_size = int(meta.getheaders(u'Content-Length')[0]) @@ -1850,7 +1850,7 @@ def changeCalendarAttendees(users): print u'ERROR: %s is not a valid argument for "gam update calattendees"' % sys.argv[i] sys.exit(2) attendee_map = dict() - csvfile = csv.reader(open(csv_file, 'rb')) + csvfile = csv.reader(open(csv_file, u'rb')) for row in csvfile: attendee_map[row[0].lower()] = row[1].lower() for user in users: @@ -2674,7 +2674,7 @@ def doPhoto(users): continue else: try: - with open(filename, 'rb') as f: + with open(filename, u'rb') as f: image_data = f.read() except IOError, e: print u' couldn\'t open %s: %s' % (filename, e.strerror) @@ -4478,7 +4478,6 @@ def doWebClips(users): callGData(service=emailsettings, function=u'UpdateWebClipSettings', soft_errors=True, username=user, enable=enable) def doVacation(users): - import cgi subject = message = u'' if sys.argv[4] in true_values: enable = u'true' @@ -4518,19 +4517,7 @@ def doVacation(users): i = 1 count = len(users) emailsettings = getEmailSettingsObject() - message = cgi.escape(message).replace(u'\\n', u' ').replace(u'"', u"'") - vacxml = u''' - - ''' % enable - vacxml += u''' - - - ''' % (subject, message, contacts_only, domain_only) - if start_date != None: - vacxml += u'''''' % start_date - if end_date != None: - vacxml += u'''''' % end_date - vacxml += u'' + message = message.replace(u'\\n', u'\n') for user in users: if user.find(u'@') > 0: emailsettings.domain = user[user.find(u'@')+1:] @@ -4538,9 +4525,11 @@ def doVacation(users): else: emailsettings.domain = domain #make sure it's back at default domain print u"Setting Vacation for %s (%s of %s)" % (user+'@'+emailsettings.domain, i, count) - uri = u'https://apps-apis.google.com/a/feeds/emailsettings/2.0/%s/%s/vacation' % (emailsettings.domain, user) i += 1 - callGData(service=emailsettings, function=u'Put', soft_errors=True, data=vacxml, uri=uri) + callGData(service=emailsettings, function=u'UpdateVacation', + soft_errors=True, + username=userName, enable=enable, subject=subject, message=message, + contacts_only=contacts_only, domain_only=domain_only, start_date=start_date, end_date=end_date) def getVacation(users): emailsettings = getEmailSettingsObject() @@ -8244,7 +8233,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, return_uids=Fa elif entity_type == u'file': users = [] filename = entity - usernames = csv.reader(open(filename, 'rb')) + usernames = csv.reader(open(filename, u'rb')) for row in usernames: try: users.append(row.pop()) From edb17ad06e6706bfca615b8469b81280c8b99f65 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 24 Nov 2015 16:51:06 -0800 Subject: [PATCH 5/8] Allow reader in doCalendarAddACL, fix infinite loops in doPop, doCreateUser, doUpdateUser --- src/gam.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index 446ce7f9..7f22e239 100755 --- a/src/gam.py +++ b/src/gam.py @@ -2433,12 +2433,12 @@ def doCalendarAddACL(calendarId=None, act_as=None, role=None, scope=None, entity body[u'role'] = role else: body[u'role'] = sys.argv[4].lower() - if body[u'role'] not in [u'freebusy', u'read', u'editor', u'owner', u'none']: + if body[u'role'] not in [u'freebusy', u'read', u'reader', u'editor', u'owner', u'none']: print u'ERROR: Role must be freebusy, read, editor, owner or none. Not %s' % body['role'] sys.exit(2) if body[u'role'] == u'freebusy': body[u'role'] = u'freeBusyReader' - elif body[u'role'] == u'read': + elif body[u'role'] in [u'read', u'reader']: body[u'role'] = u'reader' elif body[u'role'] == u'editor': body[u'role'] = u'writer' @@ -3778,6 +3778,9 @@ def doPop(users): elif sys.argv[i+1].lower() == u'newmail': enable_for = u'MAIL_FROM_NOW_ON' i += 2 + else: + print u'ERROR: %s is not a valid argument for "gam pop for"' % sys.argv[i] + sys.exit(2) elif sys.argv[i].lower() == u'action': if sys.argv[i+1].lower() == u'keep': action = u'KEEP' @@ -3788,6 +3791,9 @@ def doPop(users): elif sys.argv[i+1].lower() == u'delete': action = u'DELETE' i += 2 + else: + print u'ERROR: %s is not a valid argument for "gam pop action"' % sys.argv[i] + sys.exit(2) elif sys.argv[i].lower() == u'confirm': i += 1 else: @@ -4828,6 +4834,9 @@ def doCreateUser(): address[u'primary'] = True i += 1 break + else: + print u'ERROR: invalid argument (%s) for account address details' % sys.argv[i] + sys.exit(2) try: body[u'addresses'].append(address) except KeyError: @@ -4877,6 +4886,9 @@ def doCreateUser(): organization[u'primary'] = True i += 1 break + else: + print u'ERROR: invalid argument (%s) for account organization details' % sys.argv[i] + sys.exit(2) try: body[u'organizations'].append(organization) except KeyError: @@ -4905,6 +4917,9 @@ def doCreateUser(): phone[u'primary'] = True i += 1 break + else: + print u'ERROR: invalid argument (%s) for account phone details' % sys.argv[i] + sys.exit(2) try: body[u'phones'].append(phone) except KeyError: @@ -5305,6 +5320,9 @@ def doUpdateUser(users): address[u'primary'] = True i += 1 break + else: + print u'ERROR: invalid argument (%s) for account address details' % sys.argv[i] + sys.exit(2) try: body[u'addresses'].append(address) except KeyError: @@ -5326,7 +5344,7 @@ def doUpdateUser(users): i += 2 elif argument == u'type': organization[u'type'] = sys.argv[i+1].lower() - if organization[u'type'] not in [u'domain_only', 'school', 'unknown', 'work']: + if organization[u'type'] not in [u'domain_only', u'school', u'unknown', u'work']: print u'ERROR: organization type must be domain_only, school, unknown or work. Got %s' % organization[u'type'] sys.exit(2) i += 2 @@ -5355,6 +5373,9 @@ def doUpdateUser(users): organization[u'primary'] = True i += 1 break + else: + print u'ERROR: invalid argument (%s) for account organization details' % sys.argv[i] + sys.exit(2) try: body[u'organizations'].append(organization) except KeyError: @@ -5384,6 +5405,9 @@ def doUpdateUser(users): phone[u'primary'] = True i += 1 break + else: + print u'ERROR: invalid argument (%s) for account phone details' % sys.argv[i] + sys.exit(2) try: body[u'phones'].append(phone) except KeyError: From 05920cc7d7100e14858a93a707d6c1d5ab200d7a Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 24 Nov 2015 17:14:37 -0800 Subject: [PATCH 6/8] Boolean values were not downshifted in doCreateUser --- src/gam.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gam.py b/src/gam.py index 7f22e239..e1e892a4 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4703,18 +4703,18 @@ def doCreateUser(): need_to_hash_password = False i += 1 elif sys.argv[i].lower() == u'changepassword': - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'changePasswordAtNextLogin'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'changePasswordAtNextLogin'] = False else: print u'ERROR: changepassword should be on or off, not %s' % sys.argv[i+1] sys.exit(2) i += 2 elif sys.argv[i].lower() == u'ipwhitelisted': - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'ipWhitelisted'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'ipWhitelisted'] = False else: print u'ERROR: ipwhitelisted should be on or off, not %s' % sys.argv[i+1] @@ -4731,9 +4731,9 @@ def doCreateUser(): sys.exit(2) i += 2 elif sys.argv[i].lower() == u'agreedtoterms': - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'agreedToTerms'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'agreedToTerms'] = False else: print u'ERROR: agreedtoterms should be on or off, not %s' % sys.argv[i+1] From 4da936344f5df010f1e26716bf22bf8c46e2a691 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Tue, 24 Nov 2015 18:10:23 -0800 Subject: [PATCH 7/8] Downshift more Boolean values --- src/gam.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/gam.py b/src/gam.py index e1e892a4..dcd56bfd 100755 --- a/src/gam.py +++ b/src/gam.py @@ -3194,9 +3194,9 @@ def doUpdateDriveFile(users): elif sys.argv[i].lower() in [u'restrict', 'restricted']: if 'labels' not in body: body[u'labels'] = dict() - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'labels'][u'restricted'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'labels'][u'restricted'] = False else: print u'ERROR: value for restricted must be true or false, got %s' % sys.argv[i+1] @@ -3205,9 +3205,9 @@ def doUpdateDriveFile(users): elif sys.argv[i].lower() in [u'star', u'starred']: if u'labels' not in body: body[u'labels'] = dict() - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'labels'][u'starred'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'labels'][u'starred'] = False else: print u'ERROR: value for starred must be true or false, got %s' % sys.argv[i+1] @@ -3216,9 +3216,9 @@ def doUpdateDriveFile(users): elif sys.argv[i].lower() in [u'trash', u'trashed']: if u'labels' not in body: body[u'labels'] = dict() - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'labels'][u'trashed'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'labels'][u'trashed'] = False else: print u'ERROR: value for trashed must be true or false, got %s' % sys.argv[i+1] @@ -3227,9 +3227,9 @@ def doUpdateDriveFile(users): elif sys.argv[i].lower() in [u'view', u'viewed']: if u'labels' not in body: body[u'labels'] = dict() - if sys.argv[i+1] in true_values: + if sys.argv[i+1].lower() in true_values: body[u'labels'][u'viewed'] = True - elif sys.argv[i+1] in false_values: + elif sys.argv[i+1].lower() in false_values: body[u'labels'][u'viewed'] = False else: print u'ERROR: value for viewed must be true or false, got %s' % sys.argv[i+1] @@ -4371,9 +4371,9 @@ def doFilter(users): def doForward(users): action = forward_to = None gotAction = gotForward = False - if sys.argv[4] in true_values: + if sys.argv[4].lower() in true_values: enable = True - elif sys.argv[4] in false_values: + elif sys.argv[4].lower() in false_values: enable = False else: print u'ERROR: value for "gam forward" must be true or false, got %s' % sys.argv[4] @@ -4485,9 +4485,9 @@ def doWebClips(users): def doVacation(users): subject = message = u'' - if sys.argv[4] in true_values: + if sys.argv[4].lower() in true_values: enable = u'true' - elif sys.argv[4] in false_values: + elif sys.argv[4].lower() in false_values: enable = u'false' else: print u'ERROR: value for "gam vacation" must be true or false, got %s' % sys.argv[4] From 21f01757a3cc31b3c4835f384aee2619b7060022 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Thu, 17 Dec 2015 17:11:17 -0800 Subject: [PATCH 8/8] Make sure that callGData and callGAPI return something --- src/gam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gam.py b/src/gam.py index dcd56bfd..1d69a7d0 100755 --- a/src/gam.py +++ b/src/gam.py @@ -417,7 +417,7 @@ def callGData(service, function, soft_errors=False, throw_errors=[], **kwargs): if soft_errors: if n != 1: sys.stderr.write(u' - Giving up.\n') - return + return None sys.exit(int(e.error_code)) def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_reasons=[], retry_reasons=[], **kwargs): @@ -443,7 +443,7 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re if not silent_errors: sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e.content)) if soft_errors: - return + return None sys.exit(5) http_status = error[u'error'][u'code'] message = error[u'error'][u'errors'][0][u'message'] @@ -467,7 +467,7 @@ def callGAPI(service, function, silent_errors=False, soft_errors=False, throw_re if soft_errors: if n != 1: sys.stderr.write(u' - Giving up.\n') - return + return None sys.exit(int(http_status)) except oauth2client.client.AccessTokenRefreshError, e: sys.stderr.write(u'{0}Authentication Token Error: {1}\n'.format(ERROR_PREFIX, e))