From 0cf964073d8e18d129ebae6f262933527bc9424b Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 24 Apr 2019 10:40:35 -0700 Subject: [PATCH] Code cleanup (#900) * Code cleanup * Add missing _ * Add missing character One character was missing from the prefix, I assumed :, did you want a space? * Put missing variable back * More cleanup repairs --- src/gam.py | 74 ++++++++++++++++++++++++++-------------------------- src/utils.py | 6 ++++- src/var.py | 71 +++++++++++++++++++------------------------------ 3 files changed, 69 insertions(+), 82 deletions(-) diff --git a/src/gam.py b/src/gam.py index 86ac77c7..cb930a99 100755 --- a/src/gam.py +++ b/src/gam.py @@ -3,7 +3,7 @@ # # GAM # -# Copyright 2015, LLC All Rights Reserved. +# Copyright 2019, LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,49 +22,50 @@ With GAM you can programatically create users, turn on/off services for users li For more information, see https://git.io/gam """ -import sys -import os -import string -import time import base64 import codecs import configparser import csv import datetime import hashlib -import http.client as http_client +import importlib +import io import json import mimetypes +import os import platform import random import re +import shlex import signal import socket -import io +import ssl +import string import struct -from urllib.parse import urlencode +import sys +import time import uuid import webbrowser import zipfile +import http.client as http_client from email.mime.text import MIMEText -import shlex from multiprocessing import Pool from multiprocessing import freeze_support - -import dateutil.parser +from urllib.parse import urlencode +from passlib.hash import sha512_crypt import dns.resolver +import dateutil.parser import googleapiclient import googleapiclient.discovery import googleapiclient.errors import googleapiclient.http -import httplib2 import google.oauth2.service_account import google_auth_httplib2 +import httplib2 import oauth2client.client import oauth2client.file import oauth2client.tools -from passlib.hash import sha512_crypt from oauth2client.contrib.dictionary_storage import DictionaryStorage import utils @@ -129,9 +130,9 @@ def _request_with_user_agent(request_method): return wrapped_request_method google_auth_httplib2.Request.__call__ = _request_with_user_agent( - google_auth_httplib2.Request.__call__) + google_auth_httplib2.Request.__call__) google_auth_httplib2.AuthorizedHttp.request = _request_with_user_agent( - google_auth_httplib2.AuthorizedHttp.request) + google_auth_httplib2.AuthorizedHttp.request) def showUsage(): doGAMVersion(checkForArgs=False) @@ -690,7 +691,6 @@ def doGAMVersion(checkForArgs=True): if force_check: doGAMCheckForUpdates(forceCheck=True) if extended: - import ssl, socket, importlib print(ssl.OPENSSL_VERSION) proot = os.path.dirname(importlib.import_module('httplib2').__file__) ca_path = os.path.join(proot, 'cacerts.txt') @@ -1382,7 +1382,7 @@ def showReport(): elif myarg == 'fulldatarequired': fullDataRequired = [] fdr = sys.argv[i+1].lower() - if len(fdr) > 0 and fdr != 'all': + if fdr and fdr != 'all': fullDataRequired = fdr.replace(',', ' ').split() i += 2 elif myarg == 'start': @@ -1533,7 +1533,7 @@ def showReport(): activities = callGAPIpages(rep.activities(), 'list', 'items', page_message=page_message, applicationName=report, userKey=userKey, customerId=customerId, actorIpAddress=actorIpAddress, startTime=startTime, endTime=endTime, eventName=eventName, filters=filters) - if len(activities) > 0: + if activities: titles = ['name'] csvRows = [] for activity in activities: @@ -2443,7 +2443,7 @@ def doDeleteGuardian(): throw_reasons=[GAPI_FORBIDDEN], studentId=studentId, invitedEmailAddress=guardianId, fields='nextPageToken,guardians(studentId,guardianId)') - if len(results) > 0: + if results: for result in results: _deleteGuardian(croom, result['studentId'], result['guardianId'], guardianId) return @@ -2460,7 +2460,7 @@ def doDeleteGuardian(): throw_reasons=[GAPI_FORBIDDEN], studentId=studentId, invitedEmailAddress=guardianId, states=['PENDING',], fields='nextPageToken,guardianInvitations(studentId,invitationId)') - if len(results) > 0: + if results: for result in results: status = _cancelGuardianInvitation(croom, result['studentId'], result['invitationId']) sys.exit(status) @@ -2722,7 +2722,7 @@ def doPrintCourseParticipants(): i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam print course-participants"' % sys.argv[i]) - if len(courses) == 0: + if not courses: printGettingAllItems('Courses', None) page_message = 'Got %%num_items%% Courses...\n' all_courses = callGAPIpages(croom.courses(), 'list', 'courses', page_message=page_message, @@ -4474,7 +4474,7 @@ def deleteEmptyDriveFolders(users): for folder in feed: children = callGAPI(drive.children(), 'list', folderId=folder['id'], fields='items(id)', maxResults=1) - if not 'items' in children or len(children['items']) == 0: + if 'items' not in children or not children['items']: print(utils.convertUTF8(' deleting empty folder %s...' % folder['title'])) callGAPI(drive.files(), 'delete', fileId=folder['id']) deleted_empty = True @@ -4802,7 +4802,7 @@ def downloadDriveFile(users): safe_file_title = targetName else: safe_file_title = ''.join(c for c in result['title'] if c in safe_filename_chars) - if len(safe_file_title) < 1: + if not safe_file_title: safe_file_title = fileId filename = os.path.join(targetFolder, safe_file_title) y = 0 @@ -5390,9 +5390,9 @@ def updateSmime(users): if not smimeIdBase: result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'list', userId='me', sendAsEmail=sendAsEmail, fields='smimeInfo(id)') smimes = result.get('smimeInfo', []) - if len(smimes) == 0: + if not smimes: systemErrorExit(3, '%s has no S/MIME certificates for sendas address %s' % (user, sendAsEmail)) - elif len(smimes) > 1: + if len(smimes) > 1: systemErrorExit(3, '%s has more than one S/MIME certificate. Please specify a cert to update:\n %s' % (user, '\n '.join([smime['id'] for smime in smimes]))) smimeId = smimes[0]['id'] else: @@ -5422,9 +5422,9 @@ def deleteSmime(users): if not smimeIdBase: result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'list', userId='me', sendAsEmail=sendAsEmail, fields='smimeInfo(id)') smimes = result.get('smimeInfo', []) - if len(smimes) == 0: + if not smimes: systemErrorExit(3, '%s has no S/MIME certificates for sendas address %s' % (user, sendAsEmail)) - elif len(smimes) > 1: + if len(smimes) > 1: systemErrorExit(3, '%s has more than one S/MIME certificate. Please specify a cert to delete:\n %s' % (user, '\n '.join([smime['id'] for smime in smimes]))) smimeId = smimes[0]['id'] else: @@ -5960,7 +5960,7 @@ def renameLabels(users): print(' Merging %s label to existing %s label' % (label['name'], new_label_name)) messages_to_relabel = callGAPIpages(gmail.users().messages(), 'list', 'messages', userId=user, q='label:%s' % label['name'].lower().replace('/', '-').replace(' ', '-')) - if len(messages_to_relabel) > 0: + if messages_to_relabel: for new_label in labels['labels']: if new_label['name'].lower() == new_label_name.lower(): new_label_id = new_label['id'] @@ -9692,7 +9692,7 @@ def doGetUserInfo(user_email=None): print(' %s' % alias) if getGroups: groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user_email, fields='groups(name,email),nextPageToken') - if len(groups) > 0: + if groups: print('Groups: (%s)' % len(groups)) for group in groups: print(' %s <%s>' % (group['name'], group['email'])) @@ -10053,7 +10053,7 @@ def doSiteVerifyShow(): def doGetSiteVerifications(): verif = buildGAPIObject('siteVerification') sites = callGAPIitems(verif.webResource(), 'list', 'items') - if len(sites) > 0: + if sites: for site in sites: print('Site: %s' % site['site']['identifier']) print('Type: %s' % site['site']['type']) @@ -10251,7 +10251,7 @@ def doGetASPs(users): cd = buildGAPIObject('directory') for user in users: asps = callGAPIitems(cd.asps(), 'list', 'items', userKey=user) - if len(asps) > 0: + if asps: print('Application-Specific Passwords for %s' % user) for asp in asps: if asp['creationTime'] == '0': @@ -10480,7 +10480,7 @@ def doUndeleteUser(): for deleted_user in deleted_users: if str(deleted_user['primaryEmail']).lower() == user: matching_users.append(deleted_user) - if len(matching_users) < 1: + if not matching_users: systemErrorExit(3, 'could not find deleted user with that address.') elif len(matching_users) > 1: print('ERROR: more than one matching deleted %s user. Please select the correct one to undelete and specify with "gam undelete user uid:"' % user) @@ -10644,7 +10644,7 @@ def writeCSVfile(csvRows, titles, list_type, todrive): new_csvRows.append(row) csvRows = new_csvRows else: - if filter_str.lower()[:6] == 'regex': + if filter_str.lower()[:6] == 'regex:': filter_str = filter_str[6:] if match_column not in titles: sys.stderr.write('WARNING: Row filter %s is not in output columns\n' % match_column) @@ -11108,7 +11108,7 @@ def doPrintGroups(): else: systemErrorExit(2, '%s is not a valid argument for "gam print groups"' % sys.argv[i]) cdfields = ','.join(set(cdfieldsList)) - if len(gsfieldsList) > 0: + if gsfieldsList: getSettings = True gsfields = ','.join(set(gsfieldsList)) elif getSettings: @@ -13018,7 +13018,7 @@ Append an 'r' to grant read-only access or an 'a' to grant action-only access. if not prompt_again: return except ScopeSelectionMenu.MenuChoiceError as e: - error_message = e.message + error_message = str(e) _SINGLE_SCOPE_CHANGE_REGEX = re.compile( r'\s*(?P\d{1,2})\s*(?P[a-z]?)', re.IGNORECASE) @@ -13063,7 +13063,7 @@ Append an 'r' to grant read-only access or an 'a' to grant action-only access. # Find the restriction that the user intended to apply. if restriction_command != '': matching_restrictions = [r for r in selected_option.supported_restrictions if r.startswith(restriction_command)] - if len(matching_restrictions) < 1: + if not matching_restrictions: raise ScopeSelectionMenu.MenuChoiceError( 'Scope "%s" does not support "%s" mode!' % ( selected_option.description, restriction_command)) @@ -13268,7 +13268,7 @@ def ProcessGAMCommand(args): sys.stderr.write('{0}{1}\n'.format(ERROR_PREFIX, str(e))) errors += 1 continue - if len(argv) > 0: + if argv: cmd = argv[0].strip().lower() if (not cmd) or cmd.startswith('#') or ((len(argv) == 1) and (cmd != 'commit-batch')): continue diff --git a/src/utils.py b/src/utils.py index 5112ff80..14c98a3f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,7 +3,11 @@ import re import sys from html.entities import name2codepoint from html.parser import HTMLParser -from var import GM_Globals, GM_WINDOWS, GM_SYS_ENCODING, ONE_KILO_BYTES, ONE_MEGA_BYTES, ONE_GIGA_BYTES +from var import GM_Globals, GM_WINDOWS, GM_SYS_ENCODING + +ONE_KILO_BYTES = 1000 +ONE_MEGA_BYTES = 1000000 +ONE_GIGA_BYTES = 1000000000 def convertUTF8(data): if isinstance(data, str): diff --git a/src/var.py b/src/var.py index 870b5037..1c5132ee 100644 --- a/src/var.py +++ b/src/var.py @@ -20,8 +20,6 @@ GAM_ALL_RELEASES = 'https://api.github.com/repos/jay0lee/GAM/releases' GAM_LATEST_RELEASE = GAM_ALL_RELEASES+'/latest' GAM_PROJECT_APIS = 'https://raw.githubusercontent.com/jay0lee/GAM/master/src/project-apis.txt' -TRUE = 'true' -FALSE = 'false' true_values = ['on', 'yes', 'enabled', 'true', '1'] false_values = ['off', 'no', 'disabled', 'false', '0'] usergroup_types = ['user', 'users', @@ -30,19 +28,10 @@ usergroup_types = ['user', 'users', 'ou_and_children', 'ou_and_child', 'ou_and_children_ns', 'ou_and_child_ns', 'ou_and_children_susp', 'ou_and_child_susp', 'query', 'queries', 'license', 'licenses', 'licence', 'licences', 'file', 'csv', 'csvfile', 'all', 'cros', 'cros_sn', 'crosquery', 'crosqueries', 'crosfile', 'croscsv', 'croscsvfile'] -ERROR = 'ERROR' -ERROR_PREFIX = ERROR+': ' -WARNING = 'WARNING' -WARNING_PREFIX = WARNING+': ' -DEFAULT_CHARSET = ['mbcs', 'utf-8'][os.name != 'nt'] -ONE_KILO_BYTES = 1000 -ONE_MEGA_BYTES = 1000000 -ONE_GIGA_BYTES = 1000000000 -FN_CLIENT_SECRETS_JSON = 'client_secrets.json' +ERROR_PREFIX = 'ERROR: ' +WARNING_PREFIX = 'WARNING: ' FN_EXTRA_ARGS_TXT = 'extra-args.txt' FN_LAST_UPDATE_CHECK_TXT = 'lastupdatecheck.txt' -FN_OAUTH2SERVICE_JSON = 'oauth2service.json' -FN_OAUTH2_TXT = 'oauth2.txt' MY_CUSTOMER = 'my_customer' # See https://support.google.com/drive/answer/37603 MAX_GOOGLE_SHEET_CELLS = 5000000 @@ -360,18 +349,18 @@ GOOGLEDOC_VALID_EXTENSIONS_MAP = { MIMETYPE_GA_SPREADSHEET: ['.csv', '.ods', '.pdf', '.xlsx', '.zip'], } -MICROSOFT_FORMATS_LIST = [{'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'ext': '.docx'}, - {'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'ext': '.dotx'}, - {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'ext': '.pptx'}, - {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.template', 'ext': '.potx'}, - {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'ext': '.xlsx'}, - {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'ext': '.xltx'}, - {'mime': 'application/msword', 'ext': '.doc'}, - {'mime': 'application/msword', 'ext': '.dot'}, - {'mime': 'application/vnd.ms-powerpoint', 'ext': '.ppt'}, - {'mime': 'application/vnd.ms-powerpoint', 'ext': '.pot'}, - {'mime': 'application/vnd.ms-excel', 'ext': '.xls'}, - {'mime': 'application/vnd.ms-excel', 'ext': '.xlt'}] +_MICROSOFT_FORMATS_LIST = [{'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'ext': '.docx'}, + {'mime': 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'ext': '.dotx'}, + {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'ext': '.pptx'}, + {'mime': 'application/vnd.openxmlformats-officedocument.presentationml.template', 'ext': '.potx'}, + {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'ext': '.xlsx'}, + {'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'ext': '.xltx'}, + {'mime': 'application/msword', 'ext': '.doc'}, + {'mime': 'application/msword', 'ext': '.dot'}, + {'mime': 'application/vnd.ms-powerpoint', 'ext': '.ppt'}, + {'mime': 'application/vnd.ms-powerpoint', 'ext': '.pot'}, + {'mime': 'application/vnd.ms-excel', 'ext': '.xls'}, + {'mime': 'application/vnd.ms-excel', 'ext': '.xlt'}] DOCUMENT_FORMATS_MAP = { 'csv': [{'mime': 'text/csv', 'ext': '.csv'}], @@ -404,9 +393,9 @@ DOCUMENT_FORMATS_MAP = { 'xlsx': [{'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'ext': '.xlsx'}], 'xltx': [{'mime': 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'ext': '.xltx'}], 'zip': [{'mime': 'application/zip', 'ext': '.zip'}], - 'ms': MICROSOFT_FORMATS_LIST, - 'microsoft': MICROSOFT_FORMATS_LIST, - 'micro$oft': MICROSOFT_FORMATS_LIST, + 'ms': _MICROSOFT_FORMATS_LIST, + 'microsoft': _MICROSOFT_FORMATS_LIST, + 'micro$oft': _MICROSOFT_FORMATS_LIST, 'openoffice': [{'mime': 'application/vnd.oasis.opendocument.presentation', 'ext': '.odp'}, {'mime': 'application/x-vnd.oasis.opendocument.spreadsheet', 'ext': '.ods'}, {'mime': 'application/vnd.oasis.opendocument.spreadsheet', 'ext': '.ods'}, @@ -641,11 +630,16 @@ GM_MAP_BUILDING_ID_TO_NAME = 'bi2n' # Dictionary mapping Building Name to ID GM_MAP_BUILDING_NAME_TO_ID = 'bn2i' # +_DEFAULT_CHARSET = ['mbcs', 'utf-8'][os.name != 'nt'] +_FN_CLIENT_SECRETS_JSON = 'client_secrets.json' +_FN_OAUTH2SERVICE_JSON = 'oauth2service.json' +_FN_OAUTH2_TXT = 'oauth2.txt' +# GM_Globals = { GM_SYSEXITRC: 0, GM_GAM_PATH: None, GM_WINDOWS: os.name == 'nt', - GM_SYS_ENCODING: DEFAULT_CHARSET, + GM_SYS_ENCODING: _DEFAULT_CHARSET, GM_EXTRA_ARGS_DICT: {'prettyPrint': False}, GM_CURRENT_API_SERVICES: {}, GM_CURRENT_API_USER: None, @@ -733,8 +727,8 @@ GC_Defaults = { GC_BATCH_SIZE: 50, GC_CACHE_DIR: '', GC_CACHE_DISCOVERY_ONLY: True, - GC_CHARSET: DEFAULT_CHARSET, - GC_CLIENT_SECRETS_JSON: FN_CLIENT_SECRETS_JSON, + GC_CHARSET: _DEFAULT_CHARSET, + GC_CLIENT_SECRETS_JSON: _FN_CLIENT_SECRETS_JSON, GC_CONFIG_DIR: '', GC_CUSTOMER_ID: MY_CUSTOMER, GC_DEBUG_LEVEL: 0, @@ -748,8 +742,8 @@ GC_Defaults = { GC_NO_UPDATE_CHECK: False, GC_NO_VERIFY_SSL: False, GC_NUM_THREADS: 25, - GC_OAUTH2_TXT: FN_OAUTH2_TXT, - GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON, + GC_OAUTH2_TXT: _FN_OAUTH2_TXT, + GC_OAUTH2SERVICE_JSON: _FN_OAUTH2SERVICE_JSON, GC_SECTION: '', GC_SHOW_COUNTS_MIN: 0, GC_SHOW_GETTINGS: True, @@ -807,16 +801,9 @@ GC_VAR_INFO = { # Google API constants NEVER_TIME = '1970-01-01T00:00:00.000Z' -NEVER_START_DATE = '1970-01-01' -NEVER_END_DATE = '1969-12-31' ROLE_MANAGER = 'MANAGER' ROLE_MEMBER = 'MEMBER' ROLE_OWNER = 'OWNER' -ROLE_USER = 'USER' -ROLE_MANAGER_MEMBER = ','.join([ROLE_MANAGER, ROLE_MEMBER]) -ROLE_MANAGER_OWNER = ','.join([ROLE_MANAGER, ROLE_OWNER]) -ROLE_MANAGER_MEMBER_OWNER = ','.join([ROLE_MANAGER, ROLE_MEMBER, ROLE_OWNER]) -ROLE_MEMBER_OWNER = ','.join([ROLE_MEMBER, ROLE_OWNER]) PROJECTION_CHOICES_MAP = {'basic': 'BASIC', 'full': 'FULL',} SORTORDER_CHOICES_MAP = {'ascending': 'ASCENDING', 'descending': 'DESCENDING',} # @@ -830,14 +817,10 @@ MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = 'Header "{0}" not found in CSV headers MESSAGE_HIT_CONTROL_C_TO_UPDATE = '\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.' MESSAGE_INVALID_JSON = 'The file {0} has an invalid format.' MESSAGE_NO_DISCOVERY_INFORMATION = 'No online discovery doc and {0} does not exist locally' -MESSAGE_NO_PYTHON_SSL = 'You don\'t have the Python SSL module installed so we can\'t verify SSL Certificates. You can fix this by installing the Python SSL module or you can live on the edge and turn SSL validation off by creating a file named noverifyssl.txt in the same location as gam.exe / gam.py' MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = 'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb' -MESSAGE_REQUEST_COMPLETED_NO_FILES = 'Request completed but no results/files were returned, try requesting again' -MESSAGE_REQUEST_NOT_COMPLETE = 'Request needs to be completed before downloading, current status is: {0}' MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = 'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.' MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}. Please make sure service is enabled for user and run\n\ngam user check serviceaccount\n\nfor further instructions' MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON = 'Please run\n\ngam create project\ngam user check serviceaccount\n\nto create and configure a service account.' -MESSAGE_OAUTH2SERVICE_JSON_INVALID = 'The file {0} is missing required keys (client_email, client_id or private_key). Please remove it and recreate with the commands:\n\ngam create project\ngam user check serviceaccount' # oauth errors OAUTH2_TOKEN_ERRORS = [ 'access_denied',