From ef86508bbb26ef282435b0f1aabeb68fca82d607 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 13 Feb 2019 12:27:08 -0800 Subject: [PATCH] Add csvsheet and targetname - to get drivefile (#842) * Add csvsheet and targetname - to get drivefile Standarize id: and uid: processing * Update var.py for Sheets API * Handle revisionId in download of non Google files --- src/GamCommands.txt | 5 +- src/gam.py | 196 ++++++++++++++++++++++++++----------------- src/project-apis.txt | 1 + src/var.py | 2 + 4 files changed, 127 insertions(+), 77 deletions(-) diff --git a/src/GamCommands.txt b/src/GamCommands.txt index 1e5d0f65..0c9b8d50 100644 --- a/src/GamCommands.txt +++ b/src/GamCommands.txt @@ -1150,8 +1150,9 @@ gam show filetree [anyowner] (orderby [ gam create|add drivefile [drivefilename ] * [csv] [todrive] gam update drivefile (id )|(query ] * -gam get drivefile (id )|(drivefilename )|(query ) [revision ] [format ] - [targetfolder ] [targetname ] [overwrite] [showprogress] +gam get drivefile (id )|(drivefilename )|(query ) + [revision ] [(format )|(csvsheet )] + [targetfolder ] [targetname -|] [overwrite] [showprogress] gam delete|del drivefile ||(query:) [purge|untrash] gam transfer drive [keepuser] gam delete|del emptydrivefolders diff --git a/src/gam.py b/src/gam.py index b5953e13..fc7c4cb9 100755 --- a/src/gam.py +++ b/src/gam.py @@ -365,6 +365,8 @@ def splitEmailAddress(emailAddress): return (emailAddress.lower(), GC_Values[GC_DOMAIN].lower()) return (emailAddress[:atLoc].lower(), emailAddress[atLoc+1:].lower()) +UID_PATTERN = re.compile(r'u?id: ?(.+)', re.IGNORECASE) + # Normalize user/group email address/uid # uid:12345abc -> 12345abc # foo -> foo@domain @@ -374,11 +376,10 @@ def splitEmailAddress(emailAddress): def normalizeEmailAddressOrUID(emailAddressOrUID, noUid=False, checkForCustomerId=False, noLower=False): if checkForCustomerId and (emailAddressOrUID == GC_Values[GC_CUSTOMER_ID]): return emailAddressOrUID - if (not noUid) and (emailAddressOrUID.find(u':') != -1): - if emailAddressOrUID[:4].lower() == u'uid:': - return emailAddressOrUID[4:] - if emailAddressOrUID[:3].lower() == u'id:': - return emailAddressOrUID[3:] + if not noUid: + cg = UID_PATTERN.match(emailAddressOrUID) + if cg: + return cg.group(1) atLoc = emailAddressOrUID.find(u'@') if atLoc == 0: return emailAddressOrUID[1:].lower() if not noLower else emailAddressOrUID[1:] @@ -1949,12 +1950,7 @@ def doCreateAdmin(): user = normalizeEmailAddressOrUID(sys.argv[3]) body = {u'assignedTo': convertEmailAddressToUID(user, cd)} role = sys.argv[4] - if role[:4].lower() == u'uid:': - body[u'roleId'] = role[4:] - else: - body[u'roleId'] = roleid_from_role(role) - if not body[u'roleId']: - systemErrorExit(4, '%s is not a valid role. Please ensure role name is exactly as shown in admin console.' % role) + body[u'roleId'] = getRoleId(role) body[u'scopeType'] = sys.argv[5].upper() if body[u'scopeType'] not in [u'CUSTOMER', u'ORG_UNIT']: systemErrorExit(3, 'scope type must be customer or org_unit; got %s' % body[u'scopeType']) @@ -2006,13 +2002,7 @@ def doPrintAdmins(): userKey = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 elif myarg == u'role': - role = sys.argv[i+1] - if role[:4].lower() == u'uid:': - roleId = role[4:] - else: - roleId = roleid_from_role(role) - if not roleId: - systemErrorExit(5, '%s is not a valid role' % role) + roleId = getRoleId(sys.argv[i+1]) i += 2 elif myarg == u'todrive': todrive = True @@ -2071,6 +2061,16 @@ def roleid_from_role(role): buildRoleIdToNameToIdMap() return GM_Globals[GM_MAP_ROLE_NAME_TO_ID].get(role, None) +def getRoleId(role): + cg = UID_PATTERN.match(role) + if cg: + roleId = cg.group(1) + else: + roleId = roleid_from_role(role) + if not roleId: + systemErrorExit(4, '%s is not a valid role. Please ensure role name is exactly as shown in admin console.' % role) + return roleId + def buildUserIdToNameMap(): cd = buildGAPIObject(u'directory') result = callGAPIpages(cd.users(), u'list', u'users', @@ -2107,10 +2107,9 @@ def app2appID(dt, app): systemErrorExit(2, '%s is not a valid service for data transfer.' % app) def convertToUserID(user): - if user[:4].lower() == u'uid:': - return user[4:] - if user[:3].lower() == u'id:': - return user[3:] + cg = UID_PATTERN.match(user) + if cg: + return cg.group(1) cd = buildGAPIObject(u'directory') if user.find(u'@') == -1: user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) @@ -3975,15 +3974,17 @@ def showDriveFileACL(users): print u'' def getPermissionId(argstr): - permissionId = argstr.strip().lower() - if permissionId[:3] == u'id:': - return argstr.strip()[3:] + permissionId = argstr.strip() + cg = UID_PATTERN.match(permissionId) + if cg: + return cg.group(1) + permissionId = argstr.lower() if permissionId == u'anyone': return u'anyone' if permissionId == u'anyonewithlink': return u'anyoneWithLink' if permissionId.find(u'@') == -1: - permissionId = u'%s@%s' % (permissionId, GC_Values[GC_DOMAIN]) + permissionId = u'%s@%s' % (permissionId, GC_Values[GC_DOMAIN].lower()) # We have to use v2 here since v3 has no permissions.getIdForEmail equivalent # https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=4313 _, drive2 = buildDriveGAPIObject(_getValueFromOAuth(u'email')) @@ -4256,9 +4257,12 @@ def printDriveFileList(users): sortCSVTitles([u'Owner', u'id', u'title'], titles) writeCSVfile(csvRows, titles, u'%s %s Drive Files' % (sys.argv[1], sys.argv[2]), todrive) -def doDriveSearch(drive, query=None): - print u'Searching for files with query: "%s"...' % query - page_message = u' Got %%total_items%% files...\n' +def doDriveSearch(drive, query=None, quiet=False): + if not quiet: + print u'Searching for files with query: "%s"...' % query + page_message = u' Got %%total_items%% files...\n' + else: + page_message = None files = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, q=query, fields=u'nextPageToken,items(id)', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) @@ -4599,13 +4603,13 @@ def createDriveFile(users): def downloadDriveFile(users): i = 5 fileIdSelection = {u'fileIds': [], u'query': None} - revisionId = None + csvSheetTitle = revisionId = None exportFormatName = u'openoffice' exportFormatChoices = [exportFormatName] exportFormats = DOCUMENT_FORMATS_MAP[exportFormatName] targetFolder = GC_Values[GC_DRIVE_DIR] targetName = None - overwrite = showProgress = False + overwrite = showProgress = targetStdout = False safe_filename_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) while i < len(sys.argv): myarg = sys.argv[i].lower().replace(u'_', u'') @@ -4621,6 +4625,10 @@ def downloadDriveFile(users): elif myarg == u'revision': revisionId = getInteger(sys.argv[i+1], myarg, minVal=1) i += 2 + elif myarg == u'csvsheet': + csvSheetTitle = sys.argv[i+1] + csvSheetTitleLower = csvSheetTitle.lower() + i += 2 elif myarg == u'format': exportFormatChoices = sys.argv[i+1].replace(u',', u' ').lower().split() exportFormats = [] @@ -4637,6 +4645,7 @@ def downloadDriveFile(users): i += 2 elif myarg == u'targetname': targetName = sys.argv[i+1] + targetStdout = targetName == u'-' i += 2 elif myarg == u'overwrite': overwrite = True @@ -4650,12 +4659,20 @@ def downloadDriveFile(users): systemErrorExit(2, 'you need to specify either id, query or drivefilename in order to determine the file(s) to download') if fileIdSelection[u'query'] and fileIdSelection[u'fileIds']: systemErrorExit(2, 'you cannot specify multiple file identifiers. Choose one of id, drivefilename, query.') + if csvSheetTitle: + exportFormatName = u'csv' + exportFormatChoices = [exportFormatName] + exportFormats = DOCUMENT_FORMATS_MAP[exportFormatName] for user in users: user, drive = buildDriveGAPIObject(user) if not drive: continue + if csvSheetTitle: + sheet = buildGAPIServiceObject(u'sheets', user) + if not sheet: + continue if fileIdSelection[u'query']: - fileIdSelection[u'fileIds'] = doDriveSearch(drive, query=fileIdSelection[u'query']) + fileIdSelection[u'fileIds'] = doDriveSearch(drive, query=fileIdSelection[u'query'], quiet=targetStdout) else: fileId = fileIdSelection[u'fileIds'][0] if fileId[:8].lower() == u'https://' or fileId[:7].lower() == u'http://': @@ -4678,6 +4695,8 @@ def downloadDriveFile(users): validExtensions = GOOGLEDOC_VALID_EXTENSIONS_MAP.get(mimeType) if validExtensions: my_line = u'Downloading Google Doc: %s' + if csvSheetTitle: + my_line += u', Sheet: %s' % csvSheetTitle googleDoc = True else: if u'fileSize' in result: @@ -4686,43 +4705,67 @@ def downloadDriveFile(users): my_line = u'Downloading: %s of unknown size' googleDoc = False my_line += u' to %s' - fileDownloaded = fileDownloadFailed = False + csvSheetNotFound = fileDownloaded = fileDownloadFailed = False for exportFormat in exportFormats: extension = fileExtension or exportFormat[u'ext'] if googleDoc and (extension not in validExtensions): continue - if targetName: - safe_file_title = targetName + if targetStdout: + filename = u'stdout' else: - safe_file_title = u''.join(c for c in result[u'title'] if c in safe_filename_chars) - if len(safe_file_title) < 1: - safe_file_title = fileId - filename = os.path.join(targetFolder, safe_file_title) - y = 0 - while True: - if filename.lower()[-len(extension):] != extension.lower(): - filename += extension - if overwrite or not os.path.isfile(filename): - break - y += 1 - filename = os.path.join(targetFolder, u'({0})-{1}'.format(y, safe_file_title)) - print utils.convertUTF8(my_line % (result[u'title'], filename)) + if targetName: + safe_file_title = targetName + else: + safe_file_title = u''.join(c for c in result[u'title'] if c in safe_filename_chars) + if len(safe_file_title) < 1: + safe_file_title = fileId + filename = os.path.join(targetFolder, safe_file_title) + y = 0 + while True: + if filename.lower()[-len(extension):] != extension.lower(): + filename += extension + if overwrite or not os.path.isfile(filename): + break + y += 1 + filename = os.path.join(targetFolder, u'({0})-{1}'.format(y, safe_file_title)) + print utils.convertUTF8(my_line % (result[u'title'], filename)) + spreadsheetUrl = None if googleDoc: - request = drive.files().export_media(fileId=fileId, mimeType=exportFormat[u'mime']) - if revisionId: - request.uri = u'{0}&revision={1}'.format(request.uri, revisionId) + if csvSheetTitle is None or mimeType != MIMETYPE_GA_SPREADSHEET: + request = drive.files().export_media(fileId=fileId, mimeType=exportFormat[u'mime']) + if revisionId: + request.uri = u'{0}&revision={1}'.format(request.uri, revisionId) + else: + spreadsheet = callGAPI(sheet.spreadsheets(), u'get', + spreadsheetId=fileId, fields=u'spreadsheetUrl,sheets(properties(sheetId,title))') + for sheet in spreadsheet[u'sheets']: + if sheet[u'properties'][u'title'].lower() == csvSheetTitleLower: + spreadsheetUrl = u'{0}?format=csv&id={1}&gid={2}'.format(re.sub(u'/edit$', u'/export', spreadsheet[u'spreadsheetUrl']), + fileId, sheet[u'properties'][u'sheetId']) + break + else: + stderrErrorMsg(u'Google Doc: %s, Sheet: %s, does not exist' % (result[u'title'], csvSheetTitle)) + csvSheetNotFound = True + continue else: - request = drive.files().get_media(fileId=fileId) + request = drive.files().get_media(fileId=fileId, revisionId=revisionId) fh = None try: - fh = open(filename, u'wb') - downloader = googleapiclient.http.MediaIoBaseDownload(fh, request) - done = False - while not done: - status, done = downloader.next_chunk() - if showProgress: - print u'Downloaded: {0:>7.2%}'.format(status.progress()) - closeFile(fh) + fh = open(filename, u'wb') if not targetStdout else sys.stdout + if not spreadsheetUrl: + downloader = googleapiclient.http.MediaIoBaseDownload(fh, request) + done = False + while not done: + status, done = downloader.next_chunk() + if showProgress: + print u'Downloaded: {0:>7.2%}'.format(status.progress()) + else: + _, content = drive._http.request(uri=spreadsheetUrl, method='GET') + fh.write(content) + if targetStdout and content[-1] != u'\n': + fh.write(u'\n') + if not targetStdout: + closeFile(fh) fileDownloaded = True break except (IOError, httplib2.HttpLib2Error) as e: @@ -4732,10 +4775,10 @@ def downloadDriveFile(users): break except googleapiclient.http.HttpError: sys.stderr.write(u'Format ({0}) not available\n'.format(extension[1:])) - if fh: + if fh and not targetStdout: closeFile(fh) os.remove(filename) - if not fileDownloaded and not fileDownloadFailed: + if not fileDownloaded and not fileDownloadFailed and not csvSheetNotFound: stderrErrorMsg(u'Format ({0}) not available'.format(u','.join(exportFormatChoices))) GM_Globals[GM_SYSEXITRC] = 51 @@ -8052,8 +8095,9 @@ def doGetVaultHoldInfo(): def convertExportNameToID(v, nameOrID, matterId): nameOrID = nameOrID.lower() - if nameOrID[:4] == u'uid:': - return nameOrID[4:] + cg = UID_PATTERN.match(nameOrID) + if cg: + return cg.group(1) exports = callGAPIpages(v.matters().exports(), u'list', u'exports', matterId=matterId, fields=u'exports(id,name),nextPageToken') for export in exports: if export[u'name'].lower() == nameOrID: @@ -8062,8 +8106,9 @@ def convertExportNameToID(v, nameOrID, matterId): def convertHoldNameToID(v, nameOrID, matterId): nameOrID = nameOrID.lower() - if nameOrID[:4] == u'uid:': - return nameOrID[4:] + cg = UID_PATTERN.match(nameOrID) + if cg: + return cg.group(1) holds = callGAPIpages(v.matters().holds(), u'list', u'holds', matterId=matterId, fields=u'holds(holdId,name),nextPageToken') for hold in holds: if hold[u'name'].lower() == nameOrID: @@ -8072,8 +8117,9 @@ def convertHoldNameToID(v, nameOrID, matterId): def convertMatterNameToID(v, nameOrID): nameOrID = nameOrID.lower() - if nameOrID[:4] == u'uid:': - return nameOrID[4:] + cg = UID_PATTERN.match(nameOrID) + if cg: + return cg.group(1) matters = callGAPIpages(v.matters(), u'list', u'matters', view=u'BASIC', fields=u'matters(matterId,name),nextPageToken') for matter in matters: if matter[u'name'].lower() == nameOrID: @@ -8434,12 +8480,13 @@ def _makeBuildingIdNameMap(cd): GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building[u'buildingName']] = building[u'buildingId'] def _getBuildingByNameOrId(cd, which_building, minLen=1): - if not which_building or which_building.lower() == u'id:': + if not which_building or (minLen == 0 and which_building in [u'id:', u'uid:']): if minLen == 0: return u'' systemErrorExit(3, u'Building id/name is empty') - if which_building[:3].lower() == u'id:': - return which_building[3:] + cg = UID_PATTERN.match(which_building) + if cg: + return cg.group(1) if GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] is None: _makeBuildingIdNameMap(cd) # Exact name match, return ID @@ -12336,10 +12383,9 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No full_users = list() if entity != u'cros' and not got_uids: for user in users: - if user[:4] == u'uid:': - full_users.append(user[4:]) - elif user[:3] == u'id:': - full_users.append(user[3:]) + cg = UID_PATTERN.match(user) + if cg: + full_users.append(cg.group(1)) elif user != u'*' and user != GC_Values[GC_CUSTOMER_ID] and user.find(u'@') == -1: full_users.append(u'%s@%s' % (user, GC_Values[GC_DOMAIN])) else: diff --git a/src/project-apis.txt b/src/project-apis.txt index 58d44153..3c1ff8df 100644 --- a/src/project-apis.txt +++ b/src/project-apis.txt @@ -10,6 +10,7 @@ groupssettings.googleapis.com licensing.googleapis.com plus.googleapis.com reseller.googleapis.com +sheets.googleapis.com siteverification.googleapis.com storage-api.googleapis.com vault.googleapis.com diff --git a/src/var.py b/src/var.py index c2746916..5fb0c92e 100644 --- a/src/var.py +++ b/src/var.py @@ -111,6 +111,7 @@ API_VER_MAPPING = { u'pubsub': u'v1', u'reports': u'reports_v1', u'reseller': u'v1', + u'sheets': u'v4', u'siteVerification': u'v1', u'storage': u'v1', u'urlshortener': u'v1', @@ -128,6 +129,7 @@ API_SCOPE_MAPPING = { u'https://www.googleapis.com/auth/gmail.settings.basic', u'https://www.googleapis.com/auth/gmail.settings.sharing',], u'plus': [u'https://www.googleapis.com/auth/plus.me',], + u'sheets': [u'https://www.googleapis.com/auth/spreadsheets',], } ADDRESS_FIELDS_PRINT_ORDER = [