diff --git a/src/GamCommands.txt b/src/GamCommands.txt index 860081c1..87d621e3 100644 --- a/src/GamCommands.txt +++ b/src/GamCommands.txt @@ -1106,7 +1106,8 @@ gam show filetree [anyowner] (orderby [ gam create|add drivefile [drivefilename ] * [csv] [todrive] gam update drivefile (id )|(query ] * -gam get drivefile (id )|(drivefilename )|(query ) [format ] [targetfolder ] [revision ] +gam get drivefile (id )|(drivefilename )|(query ) [revision ] [format ] + targetfolder ] [targetname ] [overwrite] [showprogress] gam delete|del drivefile ||(query:) [purge|untrash] gam transfer drive [keepuser] gam delete|del emptydrivefolders diff --git a/src/gam.py b/src/gam.py index e884e1cd..31f8b2e1 100755 --- a/src/gam.py +++ b/src/gam.py @@ -4577,6 +4577,8 @@ def downloadDriveFile(users): exportFormatChoices = [exportFormatName] exportFormats = DOCUMENT_FORMATS_MAP[exportFormatName] targetFolder = GC_Values[GC_DRIVE_DIR] + targetName = None + overwrite = showProgress = False safe_filename_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) while i < len(sys.argv): myarg = sys.argv[i].lower().replace(u'_', u'') @@ -4606,6 +4608,15 @@ def downloadDriveFile(users): if not os.path.isdir(targetFolder): os.makedirs(targetFolder) i += 2 + elif myarg == u'targetname': + targetName = sys.argv[i+1] + i += 2 + elif myarg == u'overwrite': + overwrite = True + i += 1 + elif myarg == u'showprogress': + showProgress = True + i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam get drivefile"' % sys.argv[i]) if not fileIdSelection[u'query'] and not fileIdSelection[u'fileIds']: @@ -4626,58 +4637,80 @@ def downloadDriveFile(users): print u'No files to download for %s' % user i = 0 for fileId in fileIdSelection[u'fileIds']: - extension = None - result = callGAPI(drive.files(), u'get', fileId=fileId, - fields=u'fileSize,title,mimeType,downloadUrl,exportLinks', - supportsTeamDrives=True) - if result[u'mimeType'] == MIMETYPE_GA_FOLDER: + fileExtension = None + result = callGAPI(drive.files(), u'get', + fileId=fileId, fields=u'fileExtension,fileSize,mimeType,title', supportsTeamDrives=True) + fileExtension = result.get(u'fileExtension') + mimeType = result[u'mimeType'] + if mimeType == MIMETYPE_GA_FOLDER: print utils.convertUTF8(u'Skipping download of folder %s' % result[u'title']) continue - try: - result[u'fileSize'] = int(result[u'fileSize']) - if result[u'fileSize'] < 1024: - filesize = u'1kb' - elif result[u'fileSize'] < (1024 * 1024): - filesize = u'%skb' % (result[u'fileSize'] / 1024) - elif result[u'fileSize'] < (1024 * 1024 * 1024): - filesize = u'%smb' % (result[u'fileSize'] / 1024 / 1024) - else: - filesize = u'%sgb' % (result[u'fileSize'] / 1024 / 1024 / 1024) - my_line = u'Downloading: %%s of %s bytes' % filesize - except KeyError: - my_line = u'Downloading Google Doc: %s' - if u'downloadUrl' in result: - download_url = result[u'downloadUrl'] - elif u'exportLinks' in result: - for exportFormat in exportFormats: - if exportFormat[u'mime'] in result[u'exportLinks']: - download_url = result[u'exportLinks'][exportFormat[u'mime']] - extension = exportFormat[u'ext'] - break - else: - print utils.convertUTF8(u'Skipping download of file {0}, Format {1} not available'.format(result[u'title'], u','.join(exportFormatChoices))) - continue - else: - print utils.convertUTF8(u'Skipping download of file {0}, Format not downloadable') + if mimeType in NON_DOWNLOADABLE_MIMETYPES: + print utils.convertUTF8(u'Format of file %s not downloadable' % result[u'title']) continue - file_title = result[u'title'] - safe_file_title = u''.join(c for c in file_title if c in safe_filename_chars) - if len(safe_file_title) < 1: - safe_file_title = fileId - filename = os.path.join(targetFolder, safe_file_title) - y = 0 - while True: - if extension and filename.lower()[-len(extension):] != extension: - filename += extension - if not os.path.isfile(filename): + validExtensions = GOOGLEDOC_VALID_EXTENSIONS_MAP.get(mimeType) + if validExtensions: + my_line = u'Downloading Google Doc: %s' + googleDoc = True + else: + if u'fileSize' in result: + my_line = u'Downloading: %%s of %s bytes' % utils.formatFileSize(int(result[u'fileSize'])) + else: + my_line = u'Downloading: %s of unknown size' + googleDoc = False + my_line += u' to %s' + fileDownloaded = fileDownloadFailed = False + for exportFormat in exportFormats: + extension = fileExtension or exportFormat[u'ext'] + if googleDoc and (extension not in validExtensions): + continue + if targetName: + safe_file_title = targetName + else: + safe_file_title = u''.join(c for c in result[u'title'] if c in safe_filename_chars) + if len(safe_file_title) < 1: + safe_file_title = fileId + filename = os.path.join(targetFolder, safe_file_title) + y = 0 + while True: + if filename.lower()[-len(extension):] != extension.lower(): + filename += extension + if overwrite or not os.path.isfile(filename): + break + y += 1 + filename = os.path.join(targetFolder, u'({0})-{1}'.format(y, safe_file_title)) + print utils.convertUTF8(my_line % (result[u'title'], filename)) + if googleDoc: + request = drive.files().export_media(fileId=fileId, mimeType=exportFormat[u'mime']) + if revisionId: + request.uri = u'{0}&revision={1}'.format(request.uri, revisionId) + else: + request = drive.files().get_media(fileId=fileId) + fh = None + try: + fh = open(filename, u'wb') + downloader = googleapiclient.http.MediaIoBaseDownload(fh, request) + done = False + while not done: + status, done = downloader.next_chunk() + if showProgress: + print u'Downloaded: {0:>7.2%}'.format(status.progress()) + closeFile(fh) + fileDownloaded = True break - y += 1 - filename = os.path.join(targetFolder, u'({0})-{1}'.format(y, safe_file_title)) - print utils.convertUTF8(my_line % filename) - if revisionId: - download_url = u'{0}&revision={1}'.format(download_url, revisionId) - _, content = drive._http.request(download_url) - writeFile(filename, content, continueOnError=True) + except (IOError, httplib2.HttpLib2Error) as e: + stderrErrorMsg(str(e)) + GM_Globals[GM_SYSEXITRC] = 6 + fileDownloadFailed = True + break + except googleapiclient.http.HttpError: + sys.stderr.write(u'Format ({0}) not available\n'.format(extension[1:])) + if fh: + closeFile(fh) + os.remove(filename) + if not fileDownloaded and not fileDownloadFailed: + stderrErrorMsg(u'Format ({0}) not available'.format(u','.join(exportFormatChoices))) + GM_Globals[GM_SYSEXITRC] = 51 def showDriveFileInfo(users): fieldsList = [] diff --git a/src/utils.py b/src/utils.py index 3b54b2b6..84ce6910 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,7 +3,7 @@ import re import sys from htmlentitydefs import name2codepoint from HTMLParser import HTMLParser -from var import GM_Globals, GM_WINDOWS, GM_SYS_ENCODING +from var import GM_Globals, GM_WINDOWS, GM_SYS_ENCODING, ONE_KILO_BYTES, ONE_MEGA_BYTES, ONE_GIGA_BYTES def convertUTF8(data): if isinstance(data, str): @@ -73,6 +73,17 @@ def dehtml(text): def indentMultiLineText(message, n=0): return message.replace(u'\n', u'\n{0}'.format(u' '*n)).rstrip() +def formatFileSize(fileSize): + if fileSize == 0: + return u'0kb' + if fileSize < ONE_KILO_BYTES: + return u'1kb' + if fileSize < ONE_MEGA_BYTES: + return u'{0}kb'.format(fileSize/ONE_KILO_BYTES) + if fileSize < ONE_GIGA_BYTES: + return u'{0}mb'.format(fileSize/ONE_MEGA_BYTES) + return u'{0}gb'.format(fileSize/ONE_GIGA_BYTES) + def formatMilliSeconds(millis): seconds, millis = divmod(millis, 1000) minutes, seconds = divmod(seconds, 60) diff --git a/src/var.py b/src/var.py index 724dc89d..496ed0d1 100644 --- a/src/var.py +++ b/src/var.py @@ -328,34 +328,66 @@ DFA_OCR = u'ocr' DFA_OCRLANGUAGE = u'ocrLanguage' DFA_PARENTQUERY = u'parentQuery' +NON_DOWNLOADABLE_MIMETYPES = [MIMETYPE_GA_FORM, MIMETYPE_GA_FUSIONTABLE, MIMETYPE_GA_MAP] + +GOOGLEDOC_VALID_EXTENSIONS_MAP = { + MIMETYPE_GA_DRAWING: [u'.jpeg', u'.jpg', u'.pdf', u'.png', u'.svg'], + MIMETYPE_GA_DOCUMENT: [u'.docx', u'.html', u'.odt', u'.pdf', u'.rtf', u'.txt', u'.zip'], + MIMETYPE_GA_PRESENTATION: [u'.pdf', u'.pptx', u'.odp', u'.txt'], + MIMETYPE_GA_SPREADSHEET: [u'.csv', u'.ods', u'.pdf', u'.xlsx', u'.zip'], + } + +MICROSOFT_FORMATS_LIST = [{u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.document', u'ext': u'.docx'}, + {u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.template', u'ext': u'.dotx'}, + {u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.presentation', u'ext': u'.pptx'}, + {u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.template', u'ext': u'.potx'}, + {u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', u'ext': u'.xlsx'}, + {u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.template', u'ext': u'.xltx'}, + {u'mime': u'application/msword', u'ext': u'.doc'}, + {u'mime': u'application/msword', u'ext': u'.dot'}, + {u'mime': u'application/vnd.ms-powerpoint', u'ext': u'.ppt'}, + {u'mime': u'application/vnd.ms-powerpoint', u'ext': u'.pot'}, + {u'mime': u'application/vnd.ms-excel', u'ext': u'.xls'}, + {u'mime': u'application/vnd.ms-excel', u'ext': u'.xlt'}] + DOCUMENT_FORMATS_MAP = { u'csv': [{u'mime': u'text/csv', u'ext': u'.csv'}], + u'doc': [{u'mime': u'application/msword', u'ext': u'.doc'}], + u'dot': [{u'mime': u'application/msword', u'ext': u'.dot'}], + u'docx': [{u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.document', u'ext': u'.docx'}], + u'dotx': [{u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.template', u'ext': u'.dotx'}], + u'epub': [{u'mime': u'application/epub+zip', u'ext': u'.epub'}], u'html': [{u'mime': u'text/html', u'ext': u'.html'}], - u'txt': [{u'mime': u'text/plain', u'ext': u'.txt'}], - u'tsv': [{u'mime': u'text/tsv', u'ext': u'.tsv'}], u'jpeg': [{u'mime': u'image/jpeg', u'ext': u'.jpeg'}], u'jpg': [{u'mime': u'image/jpeg', u'ext': u'.jpg'}], - u'png': [{u'mime': u'image/png', u'ext': u'.png'}], - u'svg': [{u'mime': u'image/svg+xml', u'ext': u'.svg'}], - u'pdf': [{u'mime': u'application/pdf', u'ext': u'.pdf'}], - u'rtf': [{u'mime': u'application/rtf', u'ext': u'.rtf'}], - u'zip': [{u'mime': u'application/zip', u'ext': u'.zip'}], - u'pptx': [{u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.presentation', u'ext': u'.pptx'}], - u'xlsx': [{u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', u'ext': u'.xlsx'}], - u'docx': [{u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.document', u'ext': u'.docx'}], - u'ms': [{u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.presentation', u'ext': u'.pptx'}, - {u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', u'ext': u'.xlsx'}, - {u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.document', u'ext': u'.docx'}], - u'microsoft': [{u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.presentation', u'ext': u'.pptx'}, - {u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', u'ext': u'.xlsx'}, - {u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.document', u'ext': u'.docx'}], - u'micro$oft': [{u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.presentation', u'ext': u'.pptx'}, - {u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', u'ext': u'.xlsx'}, - {u'mime': u'application/vnd.openxmlformats-officedocument.wordprocessingml.document', u'ext': u'.docx'}], + u'mht': [{u'mime': u'message/rfc822', u'ext': u'mht'}], + u'odp': [{u'mime': u'application/vnd.oasis.opendocument.presentation', u'ext': u'.odp'}], + u'ods': [{u'mime': u'application/x-vnd.oasis.opendocument.spreadsheet', u'ext': u'.ods'}, + {u'mime': u'application/vnd.oasis.opendocument.spreadsheet', u'ext': u'.ods'}], u'odt': [{u'mime': u'application/vnd.oasis.opendocument.text', u'ext': u'.odt'}], - u'ods': [{u'mime': u'application/x-vnd.oasis.opendocument.spreadsheet', u'ext': u'.ods'}], - u'openoffice': [{u'mime': u'application/vnd.oasis.opendocument.text', u'ext': u'.odt'}, - {u'mime': u'application/x-vnd.oasis.opendocument.spreadsheet', u'ext': u'.ods'}], + u'pdf': [{u'mime': u'application/pdf', u'ext': u'.pdf'}], + u'png': [{u'mime': u'image/png', u'ext': u'.png'}], + u'ppt': [{u'mime': u'application/vnd.ms-powerpoint', u'ext': u'.ppt'}], + u'pot': [{u'mime': u'application/vnd.ms-powerpoint', u'ext': u'.pot'}], + u'potx': [{u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.template', u'ext': u'.potx'}], + u'pptx': [{u'mime': u'application/vnd.openxmlformats-officedocument.presentationml.presentation', u'ext': u'.pptx'}], + u'rtf': [{u'mime': u'application/rtf', u'ext': u'.rtf'}], + u'svg': [{u'mime': u'image/svg+xml', u'ext': u'.svg'}], + u'tsv': [{u'mime': u'text/tab-separated-values', u'ext': u'.tsv'}, + {u'mime': u'text/tsv', u'ext': u'.tsv'}], + u'txt': [{u'mime': u'text/plain', u'ext': u'.txt'}], + u'xls': [{u'mime': u'application/vnd.ms-excel', u'ext': u'.xls'}], + u'xlt': [{u'mime': u'application/vnd.ms-excel', u'ext': u'.xlt'}], + u'xlsx': [{u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', u'ext': u'.xlsx'}], + u'xltx': [{u'mime': u'application/vnd.openxmlformats-officedocument.spreadsheetml.template', u'ext': u'.xltx'}], + u'zip': [{u'mime': u'application/zip', u'ext': u'.zip'}], + u'ms': MICROSOFT_FORMATS_LIST, + u'microsoft': MICROSOFT_FORMATS_LIST, + u'micro$oft': MICROSOFT_FORMATS_LIST, + u'openoffice': [{u'mime': u'application/vnd.oasis.opendocument.presentation', u'ext': u'.odp'}, + {u'mime': u'application/x-vnd.oasis.opendocument.spreadsheet', u'ext': u'.ods'}, + {u'mime': u'application/vnd.oasis.opendocument.spreadsheet', u'ext': u'.ods'}, + {u'mime': u'application/vnd.oasis.opendocument.text', u'ext': u'.odt'}], } EMAILSETTINGS_OLD_NEW_OLD_FORWARD_ACTION_MAP = {