diff --git a/src/gam.py b/src/gam.py index 5f8f878d..68d7d917 100755 --- a/src/gam.py +++ b/src/gam.py @@ -76,6 +76,7 @@ import display import fileutils import gapi.calendar import gapi.errors +import gapi.storage import gapi.vault import gapi import transport @@ -7426,61 +7427,6 @@ def doDeleteTeamDrive(users): print(f'Deleting Team Drive {teamDriveId}') gapi.call(drive.drives(), 'delete', driveId=teamDriveId, soft_errors=True) -def _getCloudStorageObject(s, bucket, object_, local_file=None, expectedMd5=None): - if not local_file: - local_file = object_ - if os.path.exists(local_file): - sys.stdout.write(' File already exists. ') - sys.stdout.flush() - if expectedMd5: - sys.stdout.write(f'Verifying {expectedMd5} hash...') - sys.stdout.flush() - if utils.md5_matches_file(local_file, expectedMd5, False): - print('VERIFIED') - return - print('not verified. Downloading again and over-writing...') - else: - return # nothing to verify, just assume we're good. - print(f'saving to {local_file}') - request = s.objects().get_media(bucket=bucket, object=object_) - file_path = os.path.dirname(local_file) - if not os.path.exists(file_path): - os.makedirs(file_path) - f = fileutils.open_file(local_file, 'wb') - downloader = googleapiclient.http.MediaIoBaseDownload(f, request) - done = False - while not done: - status, done = downloader.next_chunk() - sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r') - sys.stdout.flush() - sys.stdout.write('\n Download complete. Flushing to disk...\n') - fileutils.close_file(f, True) - if expectedMd5: - f = fileutils.open_file(local_file, 'rb') - sys.stdout.write(f' Verifying file hash is {expectedMd5}...') - sys.stdout.flush() - utils.md5_matches_file(local_file, expectedMd5, True) - print('VERIFIED') - fileutils.close_file(f) - -def doDownloadCloudStorageBucket(): - bucket_url = sys.argv[3] - bucket_regex = r'(takeout-export-[a-f,0-9,-]*)' - bucket_match = re.search(bucket_regex, bucket_url) - if bucket_match: - bucket = bucket_match.group(1) - else: - controlflow.system_error_exit(5, 'Could not find a takeout-export-* bucket in that URL') - s = buildGAPIObject('storage') - page_message = gapi.got_total_items_msg('Files', '...') - objects = gapi.get_all_pages(s.objects(), 'list', 'items', page_message=page_message, bucket=bucket, projection='noAcl', fields='nextPageToken,items(name,id,md5Hash)') - i = 1 - for object_ in objects: - print(f'{i}/{len(objects)}') - expectedMd5 = base64.b64decode(object_['md5Hash']).hex() - _getCloudStorageObject(s, bucket, object_['name'], expectedMd5=expectedMd5) - i += 1 - def extract_nested_zip(zippedFile, toFolder, spacing=' '): """ Extract a zip file including any nested zip files Delete the zip file(s) after extraction @@ -12797,7 +12743,7 @@ def ProcessGAMCommand(args): if argument in ['export', 'vaultexport']: gapi.vault.downloadExport() elif argument in ['storagebucket']: - doDownloadCloudStorageBucket() + gapi.storage.download_bucket() else: controlflow.invalid_argument_exit(argument, "gam download") sys.exit(0) diff --git a/src/gapi/storage.py b/src/gapi/storage.py new file mode 100644 index 00000000..27219063 --- /dev/null +++ b/src/gapi/storage.py @@ -0,0 +1,73 @@ +import base64 +import os +import re +import sys + +import googleapiclient + +import __main__ +from var import * +import controlflow +import fileutils +import gapi +import utils + + +def build_gapi(): + return __main__.buildGAPIObject('storage') + + +def get_cloud_storage_object(s, bucket, object_, local_file=None, + expectedMd5=None): + if not local_file: + local_file = object_ + if os.path.exists(local_file): + sys.stdout.write(' File already exists. ') + sys.stdout.flush() + if expectedMd5: + sys.stdout.write(f'Verifying {expectedMd5} hash...') + sys.stdout.flush() + if utils.md5_matches_file(local_file, expectedMd5, False): + print('VERIFIED') + return + print('not verified. Downloading again and over-writing...') + else: + return # nothing to verify, just assume we're good. + print(f'saving to {local_file}') + request = s.objects().get_media(bucket=bucket, object=object_) + file_path = os.path.dirname(local_file) + if not os.path.exists(file_path): + os.makedirs(file_path) + f = fileutils.open_file(local_file, 'wb') + downloader = googleapiclient.http.MediaIoBaseDownload(f, request) + done = False + while not done: + status, done = downloader.next_chunk() + sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r') + sys.stdout.flush() + sys.stdout.write('\n Download complete. Flushing to disk...\n') + fileutils.close_file(f, True) + if expectedMd5: + f = fileutils.open_file(local_file, 'rb') + sys.stdout.write(f' Verifying file hash is {expectedMd5}...') + sys.stdout.flush() + utils.md5_matches_file(local_file, expectedMd5, True) + print('VERIFIED') + fileutils.close_file(f) + + +def download_bucket(): + bucket = sys.argv[3] + s = build_gapi() + page_message = gapi.got_total_items_msg('Files', '...') + fields = 'nextPageToken,items(name,id,md5Hash)' + objects = gapi.get_all_pages(s.objects(), 'list', 'items', + page_message=page_message, bucket=bucket, + projection='noAcl', fields=fields) + i = 1 + for object_ in objects: + print(f'{i}/{len(objects)}') + expectedMd5 = base64.b64decode(object_['md5Hash']).hex() + get_cloud_storage_object( + s, bucket, object_['name'], expectedMd5=expectedMd5) + i += 1 diff --git a/src/gapi/vault.py b/src/gapi/vault.py index 50a070cf..b8598d26 100644 --- a/src/gapi/vault.py +++ b/src/gapi/vault.py @@ -595,7 +595,7 @@ def downloadExport(): verifyFiles = True extractFiles = True v = buildGAPIObject() - s = __main__.buildGAPIObject('storage') + s = storage.build_gapi() matterId = getMatterItem(v, sys.argv[3]) exportId = convertExportNameToID(v, sys.argv[4], matterId) targetFolder = GC_Values[GC_DRIVE_DIR] @@ -698,7 +698,7 @@ def printExports(): ), 'list', 'matters', view='BASIC', fields=fields) for matter in matters_results: matterState = matter['state'] - matterIid = matter['id'] + matterId = matter['matterId'] if matterState != 'OPEN': print(f'ignoring matter {matterId} in state {matterState}') continue @@ -742,7 +742,7 @@ def printHolds(): ), 'list', 'matters', view='BASIC', fields=fields) for matter in matters_results: matterState = matter['state'] - matterId = matter['id'] + matterId = matter['matterId'] if matterState != 'OPEN': print(f'ignoring matter {matterId} in state {matterState}') continue