fix Vault download filenames

This commit is contained in:
Jay Lee
2023-02-06 21:43:33 +00:00
parent 62d738f5c2
commit a94ef78066
2 changed files with 14 additions and 19 deletions

View File

@@ -5,6 +5,7 @@ import sys
import time
import googleapiclient
from pathvalidate import sanitize_filepath
import gam
from gam.gapi import errors as gapi_errors
@@ -152,11 +153,12 @@ def get_cloud_storage_object(s,
expectedMd5=None):
if not local_file:
local_file = object_
local_file = sanitize_filepath(local_file, platform='auto')
if os.path.exists(local_file):
sys.stdout.write(' File already exists. ')
sys.stdout.write(f'File {local_file} already exists.')
sys.stdout.flush()
if expectedMd5:
sys.stdout.write(f'Verifying {expectedMd5} hash...')
sys.stdout.write(f' verifying {expectedMd5} hash...')
sys.stdout.flush()
if utils.md5_matches_file(local_file, expectedMd5, False):
print('VERIFIED')
@@ -164,7 +166,7 @@ def get_cloud_storage_object(s,
print('not verified. Downloading again and over-writing...')
else:
return # nothing to verify, just assume we're good.
print(f'saving to {local_file}')
print(f'Saving to {local_file}')
request = s.objects().get_media(bucket=bucket, object=object_)
file_path = os.path.dirname(local_file)
if not os.path.exists(file_path):

View File

@@ -867,24 +867,17 @@ def downloadExport():
for s_file in export['cloudStorageSink']['files']:
bucket = s_file['bucketName']
s_object = s_file['objectName']
filename = os.path.join(targetFolder, s_object.replace('/', '-'))
print(f'saving to {filename}')
request = s.objects().get_media(bucket=bucket, object=s_object)
f = fileutils.open_file(filename, 'wb')
downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
done = False
while not done:
status, done = downloader.next_chunk()
sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
sys.stdout.flush()
sys.stdout.write('\n Download complete. Flushing to disk...\n')
fileutils.close_file(f, True)
if verifyFiles:
expected_hash = s_file['md5Hash']
sys.stdout.write(f' Verifying file hash is {expected_hash}...')
sys.stdout.flush()
utils.md5_matches_file(filename, expected_hash, True)
print('VERIFIED')
else:
expected_hash = None
local_file = s_object.replace('/', '-').replace(':', '-')
filename = os.path.join(targetFolder, local_file)
gapi_storage.get_cloud_storage_object(s,
bucket,
s_object,
local_file=filename,
expectedMd5=expected_hash)
if extractFiles and re.search(r'\.zip$', filename):
gam.extract_nested_zip(filename, targetFolder)