mirror of
https://github.com/GAM-team/GAM.git
synced 2026-06-20 14:11:37 +00:00
0bd4eefeca
Automatic fixes produced by yapf formatting using `--style=google`, as well as common pre-commit checks such as trailing whitespace removal, double quote fixer, and newlines at the end of a file.
82 lines
2.7 KiB
Python
82 lines
2.7 KiB
Python
import base64
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
import googleapiclient
|
|
|
|
import gam
|
|
from gam.var import *
|
|
from gam import fileutils
|
|
from gam import gapi
|
|
from gam import utils
|
|
|
|
|
|
def build_gapi():
|
|
return gam.buildGAPIObject('storage')
|
|
|
|
|
|
def get_cloud_storage_object(s,
|
|
bucket,
|
|
object_,
|
|
local_file=None,
|
|
expectedMd5=None):
|
|
if not local_file:
|
|
local_file = object_
|
|
if os.path.exists(local_file):
|
|
sys.stdout.write(' File already exists. ')
|
|
sys.stdout.flush()
|
|
if expectedMd5:
|
|
sys.stdout.write(f'Verifying {expectedMd5} hash...')
|
|
sys.stdout.flush()
|
|
if utils.md5_matches_file(local_file, expectedMd5, False):
|
|
print('VERIFIED')
|
|
return
|
|
print('not verified. Downloading again and over-writing...')
|
|
else:
|
|
return # nothing to verify, just assume we're good.
|
|
print(f'saving to {local_file}')
|
|
request = s.objects().get_media(bucket=bucket, object=object_)
|
|
file_path = os.path.dirname(local_file)
|
|
if not os.path.exists(file_path):
|
|
os.makedirs(file_path)
|
|
f = fileutils.open_file(local_file, 'wb')
|
|
downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
|
|
done = False
|
|
while not done:
|
|
status, done = downloader.next_chunk()
|
|
sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
|
|
sys.stdout.flush()
|
|
sys.stdout.write('\n Download complete. Flushing to disk...\n')
|
|
fileutils.close_file(f, True)
|
|
if expectedMd5:
|
|
f = fileutils.open_file(local_file, 'rb')
|
|
sys.stdout.write(f' Verifying file hash is {expectedMd5}...')
|
|
sys.stdout.flush()
|
|
utils.md5_matches_file(local_file, expectedMd5, True)
|
|
print('VERIFIED')
|
|
fileutils.close_file(f)
|
|
|
|
|
|
def download_bucket():
|
|
bucket = sys.argv[3]
|
|
s = build_gapi()
|
|
page_message = gapi.got_total_items_msg('Files', '...')
|
|
fields = 'nextPageToken,items(name,id,md5Hash)'
|
|
objects = gapi.get_all_pages(s.objects(),
|
|
'list',
|
|
'items',
|
|
page_message=page_message,
|
|
bucket=bucket,
|
|
projection='noAcl',
|
|
fields=fields)
|
|
i = 1
|
|
for object_ in objects:
|
|
print(f'{i}/{len(objects)}')
|
|
expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
|
|
get_cloud_storage_object(s,
|
|
bucket,
|
|
object_['name'],
|
|
expectedMd5=expectedMd5)
|
|
i += 1
|