deepblue/GoogleDriveManagement
1
0
Fork 0
mirror of https://github.com/GAM-team/GAM.git synced 2025-07-09 06:03:35 +00:00
Code Issues Actions 19 Packages Projects Releases Wiki Activity

Improve batch performance

Browse Source 
* Use a single GAM / Python process for all threads (needs testing, will sys.exit in a function cause issues?)
    - Huge reduction in useless time spent starting Python per-thread.
  * Bump default from 5 threads to 25.
  * Introduce default_to_batch for some user commands where it makes sense.
    - most show/print commands will default to batch off.
    - most do / update commands will default to on.
This commit is contained in:
Jay Lee
2016-11-18 10:19:19 -05:00
parent 658e7beb2b
commit 349f2801c5
1 changed files with 18 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/gam.py
View File

@ -44,7 +44,6 @@ import random
import re import re
import socket import socket
import StringIO import StringIO
import subprocess
import googleapiclient import googleapiclient
import googleapiclient.discovery import googleapiclient.discovery
@ -240,7 +239,7 @@ GC_Defaults = {
GC_NO_CACHE: FALSE, GC_NO_CACHE: FALSE,
GC_NO_UPDATE_CHECK: FALSE, GC_NO_UPDATE_CHECK: FALSE,
GC_NO_VERIFY_SSL: FALSE, GC_NO_VERIFY_SSL: FALSE,
GC_NUM_THREADS: 5, GC_NUM_THREADS: 25,
GC_OAUTH2_TXT: FN_OAUTH2_TXT, GC_OAUTH2_TXT: FN_OAUTH2_TXT,
GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON, GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON,
GC_SECTION: u'', GC_SECTION: u'',
@ -10163,7 +10162,7 @@ gam create project
def batch_worker(): def batch_worker():
while True: while True:
item = GM_Globals[GM_BATCH_QUEUE].get() item = GM_Globals[GM_BATCH_QUEUE].get()
subprocess.call(item, stderr=subprocess.STDOUT) ProcessGAMCommand(item)
GM_Globals[GM_BATCH_QUEUE].task_done() GM_Globals[GM_BATCH_QUEUE].task_done()
def run_batch(items): def run_batch(items):
@ -10171,9 +10170,7 @@ def run_batch(items):
import threading import threading
total_items = len(items) total_items = len(items)
current_item = 0 current_item = 0
python_cmd = [sys.executable.lower(),] gam_cmd = [u'gam']
if not getattr(sys, u'frozen', False): # we're not frozen
python_cmd.append(os.path.realpath(sys.argv[0]))
num_worker_threads = min(total_items, GC_Values[GC_NUM_THREADS]) num_worker_threads = min(total_items, GC_Values[GC_NUM_THREADS])
GM_Globals[GM_BATCH_QUEUE] = Queue.Queue(maxsize=num_worker_threads) # GM_Globals[GM_BATCH_QUEUE].put() gets blocked when trying to create more items than there are workers GM_Globals[GM_BATCH_QUEUE] = Queue.Queue(maxsize=num_worker_threads) # GM_Globals[GM_BATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
sys.stderr.write(u'starting %s worker threads...\n' % num_worker_threads) sys.stderr.write(u'starting %s worker threads...\n' % num_worker_threads)
@ -10190,7 +10187,7 @@ def run_batch(items):
GM_Globals[GM_BATCH_QUEUE].join() GM_Globals[GM_BATCH_QUEUE].join()
sys.stderr.write(u'done with commit-batch\n') sys.stderr.write(u'done with commit-batch\n')
continue continue
GM_Globals[GM_BATCH_QUEUE].put(python_cmd+item) GM_Globals[GM_BATCH_QUEUE].put(gam_cmd+item)
GM_Globals[GM_BATCH_QUEUE].join() GM_Globals[GM_BATCH_QUEUE].join()
# #
# Process command line arguments, find substitutions # Process command line arguments, find substitutions
@ -10254,6 +10251,16 @@ def processSubFields(GAM_argv, row, subFields):
argv[GAM_argvI] = argv[GAM_argvI].encode(GM_Globals[GM_SYS_ENCODING]) argv[GAM_argvI] = argv[GAM_argvI].encode(GM_Globals[GM_SYS_ENCODING])
return argv return argv
def runCmdForUsers(cmd, users, default_to_batch=False, **kwargs):
if default_to_batch and len(users) > 1:
items = []
for user in users:
items.append([u'user', user] + sys.argv[3:])
run_batch(items)
sys.exit(0)
else:
cmd(users, **kwargs)
# Process GAM command # Process GAM command
def ProcessGAMCommand(args): def ProcessGAMCommand(args):
if args != sys.argv: if args != sys.argv:
@ -10755,7 +10762,8 @@ def ProcessGAMCommand(args):
elif delWhat == u'label': elif delWhat == u'label':
doDeleteLabel(users) doDeleteLabel(users)
elif delWhat in [u'message', u'messages']: elif delWhat in [u'message', u'messages']:
doProcessMessages(users, u'delete') #doProcessMessages(users, u'delete')
runCmdForUsers(doProcessMessages, users, default_to_batch=True, function=u'delete')
elif delWhat == u'photo': elif delWhat == u'photo':
deletePhoto(users) deletePhoto(users)
elif delWhat in [u'license', u'licence']: elif delWhat in [u'license', u'licence']:
@ -10876,7 +10884,8 @@ def ProcessGAMCommand(args):
elif command == u'profile': elif command == u'profile':
doProfile(users) doProfile(users)
elif command == u'imap': elif command == u'imap':
doImap(users) #doImap(users)
runCmdForUsers(doImap, users, default_to_batch=True)
elif command in [u'pop', u'pop3']: elif command in [u'pop', u'pop3']:
doPop(users) doPop(users)
elif command == u'sendas': elif command == u'sendas':