mirror of
https://github.com/GAM-team/GAM.git
synced 2025-07-09 06:03:35 +00:00
Improve batch performance
* Use a single GAM / Python process for all threads (needs testing, will sys.exit in a function cause issues?) - Huge reduction in useless time spent starting Python per-thread. * Bump default from 5 threads to 25. * Introduce default_to_batch for some user commands where it makes sense. - most show/print commands will default to batch off. - most do / update commands will default to on.
This commit is contained in:
27
src/gam.py
27
src/gam.py
@ -44,7 +44,6 @@ import random
|
|||||||
import re
|
import re
|
||||||
import socket
|
import socket
|
||||||
import StringIO
|
import StringIO
|
||||||
import subprocess
|
|
||||||
|
|
||||||
import googleapiclient
|
import googleapiclient
|
||||||
import googleapiclient.discovery
|
import googleapiclient.discovery
|
||||||
@ -240,7 +239,7 @@ GC_Defaults = {
|
|||||||
GC_NO_CACHE: FALSE,
|
GC_NO_CACHE: FALSE,
|
||||||
GC_NO_UPDATE_CHECK: FALSE,
|
GC_NO_UPDATE_CHECK: FALSE,
|
||||||
GC_NO_VERIFY_SSL: FALSE,
|
GC_NO_VERIFY_SSL: FALSE,
|
||||||
GC_NUM_THREADS: 5,
|
GC_NUM_THREADS: 25,
|
||||||
GC_OAUTH2_TXT: FN_OAUTH2_TXT,
|
GC_OAUTH2_TXT: FN_OAUTH2_TXT,
|
||||||
GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON,
|
GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON,
|
||||||
GC_SECTION: u'',
|
GC_SECTION: u'',
|
||||||
@ -10163,7 +10162,7 @@ gam create project
|
|||||||
def batch_worker():
|
def batch_worker():
|
||||||
while True:
|
while True:
|
||||||
item = GM_Globals[GM_BATCH_QUEUE].get()
|
item = GM_Globals[GM_BATCH_QUEUE].get()
|
||||||
subprocess.call(item, stderr=subprocess.STDOUT)
|
ProcessGAMCommand(item)
|
||||||
GM_Globals[GM_BATCH_QUEUE].task_done()
|
GM_Globals[GM_BATCH_QUEUE].task_done()
|
||||||
|
|
||||||
def run_batch(items):
|
def run_batch(items):
|
||||||
@ -10171,9 +10170,7 @@ def run_batch(items):
|
|||||||
import threading
|
import threading
|
||||||
total_items = len(items)
|
total_items = len(items)
|
||||||
current_item = 0
|
current_item = 0
|
||||||
python_cmd = [sys.executable.lower(),]
|
gam_cmd = [u'gam']
|
||||||
if not getattr(sys, u'frozen', False): # we're not frozen
|
|
||||||
python_cmd.append(os.path.realpath(sys.argv[0]))
|
|
||||||
num_worker_threads = min(total_items, GC_Values[GC_NUM_THREADS])
|
num_worker_threads = min(total_items, GC_Values[GC_NUM_THREADS])
|
||||||
GM_Globals[GM_BATCH_QUEUE] = Queue.Queue(maxsize=num_worker_threads) # GM_Globals[GM_BATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
|
GM_Globals[GM_BATCH_QUEUE] = Queue.Queue(maxsize=num_worker_threads) # GM_Globals[GM_BATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
|
||||||
sys.stderr.write(u'starting %s worker threads...\n' % num_worker_threads)
|
sys.stderr.write(u'starting %s worker threads...\n' % num_worker_threads)
|
||||||
@ -10190,7 +10187,7 @@ def run_batch(items):
|
|||||||
GM_Globals[GM_BATCH_QUEUE].join()
|
GM_Globals[GM_BATCH_QUEUE].join()
|
||||||
sys.stderr.write(u'done with commit-batch\n')
|
sys.stderr.write(u'done with commit-batch\n')
|
||||||
continue
|
continue
|
||||||
GM_Globals[GM_BATCH_QUEUE].put(python_cmd+item)
|
GM_Globals[GM_BATCH_QUEUE].put(gam_cmd+item)
|
||||||
GM_Globals[GM_BATCH_QUEUE].join()
|
GM_Globals[GM_BATCH_QUEUE].join()
|
||||||
#
|
#
|
||||||
# Process command line arguments, find substitutions
|
# Process command line arguments, find substitutions
|
||||||
@ -10254,6 +10251,16 @@ def processSubFields(GAM_argv, row, subFields):
|
|||||||
argv[GAM_argvI] = argv[GAM_argvI].encode(GM_Globals[GM_SYS_ENCODING])
|
argv[GAM_argvI] = argv[GAM_argvI].encode(GM_Globals[GM_SYS_ENCODING])
|
||||||
return argv
|
return argv
|
||||||
|
|
||||||
|
def runCmdForUsers(cmd, users, default_to_batch=False, **kwargs):
|
||||||
|
if default_to_batch and len(users) > 1:
|
||||||
|
items = []
|
||||||
|
for user in users:
|
||||||
|
items.append([u'user', user] + sys.argv[3:])
|
||||||
|
run_batch(items)
|
||||||
|
sys.exit(0)
|
||||||
|
else:
|
||||||
|
cmd(users, **kwargs)
|
||||||
|
|
||||||
# Process GAM command
|
# Process GAM command
|
||||||
def ProcessGAMCommand(args):
|
def ProcessGAMCommand(args):
|
||||||
if args != sys.argv:
|
if args != sys.argv:
|
||||||
@ -10755,7 +10762,8 @@ def ProcessGAMCommand(args):
|
|||||||
elif delWhat == u'label':
|
elif delWhat == u'label':
|
||||||
doDeleteLabel(users)
|
doDeleteLabel(users)
|
||||||
elif delWhat in [u'message', u'messages']:
|
elif delWhat in [u'message', u'messages']:
|
||||||
doProcessMessages(users, u'delete')
|
#doProcessMessages(users, u'delete')
|
||||||
|
runCmdForUsers(doProcessMessages, users, default_to_batch=True, function=u'delete')
|
||||||
elif delWhat == u'photo':
|
elif delWhat == u'photo':
|
||||||
deletePhoto(users)
|
deletePhoto(users)
|
||||||
elif delWhat in [u'license', u'licence']:
|
elif delWhat in [u'license', u'licence']:
|
||||||
@ -10876,7 +10884,8 @@ def ProcessGAMCommand(args):
|
|||||||
elif command == u'profile':
|
elif command == u'profile':
|
||||||
doProfile(users)
|
doProfile(users)
|
||||||
elif command == u'imap':
|
elif command == u'imap':
|
||||||
doImap(users)
|
#doImap(users)
|
||||||
|
runCmdForUsers(doImap, users, default_to_batch=True)
|
||||||
elif command in [u'pop', u'pop3']:
|
elif command in [u'pop', u'pop3']:
|
||||||
doPop(users)
|
doPop(users)
|
||||||
elif command == u'sendas':
|
elif command == u'sendas':
|
||||||
|
Reference in New Issue
Block a user