Fix Windows multiprocessing (#346)

* Fix Windows multiprocessing

* Clean up Windows initial setup
This commit is contained in:
Ross Scroggs 2016-11-19 10:23:33 -08:00 committed by Jay Lee
parent 25ace13a3d
commit dbd09daa33

View File

@ -10178,10 +10178,7 @@ gam create project
def run_batch(items): def run_batch(items):
from multiprocessing import Pool from multiprocessing import Pool
total_items = len(items) num_worker_threads = min(len(items), GC_Values[GC_NUM_THREADS])
current_item = 0
gam_cmd = [u'gam']
num_worker_threads = min(total_items, GC_Values[GC_NUM_THREADS])
pool = Pool(processes=num_worker_threads) pool = Pool(processes=num_worker_threads)
sys.stderr.write(u'Using %s processes...\n' % num_worker_threads) sys.stderr.write(u'Using %s processes...\n' % num_worker_threads)
for item in items: for item in items:
@ -10920,34 +10917,69 @@ def ProcessGAMCommand(args):
GM_Globals[GM_SYSEXITRC] = e.code GM_Globals[GM_SYSEXITRC] = e.code
return GM_Globals[GM_SYSEXITRC] return GM_Globals[GM_SYSEXITRC]
def win32_unicode_argv(): #
from ctypes import POINTER, byref, cdll, c_int, windll # From: https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Multiprocessing
from ctypes.wintypes import LPCWSTR, LPWSTR #
if sys.platform.startswith('win'):
from multiprocessing import freeze_support
try:
import multiprocessing.popen_spawn_win32 as forking
except ImportError:
import multiprocessing.forking as forking
GetCommandLineW = cdll.kernel32.GetCommandLineW # First define a modified version of Popen.
GetCommandLineW.argtypes = [] class _Popen(forking.Popen):
GetCommandLineW.restype = LPCWSTR def __init__(self, *args, **kw):
if hasattr(sys, 'frozen'):
# We have to set original _MEIPASS2 value from sys._MEIPASS
# to get --onefile mode working.
os.putenv('_MEIPASS2', sys._MEIPASS)
try:
super(_Popen, self).__init__(*args, **kw)
finally:
if hasattr(sys, 'frozen'):
# On some platforms (e.g. AIX) 'os.unsetenv()' is not
# available. In those cases we cannot delete the variable
# but only set it to the empty string. The bootloader
# can handle this case.
if hasattr(os, 'unsetenv'):
os.unsetenv('_MEIPASS2')
else:
os.putenv('_MEIPASS2', '')
CommandLineToArgvW = windll.shell32.CommandLineToArgvW # Second override 'Popen' class with our modified version.
CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(c_int)] forking.Popen = _Popen
CommandLineToArgvW.restype = POINTER(LPWSTR)
cmd = GetCommandLineW() def win32_unicode_argv():
argc = c_int(0) from ctypes import POINTER, byref, cdll, c_int, windll
argv = CommandLineToArgvW(cmd, byref(argc)) from ctypes.wintypes import LPCWSTR, LPWSTR
if argc.value > 0:
# Remove Python executable and commands if present GetCommandLineW = cdll.kernel32.GetCommandLineW
argc_value = int(argc.value) GetCommandLineW.argtypes = []
sys.argv = argv[argc_value-len(sys.argv):argc_value] GetCommandLineW.restype = LPCWSTR
CommandLineToArgvW = windll.shell32.CommandLineToArgvW
CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(c_int)]
CommandLineToArgvW.restype = POINTER(LPWSTR)
cmd = GetCommandLineW()
argc = c_int(0)
argv = CommandLineToArgvW(cmd, byref(argc))
if argc.value > 0:
# Remove Python executable and commands if present
argc_value = int(argc.value)
sys.argv = argv[argc_value-len(sys.argv):argc_value]
# Run from command line # Run from command line
if __name__ == "__main__": if __name__ == "__main__":
if sys.platform.startswith('win'):
freeze_support()
reload(sys) reload(sys)
if sys.version_info[:2] != (2, 7): if sys.version_info[:2] != (2, 7):
print u'ERROR: GAM requires Python 2.7. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.' % sys.version_info[:3] print u'ERROR: GAM requires Python 2.7. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.' % sys.version_info[:3]
sys.exit(5) sys.exit(5)
if hasattr(sys, u'setdefaultencoding'): if hasattr(sys, u'setdefaultencoding'):
sys.setdefaultencoding(u'UTF-8') sys.setdefaultencoding(u'UTF-8')
if GM_Globals[GM_WINDOWS]: if sys.platform.startswith('win'):
win32_unicode_argv() # cleanup sys.argv on Windows win32_unicode_argv() # cleanup sys.argv on Windows
sys.exit(ProcessGAMCommand(sys.argv)) sys.exit(ProcessGAMCommand(sys.argv))