deepblue/GoogleDriveManagement
1
0
Fork 0
mirror of https://github.com/GAM-team/GAM.git synced 2026-07-03 20:31:35 +00:00
Code Issues Actions 19 Packages Projects Releases Wiki Activity

forkserver for Linux, #1843

Browse Source
This commit is contained in:
Jay Lee
2025-10-08 14:43:28 +00:00
parent 072dc4809a
commit bed9db37ad
2 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/gam/__init__.py
View File

@@ -9596,7 +9596,7 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
clearRowFilters = False clearRowFilters = False
# if sys.platform.startswith('win'): # if sys.platform.startswith('win'):
# signal.signal(signal.SIGINT, signal.SIG_IGN) # signal.signal(signal.SIGINT, signal.SIG_IGN)
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
Cmd = glclargs.GamCLArgs() Cmd = glclargs.GamCLArgs()
else: else:
@@ -9638,7 +9638,7 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
Cmd.InitializeArguments(dataItem) Cmd.InitializeArguments(dataItem)
elif dataType == GM.REDIRECT_QUEUE_GLOBALS: elif dataType == GM.REDIRECT_QUEUE_GLOBALS:
GM.Globals = dataItem GM.Globals = dataItem
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
reopenSTDFile(GM.STDOUT) reopenSTDFile(GM.STDOUT)
reopenSTDFile(GM.STDERR) reopenSTDFile(GM.STDERR)
elif dataType == GM.REDIRECT_QUEUE_VALUES: elif dataType == GM.REDIRECT_QUEUE_VALUES:
@@ -9684,7 +9684,7 @@ def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr):
def terminateCSVFileQueueHandler(mpQueue, mpQueueHandler): def terminateCSVFileQueueHandler(mpQueue, mpQueueHandler):
GM.Globals[GM.PARSER] = None GM.Globals[GM.PARSER] = None
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, Cmd.AllArguments())) mpQueue.put((GM.REDIRECT_QUEUE_ARGS, Cmd.AllArguments()))
savedValues = saveNonPickleableValues() savedValues = saveNonPickleableValues()
mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals)) mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals))
@@ -9714,13 +9714,13 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
# if sys.platform.startswith('win'): # if sys.platform.startswith('win'):
# signal.signal(signal.SIGINT, signal.SIG_IGN) # signal.signal(signal.SIGINT, signal.SIG_IGN)
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
GM.Globals = gmGlobals.copy() GM.Globals = gmGlobals.copy()
GC.Values = gcValues.copy() GC.Values = gcValues.copy()
pid0DataItem = [KEYBOARD_INTERRUPT_RC, None] pid0DataItem = [KEYBOARD_INTERRUPT_RC, None]
pidData = {} pidData = {}
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null': if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=UTF8) fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-': elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
@@ -9808,7 +9808,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
with mplock: with mplock:
initializeLogging() initializeLogging()
# if sys.platform.startswith('win'): # if sys.platform.startswith('win'):
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
GM.Globals[GM.API_CALLS_RETRY_DATA] = {} GM.Globals[GM.API_CALLS_RETRY_DATA] = {}
GM.Globals[GM.CMDLOG_LOGGER] = None GM.Globals[GM.CMDLOG_LOGGER] = None
@@ -9949,7 +9949,7 @@ def MultiprocessGAMCommands(items, showCmds):
mpManager = multiprocessing.Manager() mpManager = multiprocessing.Manager()
l = mpManager.Lock() l = mpManager.Lock()
try: try:
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
pool = mpManager.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200) pool = mpManager.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
else: else:
pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200) pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
@@ -9958,7 +9958,7 @@ def MultiprocessGAMCommands(items, showCmds):
except AssertionError as e: except AssertionError as e:
Cmd.SetLocation(0) Cmd.SetLocation(0)
usageErrorExit(str(e)) usageErrorExit(str(e))
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() == 'fork':
savedValues = saveNonPickleableValues() savedValues = saveNonPickleableValues()
if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]: if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]:
mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values) mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values)
@@ -9973,7 +9973,7 @@ def MultiprocessGAMCommands(items, showCmds):
mpQueueStderr = mpQueueStdout mpQueueStderr = mpQueueStdout
else: else:
mpQueueStderr = None mpQueueStderr = None
if multiprocessing.get_start_method() == 'spawn': if multiprocessing.get_start_method() != 'fork':
restoreNonPickleableValues(savedValues) restoreNonPickleableValues(savedValues)
if mpQueueStdout: if mpQueueStdout:
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue())) mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()))

5
src/gam/__main__.py
View File

@@ -36,8 +36,9 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
multiprocessing.freeze_support() multiprocessing.freeze_support()
if platform.system() == 'Linux': if platform.system() == 'Linux':
# no longer default in Python 3.14+ so we set it. # set explictly since it's not default in Python < 3.14, forkserver should
multiprocessing.set_start_method('fork') # be safer than fork and less likely to see bulk command hangs.
multiprocessing.set_start_method('forkserver')
else: else:
multiprocessing.set_start_method('spawn') multiprocessing.set_start_method('spawn')
main() main()