fix: set a max jobs on win32 to avoid too many handles

This commit fixes the issue of too many handles on win32 when `--jobs=auto`.
This is done by setting the `job_count` to the minimum value between
`multiprocessing.cpu_count()` and 61.  This happens when a machine has >
61 logic processors and `multiprocessing.cpu_count()` returns a value
greater than 61.

See [Python Issue 26903](https://bugs.python.org/issue26903) for more information.

Example failure for Python 3.10.15 with flake8 7.1.0 on Windows 11:

```bash
flake8.EXE myfile.py --jobs=auto
Exception in thread Thread-1 (_handle_workers):
Traceback (most recent call last):
  File "lib\threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "lib\threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "lib\multiprocessing\pool.py", line 522, in _handle_workers
    cls._wait_for_updates(current_sentinels, change_notifier)
  File "lib\multiprocessing\pool.py", line 502, in _wait_for_updates
    wait(sentinels, timeout=timeout)
  File "lib\multiprocessing\connection.py", line 879, in wait
    ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
  File "lib\multiprocessing\connection.py", line 811, in _exhaustive_wait
    res = _winapi.WaitForMultipleObjects(L, False, timeout)
ValueError: need at most 63 handles, got a sequence of length 194
```
This commit is contained in:
pylberto 2025-02-12 14:04:24 -08:00
parent cf1542cefa
commit 52cbf91cd3

View file

@ -8,6 +8,7 @@ import logging
import multiprocessing.pool import multiprocessing.pool
import operator import operator
import signal import signal
import sys
import tokenize import tokenize
from typing import Any from typing import Any
from typing import Generator from typing import Generator
@ -151,7 +152,14 @@ class Manager:
# default to 1 # default to 1
if jobs.is_auto: if jobs.is_auto:
try: try:
return multiprocessing.cpu_count() system_cpus = multiprocessing.cpu_count()
# Work around https://bugs.python.org/issue26903
return (
min(system_cpus, 61)
if sys.platform == "win32"
else system_cpus
)
except NotImplementedError: except NotImplementedError:
return 0 return 0