From 52cbf91cd3e83fd196926db978c55496a7b15425 Mon Sep 17 00:00:00 2001 From: pylberto Date: Wed, 12 Feb 2025 14:04:24 -0800 Subject: [PATCH] fix: set a max jobs on win32 to avoid too many handles This commit fixes the issue of too many handles on win32 when `--jobs=auto`. This is done by setting the `job_count` to the minimum value between `multiprocessing.cpu_count()` and 61. This happens when a machine has > 61 logic processors and `multiprocessing.cpu_count()` returns a value greater than 61. See [Python Issue 26903](https://bugs.python.org/issue26903) for more information. Example failure for Python 3.10.15 with flake8 7.1.0 on Windows 11: ```bash flake8.EXE myfile.py --jobs=auto Exception in thread Thread-1 (_handle_workers): Traceback (most recent call last): File "lib\threading.py", line 1016, in _bootstrap_inner self.run() File "lib\threading.py", line 953, in run self._target(*self._args, **self._kwargs) File "lib\multiprocessing\pool.py", line 522, in _handle_workers cls._wait_for_updates(current_sentinels, change_notifier) File "lib\multiprocessing\pool.py", line 502, in _wait_for_updates wait(sentinels, timeout=timeout) File "lib\multiprocessing\connection.py", line 879, in wait ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout) File "lib\multiprocessing\connection.py", line 811, in _exhaustive_wait res = _winapi.WaitForMultipleObjects(L, False, timeout) ValueError: need at most 63 handles, got a sequence of length 194 ``` --- src/flake8/checker.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index 329a2cc..ab24202 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -8,6 +8,7 @@ import logging import multiprocessing.pool import operator import signal +import sys import tokenize from typing import Any from typing import Generator @@ -151,7 +152,14 @@ class Manager: # default to 1 if jobs.is_auto: try: - return multiprocessing.cpu_count() + system_cpus = multiprocessing.cpu_count() + + # Work around https://bugs.python.org/issue26903 + return ( + min(system_cpus, 61) + if sys.platform == "win32" + else system_cpus + ) except NotImplementedError: return 0