diff --git a/src/flake8/api/legacy.py b/src/flake8/api/legacy.py index 4d5c91d..d61e3d7 100644 --- a/src/flake8/api/legacy.py +++ b/src/flake8/api/legacy.py @@ -162,7 +162,9 @@ class StyleGuide: # Stop cringing... I know it's gross. self._application.make_guide() self._application.file_checker_manager = None - self._application.make_file_checker_manager([]) + self._application.make_file_checker_manager( + [], use_initialized_options=True, + ) def input_file( self, @@ -212,5 +214,5 @@ def get_style_guide(**kwargs: Any) -> StyleGuide: LOG.error('Could not update option "%s"', key) application.make_formatter() application.make_guide() - application.make_file_checker_manager([]) + application.make_file_checker_manager([], use_initialized_options=True) return StyleGuide(application) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index c6a24eb..07cbb7e 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -7,6 +7,7 @@ import errno import logging import multiprocessing.pool import operator +import pickle import signal import tokenize from collections.abc import Generator @@ -61,16 +62,24 @@ def _mp_prefork( _mp = None -def _mp_init(argv: Sequence[str]) -> None: +def _mp_init( + argv: Sequence[str], + plugins: Checkers | None = None, + options: argparse.Namespace | None = None, +) -> None: global _mp # Ensure correct signaling of ^C using multiprocessing.Pool. signal.signal(signal.SIGINT, signal.SIG_IGN) + if plugins is not None and options is not None: + _mp = plugins, options + return + # for `fork` this'll already be set if _mp is None: - plugins, options = parse_args(argv) - _mp = plugins.checkers, options + parsed_plugins, parsed_options = parse_args(argv) + _mp = parsed_plugins.checkers, parsed_options def _mp_run(filename: str) -> tuple[str, Results, dict[str, int]]: @@ -105,6 +114,7 @@ class Manager: style_guide: StyleGuideManager, plugins: Checkers, argv: Sequence[str], + use_initialized_options: bool = False, ) -> None: """Initialize our Manager instance.""" self.style_guide = style_guide @@ -119,6 +129,7 @@ class Manager: } self.exclude = (*self.options.exclude, *self.options.extend_exclude) self.argv = argv + self.use_initialized_options = use_initialized_options self.results: list[tuple[str, Results, dict[str, int]]] = [] def _process_statistics(self) -> None: @@ -191,8 +202,12 @@ class Manager: def run_parallel(self) -> None: """Run the checkers in parallel.""" + plugins = self.plugins if self.use_initialized_options else None + options = self.options if self.use_initialized_options else None with _mp_prefork(self.plugins, self.options): - pool = _try_initialize_processpool(self.jobs, self.argv) + pool = _try_initialize_processpool( + self.jobs, self.argv, plugins=plugins, options=options, + ) if pool is None: self.run_serial() @@ -547,13 +562,21 @@ class FileChecker: def _try_initialize_processpool( job_count: int, argv: Sequence[str], + *, + plugins: Checkers | None = None, + options: argparse.Namespace | None = None, ) -> multiprocessing.pool.Pool | None: """Return a new process pool instance if we are able to create one.""" try: - return multiprocessing.Pool(job_count, _mp_init, initargs=(argv,)) + return multiprocessing.Pool( + job_count, _mp_init, initargs=(argv, plugins, options), + ) except OSError as err: if err.errno not in SERIAL_RETRY_ERRNOS: raise + except (AttributeError, pickle.PicklingError, TypeError): + if plugins is None and options is None: + raise except ImportError: pass diff --git a/src/flake8/main/application.py b/src/flake8/main/application.py index 165a6ef..bbea4fc 100644 --- a/src/flake8/main/application.py +++ b/src/flake8/main/application.py @@ -79,7 +79,9 @@ class Application: self.options, self.formatter, ) - def make_file_checker_manager(self, argv: Sequence[str]) -> None: + def make_file_checker_manager( + self, argv: Sequence[str], use_initialized_options: bool = False, + ) -> None: """Initialize our FileChecker Manager.""" assert self.guide is not None assert self.plugins is not None @@ -87,6 +89,7 @@ class Application: style_guide=self.guide, plugins=self.plugins.checkers, argv=argv, + use_initialized_options=use_initialized_options, ) def run_checks(self) -> None: diff --git a/tests/integration/test_api_legacy.py b/tests/integration/test_api_legacy.py index b386bd5..aec704d 100644 --- a/tests/integration/test_api_legacy.py +++ b/tests/integration/test_api_legacy.py @@ -2,6 +2,7 @@ from __future__ import annotations from flake8.api import legacy +from flake8.main.options import JobsArgument def test_legacy_api(tmpdir): @@ -13,3 +14,17 @@ def test_legacy_api(tmpdir): style_guide = legacy.get_style_guide() report = style_guide.check_files([t_py.strpath]) assert report.total_errors == 1 + + +def test_legacy_api_uses_initialized_options_for_parallel_checks(tmpdir): + with tmpdir.as_cwd(): + a_py = tmpdir.join("a.py") + b_py = tmpdir.join("b.py") + a_py.write('x = "' + "a" * 80 + '"\n') + b_py.write('y = "' + "b" * 80 + '"\n') + + style_guide = legacy.get_style_guide(color="never", max_line_length=88) + style_guide.options.jobs = JobsArgument("2") + report = style_guide.check_files([a_py.strpath, b_py.strpath]) + + assert report.total_errors == 0 diff --git a/tests/integration/test_checker.py b/tests/integration/test_checker.py index f7f07af..a7e0347 100644 --- a/tests/integration/test_checker.py +++ b/tests/integration/test_checker.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib.metadata +import pickle from unittest import mock import pytest @@ -291,7 +292,24 @@ def test_acquire_when_multiprocessing_pool_can_initialize(): with mock.patch("multiprocessing.Pool") as pool: result = checker._try_initialize_processpool(2, []) - pool.assert_called_once_with(2, checker._mp_init, initargs=([],)) + pool.assert_called_once_with( + 2, checker._mp_init, initargs=([], None, None), + ) + assert result is pool.return_value + + +def test_acquire_when_multiprocessing_pool_uses_initialized_options(): + plugins = mock.Mock() + options = mock.Mock() + + with mock.patch("multiprocessing.Pool") as pool: + result = checker._try_initialize_processpool( + 2, [], plugins=plugins, options=options, + ) + + pool.assert_called_once_with( + 2, checker._mp_init, initargs=([], plugins, options), + ) assert result is pool.return_value @@ -310,7 +328,21 @@ def test_acquire_when_multiprocessing_pool_can_not_initialize(): with mock.patch("multiprocessing.Pool", side_effect=ImportError) as pool: result = checker._try_initialize_processpool(2, []) - pool.assert_called_once_with(2, checker._mp_init, initargs=([],)) + pool.assert_called_once_with( + 2, checker._mp_init, initargs=([], None, None), + ) + assert result is None + + +def test_acquire_falls_back_when_initialized_options_are_unpickleable(): + with mock.patch( + "multiprocessing.Pool", side_effect=pickle.PicklingError, + ) as pool: + result = checker._try_initialize_processpool( + 2, [], plugins=mock.Mock(), options=mock.Mock(), + ) + + pool.assert_called_once() assert result is None