mirror of
https://github.com/PyCQA/flake8.git
synced 2026-06-29 00:30:44 +00:00
Preserve legacy API options in parallel checks
This commit is contained in:
parent
bb943328ef
commit
0df41b2d56
5 changed files with 85 additions and 10 deletions
|
|
@ -162,7 +162,9 @@ class StyleGuide:
|
|||
# Stop cringing... I know it's gross.
|
||||
self._application.make_guide()
|
||||
self._application.file_checker_manager = None
|
||||
self._application.make_file_checker_manager([])
|
||||
self._application.make_file_checker_manager(
|
||||
[], use_initialized_options=True,
|
||||
)
|
||||
|
||||
def input_file(
|
||||
self,
|
||||
|
|
@ -212,5 +214,5 @@ def get_style_guide(**kwargs: Any) -> StyleGuide:
|
|||
LOG.error('Could not update option "%s"', key)
|
||||
application.make_formatter()
|
||||
application.make_guide()
|
||||
application.make_file_checker_manager([])
|
||||
application.make_file_checker_manager([], use_initialized_options=True)
|
||||
return StyleGuide(application)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import errno
|
|||
import logging
|
||||
import multiprocessing.pool
|
||||
import operator
|
||||
import pickle
|
||||
import signal
|
||||
import tokenize
|
||||
from collections.abc import Generator
|
||||
|
|
@ -61,16 +62,24 @@ def _mp_prefork(
|
|||
_mp = None
|
||||
|
||||
|
||||
def _mp_init(argv: Sequence[str]) -> None:
|
||||
def _mp_init(
|
||||
argv: Sequence[str],
|
||||
plugins: Checkers | None = None,
|
||||
options: argparse.Namespace | None = None,
|
||||
) -> None:
|
||||
global _mp
|
||||
|
||||
# Ensure correct signaling of ^C using multiprocessing.Pool.
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
|
||||
if plugins is not None and options is not None:
|
||||
_mp = plugins, options
|
||||
return
|
||||
|
||||
# for `fork` this'll already be set
|
||||
if _mp is None:
|
||||
plugins, options = parse_args(argv)
|
||||
_mp = plugins.checkers, options
|
||||
parsed_plugins, parsed_options = parse_args(argv)
|
||||
_mp = parsed_plugins.checkers, parsed_options
|
||||
|
||||
|
||||
def _mp_run(filename: str) -> tuple[str, Results, dict[str, int]]:
|
||||
|
|
@ -105,6 +114,7 @@ class Manager:
|
|||
style_guide: StyleGuideManager,
|
||||
plugins: Checkers,
|
||||
argv: Sequence[str],
|
||||
use_initialized_options: bool = False,
|
||||
) -> None:
|
||||
"""Initialize our Manager instance."""
|
||||
self.style_guide = style_guide
|
||||
|
|
@ -119,6 +129,7 @@ class Manager:
|
|||
}
|
||||
self.exclude = (*self.options.exclude, *self.options.extend_exclude)
|
||||
self.argv = argv
|
||||
self.use_initialized_options = use_initialized_options
|
||||
self.results: list[tuple[str, Results, dict[str, int]]] = []
|
||||
|
||||
def _process_statistics(self) -> None:
|
||||
|
|
@ -191,8 +202,12 @@ class Manager:
|
|||
|
||||
def run_parallel(self) -> None:
|
||||
"""Run the checkers in parallel."""
|
||||
plugins = self.plugins if self.use_initialized_options else None
|
||||
options = self.options if self.use_initialized_options else None
|
||||
with _mp_prefork(self.plugins, self.options):
|
||||
pool = _try_initialize_processpool(self.jobs, self.argv)
|
||||
pool = _try_initialize_processpool(
|
||||
self.jobs, self.argv, plugins=plugins, options=options,
|
||||
)
|
||||
|
||||
if pool is None:
|
||||
self.run_serial()
|
||||
|
|
@ -547,13 +562,21 @@ class FileChecker:
|
|||
def _try_initialize_processpool(
|
||||
job_count: int,
|
||||
argv: Sequence[str],
|
||||
*,
|
||||
plugins: Checkers | None = None,
|
||||
options: argparse.Namespace | None = None,
|
||||
) -> multiprocessing.pool.Pool | None:
|
||||
"""Return a new process pool instance if we are able to create one."""
|
||||
try:
|
||||
return multiprocessing.Pool(job_count, _mp_init, initargs=(argv,))
|
||||
return multiprocessing.Pool(
|
||||
job_count, _mp_init, initargs=(argv, plugins, options),
|
||||
)
|
||||
except OSError as err:
|
||||
if err.errno not in SERIAL_RETRY_ERRNOS:
|
||||
raise
|
||||
except (AttributeError, pickle.PicklingError, TypeError):
|
||||
if plugins is None and options is None:
|
||||
raise
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,9 @@ class Application:
|
|||
self.options, self.formatter,
|
||||
)
|
||||
|
||||
def make_file_checker_manager(self, argv: Sequence[str]) -> None:
|
||||
def make_file_checker_manager(
|
||||
self, argv: Sequence[str], use_initialized_options: bool = False,
|
||||
) -> None:
|
||||
"""Initialize our FileChecker Manager."""
|
||||
assert self.guide is not None
|
||||
assert self.plugins is not None
|
||||
|
|
@ -87,6 +89,7 @@ class Application:
|
|||
style_guide=self.guide,
|
||||
plugins=self.plugins.checkers,
|
||||
argv=argv,
|
||||
use_initialized_options=use_initialized_options,
|
||||
)
|
||||
|
||||
def run_checks(self) -> None:
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from flake8.api import legacy
|
||||
from flake8.main.options import JobsArgument
|
||||
|
||||
|
||||
def test_legacy_api(tmpdir):
|
||||
|
|
@ -13,3 +14,17 @@ def test_legacy_api(tmpdir):
|
|||
style_guide = legacy.get_style_guide()
|
||||
report = style_guide.check_files([t_py.strpath])
|
||||
assert report.total_errors == 1
|
||||
|
||||
|
||||
def test_legacy_api_uses_initialized_options_for_parallel_checks(tmpdir):
|
||||
with tmpdir.as_cwd():
|
||||
a_py = tmpdir.join("a.py")
|
||||
b_py = tmpdir.join("b.py")
|
||||
a_py.write('x = "' + "a" * 80 + '"\n')
|
||||
b_py.write('y = "' + "b" * 80 + '"\n')
|
||||
|
||||
style_guide = legacy.get_style_guide(color="never", max_line_length=88)
|
||||
style_guide.options.jobs = JobsArgument("2")
|
||||
report = style_guide.check_files([a_py.strpath, b_py.strpath])
|
||||
|
||||
assert report.total_errors == 0
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
import pickle
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -291,7 +292,24 @@ def test_acquire_when_multiprocessing_pool_can_initialize():
|
|||
with mock.patch("multiprocessing.Pool") as pool:
|
||||
result = checker._try_initialize_processpool(2, [])
|
||||
|
||||
pool.assert_called_once_with(2, checker._mp_init, initargs=([],))
|
||||
pool.assert_called_once_with(
|
||||
2, checker._mp_init, initargs=([], None, None),
|
||||
)
|
||||
assert result is pool.return_value
|
||||
|
||||
|
||||
def test_acquire_when_multiprocessing_pool_uses_initialized_options():
|
||||
plugins = mock.Mock()
|
||||
options = mock.Mock()
|
||||
|
||||
with mock.patch("multiprocessing.Pool") as pool:
|
||||
result = checker._try_initialize_processpool(
|
||||
2, [], plugins=plugins, options=options,
|
||||
)
|
||||
|
||||
pool.assert_called_once_with(
|
||||
2, checker._mp_init, initargs=([], plugins, options),
|
||||
)
|
||||
assert result is pool.return_value
|
||||
|
||||
|
||||
|
|
@ -310,7 +328,21 @@ def test_acquire_when_multiprocessing_pool_can_not_initialize():
|
|||
with mock.patch("multiprocessing.Pool", side_effect=ImportError) as pool:
|
||||
result = checker._try_initialize_processpool(2, [])
|
||||
|
||||
pool.assert_called_once_with(2, checker._mp_init, initargs=([],))
|
||||
pool.assert_called_once_with(
|
||||
2, checker._mp_init, initargs=([], None, None),
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_acquire_falls_back_when_initialized_options_are_unpickleable():
|
||||
with mock.patch(
|
||||
"multiprocessing.Pool", side_effect=pickle.PicklingError,
|
||||
) as pool:
|
||||
result = checker._try_initialize_processpool(
|
||||
2, [], plugins=mock.Mock(), options=mock.Mock(),
|
||||
)
|
||||
|
||||
pool.assert_called_once()
|
||||
assert result is None
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue