From 79a43160559ea11c0e6cd4c845510687fdb2e37e Mon Sep 17 00:00:00 2001 From: Florent <284942-Hewyn@users.noreply.gitlab.com> Date: Sun, 24 Jul 2022 14:06:56 +0200 Subject: [PATCH] Enforce usage of fork method on macOS --- src/flake8/checker.py | 10 ++++++++-- tests/integration/test_checker.py | 11 ++++++----- tests/unit/test_checker_manager.py | 17 ++++++++++++++++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/flake8/checker.py b/src/flake8/checker.py index 2bbd6f2..de4c67b 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -5,6 +5,7 @@ import errno import itertools import logging import multiprocessing.pool +import platform import signal import tokenize from typing import Any @@ -101,7 +102,11 @@ class Manager: # - the user provided some awful input # class state is only preserved when using the `fork` strategy. - if multiprocessing.get_start_method() != "fork": + # since Python 3.8, macOS has the method `spawn` as the default value + if ( + platform.system() != "Darwin" + and multiprocessing.get_start_method() != "fork" + ): LOG.warning( "The multiprocessing module is not available. " "Ignoring --jobs arguments." @@ -595,7 +600,8 @@ def _try_initialize_processpool( ) -> Optional[multiprocessing.pool.Pool]: """Return a new process pool instance if we are able to create one.""" try: - return multiprocessing.Pool(job_count, _pool_init) + context = multiprocessing.get_context("fork") + return context.Pool(job_count, _pool_init) except OSError as err: if err.errno not in SERIAL_RETRY_ERRNOS: raise diff --git a/tests/integration/test_checker.py b/tests/integration/test_checker.py index 9223ec4..bc0eee0 100644 --- a/tests/integration/test_checker.py +++ b/tests/integration/test_checker.py @@ -292,11 +292,11 @@ def test_acquire_when_multiprocessing_pool_can_initialize(): This simulates the behaviour on most common platforms. """ - with mock.patch("multiprocessing.Pool") as pool: + with mock.patch("multiprocessing.get_context") as context: result = checker._try_initialize_processpool(2) - pool.assert_called_once_with(2, checker._pool_init) - assert result is pool.return_value + context.return_value.Pool.assert_called_once_with(2, checker._pool_init) + assert result is context.return_value.Pool.return_value def test_acquire_when_multiprocessing_pool_can_not_initialize(): @@ -311,10 +311,11 @@ def test_acquire_when_multiprocessing_pool_can_not_initialize(): https://github.com/python/cpython/blob/4e02981de0952f54bf87967f8e10d169d6946b40/Lib/multiprocessing/synchronize.py#L30-L33 """ - with mock.patch("multiprocessing.Pool", side_effect=ImportError) as pool: + with mock.patch("multiprocessing.get_context") as context: + context.return_value.Pool.side_effect = ImportError result = checker._try_initialize_processpool(2) - pool.assert_called_once_with(2, checker._pool_init) + context.return_value.Pool.assert_called_once_with(2, checker._pool_init) assert result is None diff --git a/tests/unit/test_checker_manager.py b/tests/unit/test_checker_manager.py index c6114f6..fc4a79e 100644 --- a/tests/unit/test_checker_manager.py +++ b/tests/unit/test_checker_manager.py @@ -1,6 +1,7 @@ """Tests for the Manager object for FileCheckers.""" import errno import multiprocessing +import platform from unittest import mock import pytest @@ -51,14 +52,28 @@ def test_oserrors_are_reraised(_): assert serial.call_count == 0 +@mock.patch.object(platform, "system") @mock.patch.object(multiprocessing, "get_start_method", return_value="spawn") -def test_multiprocessing_is_disabled(_): +def test_multiprocessing_is_disabled(_, mock_system): """Verify not being able to import multiprocessing forces jobs to 0.""" style_guide = style_guide_mock() manager = checker.Manager(style_guide, finder.Checkers([], [], [])) assert manager.jobs == 0 +@mock.patch.object(platform, "system", return_value="Darwin") +@mock.patch.object(multiprocessing, "get_start_method", return_value="spawn") +def test_multiprocessing_is_enabled_for_macos(_, mock_system): + """Verify jobs are returned on macOS. + + Since Python 3.8, `spawn` is the default value on macOS, which is + not currently supported by flake8. + """ + style_guide = style_guide_mock() + manager = checker.Manager(style_guide, finder.Checkers([], [], [])) + assert manager.jobs > 0 + + def test_multiprocessing_cpu_count_not_implemented(): """Verify that jobs is 0 if cpu_count is unavailable.""" style_guide = style_guide_mock()