From 04f49a7a777c3ceb14b8e8d5b52b12366f84f9a7 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 28 Oct 2019 09:17:59 -0700 Subject: [PATCH 1/3] Only use multiprocessing when the method is fork In python3.8 on macos and in all versions on windows the multiprocessing method is `spawn` which does not preserve class state. --- docs/source/internal/utils.rst | 6 ------ src/flake8/checker.py | 28 ++++++++++++++-------------- src/flake8/utils.py | 24 ------------------------ tests/unit/test_checker_manager.py | 2 +- 4 files changed, 15 insertions(+), 45 deletions(-) diff --git a/docs/source/internal/utils.rst b/docs/source/internal/utils.rst index 31e0d39..bd1479e 100644 --- a/docs/source/internal/utils.rst +++ b/docs/source/internal/utils.rst @@ -64,12 +64,6 @@ allows plugins to use this to retrieve ``stdin`` if necessary. This provides a convenient and explicitly named function that checks if we are currently running on a Windows (or ``nt``) operating system. -.. autofunction:: flake8.utils.can_run_multiprocessing_on_windows - -This provides a separate and distinct check from -:func:`~flake8.utils.is_windows` that allows us to check if the version of -Python we're using can actually use multiprocessing on Windows. - .. autofunction:: flake8.utils.is_using_stdin Another helpful function that is named only to be explicit given it is a very diff --git a/src/flake8/checker.py b/src/flake8/checker.py index 52a6a70..ff8eb33 100644 --- a/src/flake8/checker.py +++ b/src/flake8/checker.py @@ -3,6 +3,7 @@ import collections import errno import logging import signal +import sys import tokenize from typing import Dict, List, Optional, Tuple @@ -34,6 +35,18 @@ SERIAL_RETRY_ERRNOS = { } +def _multiprocessing_is_fork(): # type () -> bool + """Class state is only preserved when using the `fork` strategy.""" + if sys.version_info >= (3, 4): + return ( + multiprocessing + # https://github.com/python/typeshed/pull/3415 + and multiprocessing.get_start_method() == "fork" # type: ignore + ) + else: + return multiprocessing and not utils.is_windows() + + class Manager(object): """Manage the parallelism and checker instances for each plugin and file. @@ -97,26 +110,13 @@ class Manager(object): # - we're processing a diff, which again does not work well with # multiprocessing and which really shouldn't require multiprocessing # - the user provided some awful input - if not multiprocessing: + if not _multiprocessing_is_fork(): LOG.warning( "The multiprocessing module is not available. " "Ignoring --jobs arguments." ) return 0 - if ( - utils.is_windows() - and not utils.can_run_multiprocessing_on_windows() - ): - LOG.warning( - "The --jobs option is not available on Windows due to" - " a bug (https://bugs.python.org/issue27649) in " - "Python 2.7.11+ and 3.3+. We have detected that you " - "are running an unsupported version of Python on " - "Windows. Ignoring --jobs arguments." - ) - return 0 - if utils.is_using_stdin(self.arguments): LOG.warning( "The --jobs option is not compatible with supplying " diff --git a/src/flake8/utils.py b/src/flake8/utils.py index f822388..b7660bc 100644 --- a/src/flake8/utils.py +++ b/src/flake8/utils.py @@ -302,30 +302,6 @@ def is_windows(): return os.name == "nt" -# NOTE(sigmavirus24): If and when https://bugs.python.org/issue27649 is fixed, -# re-enable multiprocessing support on Windows. -def can_run_multiprocessing_on_windows(): - # type: () -> bool - """Determine if we can use multiprocessing on Windows. - - This presently will **always** return False due to a `bug`_ in the - :mod:`multiprocessing` module on Windows. Once fixed, we will check - to ensure that the version of Python contains that fix (via version - inspection) and *conditionally* re-enable support on Windows. - - .. _bug: - https://bugs.python.org/issue27649 - - :returns: - True if the version of Python is modern enough, otherwise False - :rtype: - bool - """ - is_new_enough_python27 = (2, 7, 11) <= sys.version_info < (3, 0) - is_new_enough_python3 = sys.version_info > (3, 2) - return False and (is_new_enough_python27 or is_new_enough_python3) - - def is_using_stdin(paths): # type: (List[str]) -> bool """Determine if we're going to read from stdin. diff --git a/tests/unit/test_checker_manager.py b/tests/unit/test_checker_manager.py index 50f4084..1c2f7e3 100644 --- a/tests/unit/test_checker_manager.py +++ b/tests/unit/test_checker_manager.py @@ -35,7 +35,7 @@ def test_oserrors_cause_serial_fall_back(): assert serial.call_count == 1 -@mock.patch('flake8.utils.is_windows', return_value=False) +@mock.patch('flake8.checker._multiprocessing_is_fork', return_value=True) def test_oserrors_are_reraised(is_windows): """Verify that unexpected OSErrors will cause the Manager to reraise.""" err = OSError(errno.EAGAIN, 'Ominous message') From 04d3f9dcafcc8fe3eabf66ba13b999c2e5b38f60 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 10 Aug 2019 13:11:29 -0700 Subject: [PATCH 2/3] Fix travis-ci --- .travis.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index b47d090..7e7f997 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ -sudo: false +dist: xenial language: python cache: pip before_script: -- pip install tox +- pip install --upgrade tox setuptools virtualenv # test script -script: tox +script: tox notifications: on_success: change on_failure: always @@ -20,20 +20,19 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - # https://github.com/travis-ci/travis-ci/issues/9069 - #- python: 3.7 - # env: TOXENV=py37 + - python: 3.7 + env: TOXENV=py37 - python: pypy env: TOXENV=pypy - python: 2.7 env: TOXENV=readme - - python: 3.4 + - python: 3.7 env: TOXENV=flake8 - - python: 3.4 + - python: 3.7 env: TOXENV=pylint - - python: 3.4 + - python: 3.7 env: TOXENV=doc8 - - python: 3.4 + - python: 3.7 env: TOXENV=bandit - - python: 3.4 + - python: 3.7 env: TOXENV=docs From ee2920d775df18481d638c2da084d229d56f95b9 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 28 Oct 2019 10:34:54 -0700 Subject: [PATCH 3/3] Release 3.7.9 --- docs/source/release-notes/3.7.9.rst | 23 +++++++++++++++++++++++ docs/source/release-notes/index.rst | 1 + src/flake8/__init__.py | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 docs/source/release-notes/3.7.9.rst diff --git a/docs/source/release-notes/3.7.9.rst b/docs/source/release-notes/3.7.9.rst new file mode 100644 index 0000000..b1a2ce2 --- /dev/null +++ b/docs/source/release-notes/3.7.9.rst @@ -0,0 +1,23 @@ +3.7.9 -- 2019-10-28 +------------------- + +You can view the `3.7.9 milestone`_ on GitLab for more details. + +Bugs Fixed +~~~~~~~~~~ + +- Disable multiprocessing when the multiprocessing method is ``spawn`` (such + as on macos in python3.8) (See also `GitLab!367`_, `GitLab#587`_) + + +.. all links +.. _3.7.9 milestone: + https://gitlab.com/pycqa/flake8/milestones/33 + +.. issue links +.. _GitLab#587: + https://gitlab.com/pycqa/flake8/issues/587 + +.. merge request links +.. _GitLab!367: + https://gitlab.com/pycqa/flake8/merge_requests/367 diff --git a/docs/source/release-notes/index.rst b/docs/source/release-notes/index.rst index 96aa184..e532fd6 100644 --- a/docs/source/release-notes/index.rst +++ b/docs/source/release-notes/index.rst @@ -9,6 +9,7 @@ with the newest releases first. ================== .. toctree:: + 3.7.9 3.7.8 3.7.7 3.7.6 diff --git a/src/flake8/__init__.py b/src/flake8/__init__.py index f370ffb..06b3f34 100644 --- a/src/flake8/__init__.py +++ b/src/flake8/__init__.py @@ -18,7 +18,7 @@ if False: # `typing.TYPE_CHECKING` was introduced in 3.5.2 LOG = logging.getLogger(__name__) LOG.addHandler(logging.NullHandler()) -__version__ = "3.7.8" +__version__ = "3.7.9" __version_info__ = tuple( int(i) for i in __version__.split(".") if i.isdigit() )