From 2d72fc23c804be2697446509e55234f8d96c00fe Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 6 Dec 2015 12:23:45 -0600 Subject: [PATCH 001/204] Add skeleton of DESIGN documentation --- DESIGN.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 DESIGN.rst diff --git a/DESIGN.rst b/DESIGN.rst new file mode 100644 index 0000000..6f40e51 --- /dev/null +++ b/DESIGN.rst @@ -0,0 +1,77 @@ +============== + Design Goals +============== + +Outline +------- + +#. :ref:`plugins` + + #. :ref:`checking` + + #. :ref:`autofixing` + + #. :ref:`options-passing` + +#. :ref:`options` + + #. :ref:`better-select-ignore` + +#. :ref:`standard-in` + +#. :ref:`multiprocessing` + +#. :ref:`formatting` + +#. :ref:`report-generation` + +.. _plugins: + +Better Plugins Support +---------------------- + +.. _checking: + +Support for Plugins that Only Run Checks +++++++++++++++++++++++++++++++++++++++++ + +.. _autofixing: + +Support for Plugins that Autofix Errors ++++++++++++++++++++++++++++++++++++++++ + +.. _options-passing: + +Support for Plugins Require Parsed Options +++++++++++++++++++++++++++++++++++++++++++ + +.. _options: + +Better Options Support +---------------------- + +.. _better-select-ignore: + +Support for Better Select/Ignore Handling ++++++++++++++++++++++++++++++++++++++++++ + +.. _standard-in: + +Better stdin support +-------------------- + +.. _multiprocessing: + +Multiprocessing Support +----------------------- + +.. _formatting: + +Better (Flake8-centralized) Formatting Support +---------------------------------------------- + +.. _report-generation: + + +Better (Flake8-centralized) Report Generation Support +----------------------------------------------------- From eb29f117386641fdc0bca2c3833ec14fea0a5751 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 6 Dec 2015 13:36:01 -0600 Subject: [PATCH 002/204] Flesh out some portions of the design goals --- DESIGN.rst | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/DESIGN.rst b/DESIGN.rst index 6f40e51..dee98ee 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -11,8 +11,12 @@ Outline #. :ref:`autofixing` + #. :ref:`reporter-plugins` + #. :ref:`options-passing` + #. :ref:`plugin-default-ignore` + #. :ref:`options` #. :ref:`better-select-ignore` @@ -30,31 +34,106 @@ Outline Better Plugins Support ---------------------- +Currently, Flake8 has some rather excellent support for plugins. It currently +allows for the following: + +- Third-party packages to register checks + +- Checks to be disabled by default + +- Checks to accept an AST compiled tree, physical lines, or logical lines. + +- Flake8 handles running those checks in separate subprocesses as necessary + +That said, plugins cannot access the options passed on the command-line, or +options parsed from config files (without parsing them, themselves) and all +reporting is handled by pep8 instead of flake8 which reduces the flexibility +users have in aggregating reports. + .. _checking: Support for Plugins that Only Run Checks ++++++++++++++++++++++++++++++++++++++++ +Flake8 currently already supports plugins that only run checks. This support +needs to continue and should be trivial to continue. + .. _autofixing: Support for Plugins that Autofix Errors +++++++++++++++++++++++++++++++++++++++ +Flake8 should enable people writing plugins for both core Flake8 checkers and +third-party checkers that allow the code to be automatically fixed. The trick +is in how to do this. + +Once Flake8 has control over running plugins and treats pep8, flake8, and +mccabe as "plugins", it will aggregate the errors returned by all of the +plugins and be able to "notify" other plugins that have chosen to listen for +errors so those plugins can auto-fix the problems in the file. + +See https://gitlab.com/pycqa/flake8/issues/84 + +.. _reporter-plugins: + +Support for Plugins that Format Output +++++++++++++++++++++++++++++++++++++++ + +Flake8 currently supports formatting output via pep8's ``--format`` option. +This works but is fundamentally a bit limiting. Allowing users to replace or +compose formatters would allow for certain formatters to highlight more +important information over less important information as the user deems +necessary. + +See https://gitlab.com/pycqa/flake8/issues/66 + .. _options-passing: Support for Plugins Require Parsed Options ++++++++++++++++++++++++++++++++++++++++++ +Plugins currently are able to use ``add_options`` and ``parse_options`` +classmethods to register and retrieve options information. This is admittedly +a little awkward and could be improved, but should at least be preserved in +this rewrite. + +See potential improvements as a result of +https://gitlab.com/pycqa/flake8/issues/88 + +.. _plugin-default-ignore: + +Support for Plugins Specifying Default Ignore list +++++++++++++++++++++++++++++++++++++++++++++++++++ + +Plugins currently have no way of extending the default ignore list. This means +they have to hard-code checks to auto-ignore errors. + .. _options: Better Options Support ---------------------- +Currently there are some options handled by pep8 that are handled poorly. +Further, the way the options work is confusing to some, e.g., when specifying +``--ignore``, users do not expect it to override the ``DEFAULT_IGNORE`` list. +Users also don't expect ``--ignore`` and ``--select`` to step on each other's +toes. + .. _better-select-ignore: Support for Better Select/Ignore Handling +++++++++++++++++++++++++++++++++++++++++ +Currently ``--select`` and ``--ignore`` cause one or the other to be ignored. +Users presently cannot specify both for granularity. This should be +significantly improved. + +Further, new tools have developed ``--add-select`` and ``--add-ignore`` which +allows an add-only interface. This seems to be a good direction to follow. +Flake8 should support this. + +See https://github.com/PyCQA/pep8/issues/390 + .. _standard-in: Better stdin support From 00dfe8698bf4ecdb0f9814eae782b15f0b16694a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 6 Dec 2015 15:11:23 -0600 Subject: [PATCH 003/204] Wrap up design goals --- DESIGN.rst | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/DESIGN.rst b/DESIGN.rst index dee98ee..c5b5e82 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -17,6 +17,8 @@ Outline #. :ref:`plugin-default-ignore` + #. :ref:`report-generation` + #. :ref:`options` #. :ref:`better-select-ignore` @@ -25,10 +27,6 @@ Outline #. :ref:`multiprocessing` -#. :ref:`formatting` - -#. :ref:`report-generation` - .. _plugins: Better Plugins Support @@ -87,6 +85,14 @@ necessary. See https://gitlab.com/pycqa/flake8/issues/66 +.. _report-generation: + +Support for Report Generation ++++++++++++++++++++++++++++++ + +Flake8 should support pluggable report formats. See also pluggable report +formats for https://github.com/openstack/bandit + .. _options-passing: Support for Plugins Require Parsed Options @@ -139,18 +145,25 @@ See https://github.com/PyCQA/pep8/issues/390 Better stdin support -------------------- +Currently, flake8 accepts input from standard-in to check. It also currently +monkey-patches pep8 to cache that value. It would be better if there was one +way to retrieve the stdin input for plugins. Flake8 should provide this +directly instead of pep8 providing it. + +See +https://gitlab.com/pycqa/flake8/commit/41393c9b6de513ea169b61c175b71018e8a12336 + .. _multiprocessing: Multiprocessing Support ----------------------- -.. _formatting: +Flake8's existing multiprocessing support (and handling for different error +cases needs to persist through this redesign). -Better (Flake8-centralized) Formatting Support ----------------------------------------------- +See: -.. _report-generation: - - -Better (Flake8-centralized) Report Generation Support ------------------------------------------------------ +- https://gitlab.com/pycqa/flake8/issues/8 +- https://gitlab.com/pycqa/flake8/issues/17 +- https://gitlab.com/pycqa/flake8/issues/44 +- https://gitlab.com/pycqa/flake8/issues/74 From ae1ad0e2bf15a3043326c008a7a7b048050dec49 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 8 Dec 2015 15:41:49 -0600 Subject: [PATCH 004/204] Add design note that has been roaming my brain --- DESIGN.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DESIGN.rst b/DESIGN.rst index c5b5e82..7cdada3 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -72,6 +72,8 @@ errors so those plugins can auto-fix the problems in the file. See https://gitlab.com/pycqa/flake8/issues/84 +.. note:: Will probably need a Trie implementation for this + .. _reporter-plugins: Support for Plugins that Format Output From f013698072ccba9c2c188184840ce6bb13f9723f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 18 Dec 2015 13:49:59 -0600 Subject: [PATCH 005/204] Update with extra info about plugins --- DESIGN.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DESIGN.rst b/DESIGN.rst index 7cdada3..573e093 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -70,6 +70,9 @@ mccabe as "plugins", it will aggregate the errors returned by all of the plugins and be able to "notify" other plugins that have chosen to listen for errors so those plugins can auto-fix the problems in the file. +We should also be considerate of allowing these plugins to be composable. Each +plugin should have a way of defining its capabilities. + See https://gitlab.com/pycqa/flake8/issues/84 .. note:: Will probably need a Trie implementation for this @@ -95,6 +98,10 @@ Support for Report Generation Flake8 should support pluggable report formats. See also pluggable report formats for https://github.com/openstack/bandit +Report generation plugins may also choose to implement a way to store previous +runs of flake8. As such these plugins should be designed to be composable as +well. + .. _options-passing: Support for Plugins Require Parsed Options From 7b2a1c157b947150c19ba559d74b0904a5daa8f8 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 28 Dec 2015 23:32:42 -0600 Subject: [PATCH 006/204] Bare bones of a notification system --- .gitignore | 5 +++ example.py | 10 ++++++ flake8/__init__.py | 1 + flake8/_trie.py | 74 ++++++++++++++++++++++++++++++++++++++++++ flake8/notifier.py | 22 +++++++++++++ flake8/style_guide.py | 0 setup.py | 75 +++++++++++++++++++++++++++++++++++++++++++ tests/test_trie.py | 14 ++++++++ tox.ini | 8 +++++ 9 files changed, 209 insertions(+) create mode 100644 .gitignore create mode 100644 example.py create mode 100644 flake8/__init__.py create mode 100644 flake8/_trie.py create mode 100644 flake8/notifier.py create mode 100644 flake8/style_guide.py create mode 100644 setup.py create mode 100644 tests/test_trie.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a41da07 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.cache/* +.tox/* +*.pyc +*.sw* +*.egg-info diff --git a/example.py b/example.py new file mode 100644 index 0000000..a201782 --- /dev/null +++ b/example.py @@ -0,0 +1,10 @@ +from flake8 import _trie as trie + + +tree = trie.Trie() +for i in range(5): + tree.add('E103', 'E103-listener-{0}'.format(i)) + j = i + 1 + tree.add('E1{0}3'.format(j), 'E1{0}3-listener'.format(j)) +for i in range(10): + tree.add('W1{0:02d}'.format(i), 'W1{0:02d}-listener'.format(i)) diff --git a/flake8/__init__.py b/flake8/__init__.py new file mode 100644 index 0000000..6e648d3 --- /dev/null +++ b/flake8/__init__.py @@ -0,0 +1 @@ +__version__ = '3.0.0a1' diff --git a/flake8/_trie.py b/flake8/_trie.py new file mode 100644 index 0000000..ddd3c19 --- /dev/null +++ b/flake8/_trie.py @@ -0,0 +1,74 @@ +"""Independent implementation of a Trie tree.""" + +__all__ = ('Trie', 'TrieNode') + + +def _iterate_stringlike_objects(string): + for i in range(len(string)): + yield string[i:i+1] + + +class Trie(object): + """The object that manages the trie nodes.""" + + def __init__(self): + """Initialize an empty trie.""" + self.root = TrieNode(None, None) + + def add(self, path, node_data): + """Add the node data to the path described.""" + node = self.root + for prefix in _iterate_stringlike_objects(path): + child = node.find_prefix(prefix) + if child is None: + child = node.add_child(prefix, []) + node = child + node.data.append(node_data) + + def find(self, path): + """Find a node based on the path provided.""" + node = self.root + for prefix in _iterate_stringlike_objects(path): + child = node.find_prefix(prefix) + if child is None: + return None + node = child + return node + + +class TrieNode(object): + """The majority of the implementation details of a Trie.""" + + def __init__(self, prefix, data, children=None): + """Initialize a TrieNode with data and children.""" + self.children = children or {} + self.data = data + self.prefix = prefix + + def __repr__(self): + """Generate an easy to read representation of the node.""" + return 'TrieNode(prefix={0}, data={1}, children={2})'.format( + self.prefix, self.data, dict(self.children) + ) + + def find_prefix(self, prefix): + """Find the prefix in the children of this node. + + :returns: A child matching the prefix or None. + :rtype: :class:`~TrieNode` or None + """ + return self.children.get(prefix, None) + + def add_child(self, prefix, data, children=None): + """Create and add a new child node. + + :returns: The newly created node + :rtype: :class:`~TrieNode` + """ + new_node = TrieNode(prefix, data, children) + self.children[prefix] = new_node + return new_node + + def traverse(self): + """Traverse children of this node in order.""" + raise NotImplementedError() diff --git a/flake8/notifier.py b/flake8/notifier.py new file mode 100644 index 0000000..7b24b2f --- /dev/null +++ b/flake8/notifier.py @@ -0,0 +1,22 @@ +from flake8 import _trie + + +class Notifier(object): + def __init__(self): + self.listeners = _trie.Trie() + + def listeners_for(self, error_code): + node = self.listeners.find(error_code) + for listener in node.data: + yield listener + if node.children: + for child in node.traverse(): + for listener in child.data: + yield listener + + def notify(self, error_code, *args, **kwargs): + for listener in self.listeners_for(error_code): + listener.notify(*args, **kwargs) + + def register_listener(self, error_code, listener): + self.listeners.add(error_code, listener) diff --git a/flake8/style_guide.py b/flake8/style_guide.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fb0585e --- /dev/null +++ b/setup.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +from setuptools import setup +try: + # Work around a traceback with Nose on Python 2.6 + # http://bugs.python.org/issue15881#msg170215 + __import__('multiprocessing') +except ImportError: + pass + +try: + # Use https://docs.python.org/3/library/unittest.mock.html + from unittest import mock +except ImportError: + # < Python 3.3 + mock = None + + +tests_require = ['nose'] +if mock is None: + tests_require += ['mock'] + + +def get_version(fname='flake8/__init__.py'): + with open(fname) as f: + for line in f: + if line.startswith('__version__'): + return eval(line.split('=')[-1]) + + +def get_long_description(): + descr = [] + for fname in ('README.rst', 'CHANGES.rst'): + with open(fname) as f: + descr.append(f.read()) + return '\n\n'.join(descr) + + +setup( + name="flake8", + license="MIT", + version=get_version(), + description="the modular source code checker: pep8, pyflakes and co", + # long_description=get_long_description(), + author="Tarek Ziade", + author_email="tarek@ziade.org", + maintainer="Ian Cordasco", + maintainer_email="graffatcolmingov@gmail.com", + url="https://gitlab.com/pycqa/flake8", + packages=["flake8"], + install_requires=[ + "pyflakes >= 0.8.1, < 1.1", + "pep8 >= 1.5.7, != 1.6.0, != 1.6.1, != 1.6.2", + "mccabe >= 0.2.1, < 0.4", + ], + entry_points={ + 'distutils.commands': ['flake8 = flake8.main:Flake8Command'], + 'console_scripts': ['flake8 = flake8.main:main'], + 'flake8.extension': [ + 'F = flake8._pyflakes:FlakesChecker', + ], + }, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", + ], + tests_require=tests_require, + test_suite='nose.collector', +) diff --git a/tests/test_trie.py b/tests/test_trie.py new file mode 100644 index 0000000..10a10aa --- /dev/null +++ b/tests/test_trie.py @@ -0,0 +1,14 @@ +from flake8 import _trie as trie + + +def test_build_tree(): + tree = trie.Trie() + for i in range(5): + tree.add('E103', 'E103-listener-{0}'.format(i)) + j = i + 1 + tree.add('E1{0}3'.format(j), 'E1{0}3-listener'.format(j)) + for i in range(10): + tree.add('W1{0:02d}'.format(i), 'W1{0:02d}-listener'.format(i)) + + assert tree.find('E103') is not None + assert tree.find('E200') is None diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..de47800 --- /dev/null +++ b/tox.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py26,py27,py32,py33,py34,py35 + +[testenv] +deps = + pytest +commands = + py.test From 75f15340ac3b57bacab06a0dd47177c4eaa02d9b Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 29 Dec 2015 08:00:37 -0600 Subject: [PATCH 007/204] Remove toy file from git --- example.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 example.py diff --git a/example.py b/example.py deleted file mode 100644 index a201782..0000000 --- a/example.py +++ /dev/null @@ -1,10 +0,0 @@ -from flake8 import _trie as trie - - -tree = trie.Trie() -for i in range(5): - tree.add('E103', 'E103-listener-{0}'.format(i)) - j = i + 1 - tree.add('E1{0}3'.format(j), 'E1{0}3-listener'.format(j)) -for i in range(10): - tree.add('W1{0:02d}'.format(i), 'W1{0:02d}-listener'.format(i)) From 57e8583e2958c888d1d3c1df4bfb871e7956e308 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 29 Dec 2015 08:07:47 -0600 Subject: [PATCH 008/204] Add tree/sub-tree traversal to Trie Accurately retrieve listeners for a specific error code --- flake8/_trie.py | 31 +++++++++++++++++++++++++++---- flake8/notifier.py | 9 +++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/flake8/_trie.py b/flake8/_trie.py index ddd3c19..858c067 100644 --- a/flake8/_trie.py +++ b/flake8/_trie.py @@ -35,6 +35,16 @@ class Trie(object): node = child return node + def traverse(self): + """Traverse this tree. + + This performs a depth-first pre-order traversal of children in this + tree. It returns the results consistently by first sorting the + children based on their prefix and then traversing them in + alphabetical order. + """ + return self.root.traverse() + class TrieNode(object): """The majority of the implementation details of a Trie.""" @@ -47,8 +57,8 @@ class TrieNode(object): def __repr__(self): """Generate an easy to read representation of the node.""" - return 'TrieNode(prefix={0}, data={1}, children={2})'.format( - self.prefix, self.data, dict(self.children) + return 'TrieNode(prefix={0}, data={1})'.format( + self.prefix, self.data ) def find_prefix(self, prefix): @@ -70,5 +80,18 @@ class TrieNode(object): return new_node def traverse(self): - """Traverse children of this node in order.""" - raise NotImplementedError() + """Traverse children of this node. + + This performs a depth-first pre-order traversal of the remaining + children in this sub-tree. It returns the results consistently by + first sorting the children based on their prefix and then traversing + them in alphabetical order. + """ + if not self.children: + return + + for prefix in sorted(self.children.keys()): + child = self.children[prefix] + yield child + for child in child.traverse(): + yield child diff --git a/flake8/notifier.py b/flake8/notifier.py index 7b24b2f..625acd7 100644 --- a/flake8/notifier.py +++ b/flake8/notifier.py @@ -7,12 +7,13 @@ class Notifier(object): def listeners_for(self, error_code): node = self.listeners.find(error_code) + if node is None: + return for listener in node.data: yield listener - if node.children: - for child in node.traverse(): - for listener in child.data: - yield listener + for child in node.traverse(): + for listener in child.data: + yield listener def notify(self, error_code, *args, **kwargs): for listener in self.listeners_for(error_code): From d3a9f7d58c47a1fd22cb415d987ed1212db8b66f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 29 Dec 2015 08:12:54 -0600 Subject: [PATCH 009/204] Add docstrings to Notifier class --- flake8/notifier.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/flake8/notifier.py b/flake8/notifier.py index 625acd7..d366fbf 100644 --- a/flake8/notifier.py +++ b/flake8/notifier.py @@ -3,9 +3,31 @@ from flake8 import _trie class Notifier(object): def __init__(self): + """Initialize an empty notifier object.""" self.listeners = _trie.Trie() def listeners_for(self, error_code): + """Retrieve listeners for an error_code. + + The error code does not need to be a specific error code. For example, + There may be listeners registered for E100, E101, E110, E112, and + E126. If you wanted to get all listeners starting with 'E1' then you + would pass 'E1' as the error code here. + + Example usage + + .. code-block:: python + + from flake8 import notifier + + n = notifier.Notifier() + # register listeners + for listener in n.listeners_for('E1'): + listener.notify(...) + + for listener in n.listeners_for('W102'): + listener.notify(...) + """ node = self.listeners.find(error_code) if node is None: return @@ -16,8 +38,10 @@ class Notifier(object): yield listener def notify(self, error_code, *args, **kwargs): + """Notify all listeners for the specified error code.""" for listener in self.listeners_for(error_code): listener.notify(*args, **kwargs) def register_listener(self, error_code, listener): + """Register a listener for a specific error_code.""" self.listeners.add(error_code, listener) From d1d1d6003249c6f857ca34941728c65ba7d63359 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 29 Dec 2015 22:49:05 -0600 Subject: [PATCH 010/204] Add tests for the _trie module --- tests/test_trie.py | 127 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 117 insertions(+), 10 deletions(-) diff --git a/tests/test_trie.py b/tests/test_trie.py index 10a10aa..0a5a0a9 100644 --- a/tests/test_trie.py +++ b/tests/test_trie.py @@ -1,14 +1,121 @@ from flake8 import _trie as trie -def test_build_tree(): - tree = trie.Trie() - for i in range(5): - tree.add('E103', 'E103-listener-{0}'.format(i)) - j = i + 1 - tree.add('E1{0}3'.format(j), 'E1{0}3-listener'.format(j)) - for i in range(10): - tree.add('W1{0:02d}'.format(i), 'W1{0:02d}-listener'.format(i)) +class TestTrie(object): + """Collection of tests for the Trie class.""" - assert tree.find('E103') is not None - assert tree.find('E200') is None + def test_traverse_without_data(self): + """Verify the behaviour when traversing an empty Trie.""" + tree = trie.Trie() + assert list(tree.traverse()) == [] + + def test_traverse_with_data(self): + """Verify that traversal of our Trie is depth-first and pre-order.""" + tree = trie.Trie() + tree.add('A', 'A') + tree.add('a', 'a') + tree.add('AB', 'B') + tree.add('Ab', 'b') + tree.add('AbC', 'C') + tree.add('Abc', 'c') + # The trie tree here should look something like + # + # + # / \ + # A a + # / | + # B b + # / \ + # C c + # + # And the traversal should look like: + # + # A B b C c a + expected_order = ['A', 'B', 'b', 'C', 'c', 'a'] + for expected, actual_node in zip(expected_order, tree.traverse()): + assert actual_node.prefix == expected + + def test_find(self): + """Exercise the Trie.find method.""" + tree = trie.Trie() + tree.add('A', 'A') + tree.add('a', 'a') + tree.add('AB', 'AB') + tree.add('Ab', 'Ab') + tree.add('AbC', 'AbC') + tree.add('Abc', 'Abc') + + assert tree.find('AB').data == ['AB'] + assert tree.find('AbC').data == ['AbC'] + assert tree.find('A').data == ['A'] + assert tree.find('X') is None + + +class TestTrieNode(object): + """Collection of tests for the TrieNode class.""" + + def test_add_child(self): + """Verify we add children appropriately.""" + node = trie.TrieNode('E', 'E is for Eat') + assert node.find_prefix('a') is None + added = node.add_child('a', 'a is for Apple') + assert node.find_prefix('a') is added + + def test_add_child_overrides_previous_child(self): + """Verify adding a child will replace the previous child.""" + node = trie.TrieNode('E', 'E is for Eat', children={ + 'a': trie.TrieNode('a', 'a is for Apple') + }) + previous = node.find_prefix('a') + assert previous is not None + + added = node.add_child('a', 'a is for Ascertain') + assert node.find_prefix('a') is added + + def test_find_prefix(self): + """Verify we can find a child with the specified prefix.""" + node = trie.TrieNode('E', 'E is for Eat', children={ + 'a': trie.TrieNode('a', 'a is for Apple') + }) + child = node.find_prefix('a') + assert child is not None + assert child.prefix == 'a' + assert child.data == 'a is for Apple' + + def test_find_prefix_returns_None_when_no_children_have_the_prefix(self): + """Verify we receive None from find_prefix for missing children.""" + node = trie.TrieNode('E', 'E is for Eat', children={ + 'a': trie.TrieNode('a', 'a is for Apple') + }) + assert node.find_prefix('b') is None + + def test_traverse_does_nothing_when_a_node_has_no_children(self): + """Verify traversing a node with no children does nothing.""" + node = trie.TrieNode('E', 'E is for Eat') + assert list(node.traverse()) == [] + + def test_traverse(self): + """Verify traversal is depth-first and pre-order.""" + root = trie.TrieNode(None, None) + node = root.add_child('A', 'A') + root.add_child('a', 'a') + node.add_child('B', 'B') + node = node.add_child('b', 'b') + node.add_child('C', 'C') + node.add_child('c', 'c') + # The sub-tree here should look something like + # + # + # / \ + # A a + # / | + # B b + # / \ + # C c + # + # And the traversal should look like: + # + # A B b C c a + expected_order = ['A', 'B', 'b', 'C', 'c', 'a'] + for expected, actual_node in zip(expected_order, root.traverse()): + assert actual_node.prefix == expected From 222be9ac4918d2ea85e9ac3d20832106c10d30a1 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 29 Dec 2015 23:04:36 -0600 Subject: [PATCH 011/204] Start adding tests for Notifier class --- flake8/notifier.py | 2 +- tests/test_notifier.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/test_notifier.py diff --git a/flake8/notifier.py b/flake8/notifier.py index d366fbf..5a75f87 100644 --- a/flake8/notifier.py +++ b/flake8/notifier.py @@ -40,7 +40,7 @@ class Notifier(object): def notify(self, error_code, *args, **kwargs): """Notify all listeners for the specified error code.""" for listener in self.listeners_for(error_code): - listener.notify(*args, **kwargs) + listener.notify(error_code, *args, **kwargs) def register_listener(self, error_code, listener): """Register a listener for a specific error_code.""" diff --git a/tests/test_notifier.py b/tests/test_notifier.py new file mode 100644 index 0000000..44f2dc9 --- /dev/null +++ b/tests/test_notifier.py @@ -0,0 +1,31 @@ +import pytest + +from flake8 import notifier + +class _Listener(object): + def __init__(self, error_code): + self.error_code = error_code + self.was_notified = False + + def notify(self, error_code, *args, **kwargs): + assert self.error_code == error_code + self.was_notified = True + + +class TestNotifier(object): + @pytest.fixture(autouse=True) + def setup(self): + self.notifier = notifier.Notifier() + self.listener_map = {} + + for i in range(10): + for j in range(30): + error_code = 'E{0}{1:02d}'.format(i, j) + listener = _Listener(error_code) + self.listener_map[error_code] = listener + self.notifier.register_listener(error_code, listener) + + def test_notify_a_single_error_code(self): + """Show that we notify a specific error code.""" + self.notifier.notify('E111', 'extra', 'args') + assert self.listener_map['E111'].was_notified is True From 37b92cd4b47723115f9e27760307b10508808bc6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 29 Dec 2015 23:28:20 -0600 Subject: [PATCH 012/204] Fix logic for Notifier.listeners_for Add tests for proper logic around notifier --- DESIGN.rst | 12 ++++++++++++ flake8/notifier.py | 25 ++++++++++--------------- tests/test_notifier.py | 20 ++++++++++++-------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/DESIGN.rst b/DESIGN.rst index 573e093..e472671 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -77,6 +77,18 @@ See https://gitlab.com/pycqa/flake8/issues/84 .. note:: Will probably need a Trie implementation for this +What we *might* want is for a autofix plugin to register something like + +:: + + 'flake8.autofix_extension': [ + 'E1 = my_fixer.E1Listener', + 'E2 = my_fixer.E2Listener', + ] + +This means that the notifer would need to take an error code like ``E111`` and +then notify anything listening for ``E111``, ``E11``, ``E1``, and ``E``. + .. _reporter-plugins: Support for Plugins that Format Output diff --git a/flake8/notifier.py b/flake8/notifier.py index 5a75f87..6bf51ef 100644 --- a/flake8/notifier.py +++ b/flake8/notifier.py @@ -9,12 +9,12 @@ class Notifier(object): def listeners_for(self, error_code): """Retrieve listeners for an error_code. - The error code does not need to be a specific error code. For example, - There may be listeners registered for E100, E101, E110, E112, and - E126. If you wanted to get all listeners starting with 'E1' then you - would pass 'E1' as the error code here. + There may be listeners registered for E1, E100, E101, E110, E112, and + E126. To get all the listeners for one of E100, E101, E110, E112, or + E126 you would also need to incorporate the listeners for E1 (since + they're all in the same class). - Example usage + Example usage: .. code-block:: python @@ -22,20 +22,15 @@ class Notifier(object): n = notifier.Notifier() # register listeners - for listener in n.listeners_for('E1'): - listener.notify(...) - for listener in n.listeners_for('W102'): listener.notify(...) """ - node = self.listeners.find(error_code) - if node is None: - return - for listener in node.data: - yield listener - for child in node.traverse(): - for listener in child.data: + path = error_code + while path: + node = self.listeners.find(path) + for listener in node.data: yield listener + path = path[:-1] def notify(self, error_code, *args, **kwargs): """Notify all listeners for the specified error code.""" diff --git a/tests/test_notifier.py b/tests/test_notifier.py index 44f2dc9..b0365f3 100644 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -8,7 +8,7 @@ class _Listener(object): self.was_notified = False def notify(self, error_code, *args, **kwargs): - assert self.error_code == error_code + assert error_code.startswith(self.error_code) self.was_notified = True @@ -18,14 +18,18 @@ class TestNotifier(object): self.notifier = notifier.Notifier() self.listener_map = {} - for i in range(10): - for j in range(30): - error_code = 'E{0}{1:02d}'.format(i, j) - listener = _Listener(error_code) - self.listener_map[error_code] = listener - self.notifier.register_listener(error_code, listener) + def add_listener(error_code): + listener = _Listener(error_code) + self.listener_map[error_code] = listener + self.notifier.register_listener(error_code, listener) - def test_notify_a_single_error_code(self): + for i in range(10): + add_listener('E{0}'.format(i)) + for j in range(30): + add_listener('E{0}{1:02d}'.format(i, j)) + + def test_notify(self): """Show that we notify a specific error code.""" self.notifier.notify('E111', 'extra', 'args') assert self.listener_map['E111'].was_notified is True + assert self.listener_map['E1'].was_notified is True From f539464368bd06b16a624b785ab97bb055a573aa Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 30 Dec 2015 17:26:28 -0600 Subject: [PATCH 013/204] Start working on our option handling story --- flake8/option_parser.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 flake8/option_parser.py diff --git a/flake8/option_parser.py b/flake8/option_parser.py new file mode 100644 index 0000000..9d006ba --- /dev/null +++ b/flake8/option_parser.py @@ -0,0 +1,32 @@ +import optparse + + +class Option(object): + def __init__(self, short_option_name=None, long_option_name=None, + # Options below here are taken from the optparse.Option class + action=None, default=None, type=None, dest=None, + nargs=None, const=None, choices=None, callback=None, + callback_args=None, callback_kwargs=None, help=None, + metavar=None, + # Options below here are specific to Flake8 + parse_from_config=False + ): + self.short_option_name = short_option_name + self.long_option_name = long_option_name + self.option_args = filter(None, (short_option_name, long_option_name)) + self.option_kwargs = { + 'action': action, + 'default': default, + 'type': type, + 'dest': dest, + 'callback': callback, + 'callback_args': callback_args, + 'callback_kwargs': callback_kwargs, + 'help': help, + 'metavar': metavar, + } + for key, value in self.option_kwargs.items(): + setattr(self, key, value) + self.parse_from_config = parse_from_config + self._option = optparse.Option(*self.option_args, + **self.option_kwargs) From a6d790cc10ff5bce7d3d506b52e4d4df0b73009b Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 4 Jan 2016 19:34:10 -0600 Subject: [PATCH 014/204] Start adding logging for --verbose --- flake8/__init__.py | 11 +++++++++++ flake8/option_parser.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/flake8/__init__.py b/flake8/__init__.py index 6e648d3..49cbabd 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -1 +1,12 @@ +import logging + +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + __version__ = '3.0.0a1' diff --git a/flake8/option_parser.py b/flake8/option_parser.py index 9d006ba..0343158 100644 --- a/flake8/option_parser.py +++ b/flake8/option_parser.py @@ -1,4 +1,16 @@ +import logging import optparse +import os.path +import sys + +if sys.version_info < (3, 0): + import ConfigParser as configparser +else: + import configparser + +from . import utils + +LOG = logging.getLogger(__name__) class Option(object): From 7ec5a2ada3df656bb319f88ebce0c7574567c07c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 4 Jan 2016 19:34:41 -0600 Subject: [PATCH 015/204] Sketch out config parsing and option managing --- flake8/option_parser.py | 208 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 205 insertions(+), 3 deletions(-) diff --git a/flake8/option_parser.py b/flake8/option_parser.py index 0343158..5b369c0 100644 --- a/flake8/option_parser.py +++ b/flake8/option_parser.py @@ -21,8 +21,30 @@ class Option(object): callback_args=None, callback_kwargs=None, help=None, metavar=None, # Options below here are specific to Flake8 - parse_from_config=False + parse_from_config=False, comma_separated_list=False, ): + """Initialize an Option instance wrapping optparse.Option. + + :param str short_option_name: + The short name of the option (e.g., ``-x``). This will be the + first argument passed to :class:`~optparse.Option`. + :param str long_option_name: + The long name of the option (e.g., ``--xtra-long-option``). This + will be the second argument passed to :class:`~optparse.Option`. + :param str action: + Any action allowed by :mod:`optparse`. + :param default: + Default value of the option. + :param type: + Any type allowed by :mod:`optparse`. + :param dest: + Attribute name to store parsed option value as. + :param nargs: + Number of arguments to parse for this option. + :param const: + Constant value to store on a common destination. + + """ self.short_option_name = short_option_name self.long_option_name = long_option_name self.option_args = filter(None, (short_option_name, long_option_name)) @@ -40,5 +62,185 @@ class Option(object): for key, value in self.option_kwargs.items(): setattr(self, key, value) self.parse_from_config = parse_from_config - self._option = optparse.Option(*self.option_args, - **self.option_kwargs) + self.comma_separated_list = comma_separated_list + + if parse_from_config: + if not long_option_name: + raise ValueError('When specifying parse_from_config=True, ' + 'a long_option_name must also be specified.') + self.config_name = long_option_name.strip('-').replace('-', '_') + + def to_optparse(self): + """Convert a Flake8 Option to an optparse Option.""" + return optparse.Option(*self.option_args, + **self.option_kwargs) + + +class OptionManager(object): + def __init__(self, prog=None, version=None, + usage='%prog [options] input'): + self.parser = optparse.OptionParser(prog=prog, version=version, + usage=usage) + self.config_options_dict = {} + self.options = [] + self.program_name = prog + self.version = version + + def add_option(self, *args, **kwargs): + option = Option(*args, **kwargs) + self.parser.add_option(option.to_optparse()) + self.options.append(option) + if option.parse_from_config: + self.config_options_dict[option.config_name] = option + + def parse_args(self, args=None, values=None): + return self.parser.parse_args(args, values) + + +class ConfigFileFinder(object): + PROJECT_FILENAMES = ('setup.cfg', 'tox.ini') + + def __init__(self, program_name, args): + # Platform specific settings + self.is_windows = sys.platform == 'win32' + self.xdg_home = os.environ.get('XDG_CONFIG_HOME', + os.path.expanduser('~/.config')) + + # Look for '.' files + self.program_config = '.' + program_name + self.program_name = program_name + + # List of filenames to find in the local/project directory + self.project_filenames = ('setup.cfg', 'tox.ini', self.program_config) + # List of filenames to find "globally" + self.global_filenames = (self.program_config,) + + self.local_directory = os.curdir + + if not args: + args = ['.'] + self.parent = self.tail = os.path.abspath(os.path.commonprefix(args)) + + @staticmethod + def _read_config(files): + config = configparser.RawConfigParser() + found_files = config.read(files) + return (config, found_files) + + def generate_possible_local_config_files(self): + tail = self.tail + parent = self.parent + while tail: + for project_filename in self.project_filenames: + filename = os.path.abspath(os.path.join(parent, + project_filename)) + yield filename + (parent, tail) = os.path.split(parent) + + def local_config_files(self): + return [ + filename + for filename in self.generate_possible_local_config_files() + if os.path.exists(filename) + ] + + def local_configs(self): + config, found_files = self._read_config(self.local_config_files()) + if found_files: + LOG.debug('Found local configuration files: %s', found_files) + return config + + def user_config_file(self): + if self.is_windows: + return os.path.expanduser('~\\' + self.program_config) + return os.path.join(self.xdg_home, self.program_name) + + def user_config(self): + config, found_files = self._read_config(self.user_config_file()) + if found_files: + LOG.debug('Found user configuration files: %s', found_files) + return config + + +class MergedConfigParser(object): + GETINT_METHODS = set(['int', 'count']) + GETBOOL_METHODS = set(['store_true', 'store_false']) + + def __init__(self, option_manager, args=None): + self.option_manager = option_manager + self.program_name = option_manager.program_name + self.args = args + self.config_finder = ConfigFileFinder(self.program_name, self.args) + self.config_options = option_manager.config_options_dict + + def _parse_config(self, config_parser): + config = self.config_finder.local_configs() + if not config.has_section(self.program_name): + LOG.debug('Local configuration files have no %s section', + self.program_name) + return {} + + config_dict = {} + for option_name in config_parser.options(self.program_name): + if option_name not in self.config_options: + LOG.debug('Option "%s" is not registered. Ignoring.', + option_name) + continue + option = self.config_options[option_name] + + # Use the appropriate method to parse the config value + method = config.get + if option.type in self.GETINT_METHODS: + method = config.getint + elif option.action in self.GETBOOL_METHODS: + method = config.getboolean + + value = method(self.program_name, option_name) + LOG.debug('Option "%s" returned value: %r', option_name, value) + + final_value = value + if option.comma_separated_list: + final_value = utils.parse_comma_separated_list(value) + + config_dict[option_name] = final_value + + def is_configured_by(self, config): + """Check if the specified config parser has an appropriate section.""" + return config.has_section(self.program_name) + + def parse_local_config(self): + """Parse and return the local configuration files.""" + config = self.config_finder.local_configs() + if not self.is_configured_by(config): + LOG.debug('Local configuration files have no %s section', + self.program_name) + return + + LOG.debug('Parsing local configuration files.') + return self._parse_config(config) + + def parse_user_config(self): + """Parse and return the user configuration files.""" + config = self.config_finder.user_config() + if not self.is_configured_by(config): + LOG.debug('User configuration files have no %s section', + self.program_name) + return + + LOG.debug('Parsing user configuration files.') + return self._parse_config(config) + + def parse(self): + """Parse and return the local and user config files. + + First this copies over the parsed local configuration and then + iterates over the options in the user configuration and sets them if + they were not set by the local configuration file. + """ + user_config = self.parse_user_config() + config = self.parse_local_config() + + for option, value in user_config.items(): + config.setdefault(option, value) + + return config From 07a29e45db86ed4b6eb95134363b88acd93f6384 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 4 Jan 2016 23:24:11 -0600 Subject: [PATCH 016/204] Split flake8.option_parser into submodule --- flake8/options/__init__.py | 0 .../{option_parser.py => options/config.py} | 85 ------------- flake8/options/manager.py | 119 ++++++++++++++++++ 3 files changed, 119 insertions(+), 85 deletions(-) create mode 100644 flake8/options/__init__.py rename flake8/{option_parser.py => options/config.py} (62%) create mode 100644 flake8/options/manager.py diff --git a/flake8/options/__init__.py b/flake8/options/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flake8/option_parser.py b/flake8/options/config.py similarity index 62% rename from flake8/option_parser.py rename to flake8/options/config.py index 5b369c0..708cd90 100644 --- a/flake8/option_parser.py +++ b/flake8/options/config.py @@ -1,5 +1,4 @@ import logging -import optparse import os.path import sys @@ -13,90 +12,6 @@ from . import utils LOG = logging.getLogger(__name__) -class Option(object): - def __init__(self, short_option_name=None, long_option_name=None, - # Options below here are taken from the optparse.Option class - action=None, default=None, type=None, dest=None, - nargs=None, const=None, choices=None, callback=None, - callback_args=None, callback_kwargs=None, help=None, - metavar=None, - # Options below here are specific to Flake8 - parse_from_config=False, comma_separated_list=False, - ): - """Initialize an Option instance wrapping optparse.Option. - - :param str short_option_name: - The short name of the option (e.g., ``-x``). This will be the - first argument passed to :class:`~optparse.Option`. - :param str long_option_name: - The long name of the option (e.g., ``--xtra-long-option``). This - will be the second argument passed to :class:`~optparse.Option`. - :param str action: - Any action allowed by :mod:`optparse`. - :param default: - Default value of the option. - :param type: - Any type allowed by :mod:`optparse`. - :param dest: - Attribute name to store parsed option value as. - :param nargs: - Number of arguments to parse for this option. - :param const: - Constant value to store on a common destination. - - """ - self.short_option_name = short_option_name - self.long_option_name = long_option_name - self.option_args = filter(None, (short_option_name, long_option_name)) - self.option_kwargs = { - 'action': action, - 'default': default, - 'type': type, - 'dest': dest, - 'callback': callback, - 'callback_args': callback_args, - 'callback_kwargs': callback_kwargs, - 'help': help, - 'metavar': metavar, - } - for key, value in self.option_kwargs.items(): - setattr(self, key, value) - self.parse_from_config = parse_from_config - self.comma_separated_list = comma_separated_list - - if parse_from_config: - if not long_option_name: - raise ValueError('When specifying parse_from_config=True, ' - 'a long_option_name must also be specified.') - self.config_name = long_option_name.strip('-').replace('-', '_') - - def to_optparse(self): - """Convert a Flake8 Option to an optparse Option.""" - return optparse.Option(*self.option_args, - **self.option_kwargs) - - -class OptionManager(object): - def __init__(self, prog=None, version=None, - usage='%prog [options] input'): - self.parser = optparse.OptionParser(prog=prog, version=version, - usage=usage) - self.config_options_dict = {} - self.options = [] - self.program_name = prog - self.version = version - - def add_option(self, *args, **kwargs): - option = Option(*args, **kwargs) - self.parser.add_option(option.to_optparse()) - self.options.append(option) - if option.parse_from_config: - self.config_options_dict[option.config_name] = option - - def parse_args(self, args=None, values=None): - return self.parser.parse_args(args, values) - - class ConfigFileFinder(object): PROJECT_FILENAMES = ('setup.cfg', 'tox.ini') diff --git a/flake8/options/manager.py b/flake8/options/manager.py new file mode 100644 index 0000000..4b32234 --- /dev/null +++ b/flake8/options/manager.py @@ -0,0 +1,119 @@ +import logging +import optparse + +LOG = logging.getLogger(__name__) + + +class Option(object): + def __init__(self, short_option_name=None, long_option_name=None, + # Options below here are taken from the optparse.Option class + action=None, default=None, type=None, dest=None, + nargs=None, const=None, choices=None, callback=None, + callback_args=None, callback_kwargs=None, help=None, + metavar=None, + # Options below here are specific to Flake8 + parse_from_config=False, comma_separated_list=False, + ): + """Initialize an Option instance wrapping optparse.Option. + + The following are all passed directly through to optparse. + + :param str short_option_name: + The short name of the option (e.g., ``-x``). This will be the + first argument passed to :class:`~optparse.Option`. + :param str long_option_name: + The long name of the option (e.g., ``--xtra-long-option``). This + will be the second argument passed to :class:`~optparse.Option`. + :param str action: + Any action allowed by :mod:`optparse`. + :param default: + Default value of the option. + :param type: + Any type allowed by :mod:`optparse`. + :param dest: + Attribute name to store parsed option value as. + :param nargs: + Number of arguments to parse for this option. + :param const: + Constant value to store on a common destination. Usually used in + conjuntion with ``action="store_const"``. + :param iterable choices: + Possible values for the option. + :param callable callback: + Callback used if the action is ``"callback"``. + :param iterable callback_args: + Additional positional arguments to the callback callable. + :param dictionary callback_kwargs: + Keyword arguments to the callback callable. + :param str help: + Help text displayed in the usage information. + :param str metavar: + Name to use instead of the long option name for help text. + + The following parameters are for Flake8's option handling alone. + + :param bool parse_from_config: + Whether or not this option should be parsed out of config files. + :param bool comma_separated_list: + Whether the option is a comma separated list when parsing from a + config file. + """ + self.short_option_name = short_option_name + self.long_option_name = long_option_name + self.option_args = filter(None, (short_option_name, long_option_name)) + self.option_kwargs = { + 'action': action, + 'default': default, + 'type': type, + 'dest': dest, + 'callback': callback, + 'callback_args': callback_args, + 'callback_kwargs': callback_kwargs, + 'help': help, + 'metavar': metavar, + } + for key, value in self.option_kwargs.items(): + setattr(self, key, value) + self.parse_from_config = parse_from_config + self.comma_separated_list = comma_separated_list + + if parse_from_config: + if not long_option_name: + raise ValueError('When specifying parse_from_config=True, ' + 'a long_option_name must also be specified.') + self.config_name = long_option_name.strip('-').replace('-', '_') + + def __repr__(self): + return ( + 'Option({0}, {1}, action={action}, default={default}, ' + 'dest={dest}, type={type}, callback={callback}, help={help},' + ' callback={callback}, callback_args={callback_args}, ' + 'callback_kwargs={callback_kwargs}, metavar={metavar})' + ).format(*self.option_args, **self.option_kwargs) + + def to_optparse(self): + """Convert a Flake8 Option to an optparse Option.""" + return optparse.Option(*self.option_args, + **self.option_kwargs) + + +class OptionManager(object): + def __init__(self, prog=None, version=None, + usage='%prog [options] input'): + self.parser = optparse.OptionParser(prog=prog, version=version, + usage=usage) + self.config_options_dict = {} + self.options = [] + self.program_name = prog + self.version = version + + def add_option(self, *args, **kwargs): + option = Option(*args, **kwargs) + self.parser.add_option(option.to_optparse()) + self.options.append(option) + if option.parse_from_config: + self.config_options_dict[option.config_name] = option + LOG.debug('Registered option "%s".', option) + + def parse_args(self, args=None, values=None): + return self.parser.parse_args(args, values) From 67aff6d2ec0dbb558bcbee98e0c4a932d87f739c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 7 Jan 2016 09:37:34 -0600 Subject: [PATCH 017/204] Updates to config/options aggregation - Adds another flag to the Option class - Adds normalization to the config parsing - Removes dead code from _parse_config - Handles user specifying --append-config, --config, and --isolated but does not handle their mutual exclusion --- flake8/options/aggregator.py | 51 +++++++++++++++++++++ flake8/options/config.py | 88 ++++++++++++++++++++++++++++-------- flake8/options/manager.py | 27 +++++++++++ 3 files changed, 148 insertions(+), 18 deletions(-) create mode 100644 flake8/options/aggregator.py diff --git a/flake8/options/aggregator.py b/flake8/options/aggregator.py new file mode 100644 index 0000000..863fa05 --- /dev/null +++ b/flake8/options/aggregator.py @@ -0,0 +1,51 @@ +"""Aggregation function for CLI specified options and config file options. + +This holds the logic that uses the collected and merged config files and +applies the user-specified command-line configuration on top of it. +""" +import logging + +from flake8.options import config +from flake8 import utils + +LOG = logging.getLogger(__name__) + + +def aggregate_options(manager, arglist=None, values=None): + """Function that aggregates the CLI and Config options.""" + # Get defaults from the option parser + default_values, _ = manager.parse_args([], values=values) + # Get original CLI values so we can find additional config file paths and + # see if --config was specified. + original_values, original_args = manager.parse_args() + extra_config_files = utils.normalize_paths(original_values.append_config) + + # Make our new configuration file mergerator + config_parser = config.MergedConfigParser( + option_manager=manager, + extra_config_files=extra_config_files, + args=original_args, + ) + + # Get the parsed config + parsed_config = config_parser.parse(original_values.config, + original_values.isolated) + + # Merge values parsed from config onto the default values returned + for config_name, value in parsed_config.items(): + dest_name = config_name + # If the config name is somehow different from the destination name, + # fetch the destination name from our Option + if not hasattr(default_values, config_name): + dest_name = config_parser.config_options[config_name].dest + + LOG.debug('Overriding default value of (%s) for "%s" with (%s)', + getattr(default_values, dest_name, None), + dest_name, + value) + # Override the default values with the config values + setattr(default_values, dest_name, value) + + # Finally parse the command-line options + final_values, args = manager.parse_args(arglist, default_values) + return final_values, args diff --git a/flake8/options/config.py b/flake8/options/config.py index 708cd90..9317e7a 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -11,11 +11,16 @@ from . import utils LOG = logging.getLogger(__name__) +__all__('ConfigFileFinder', 'MergedConfigParser') + class ConfigFileFinder(object): PROJECT_FILENAMES = ('setup.cfg', 'tox.ini') - def __init__(self, program_name, args): + def __init__(self, program_name, args, extra_config_files): + # The values of --append-config from the CLI + self.extra_config_files = extra_config_files + # Platform specific settings self.is_windows = sys.platform == 'win32' self.xdg_home = os.environ.get('XDG_CONFIG_HOME', @@ -27,8 +32,6 @@ class ConfigFileFinder(object): # List of filenames to find in the local/project directory self.project_filenames = ('setup.cfg', 'tox.ini', self.program_config) - # List of filenames to find "globally" - self.global_filenames = (self.program_config,) self.local_directory = os.curdir @@ -42,6 +45,12 @@ class ConfigFileFinder(object): found_files = config.read(files) return (config, found_files) + def cli_config(self, files): + config, found_files = self._read_config(files) + if found_files: + LOG.debug('Found cli configuration files: %s', found_files) + return config + def generate_possible_local_config_files(self): tail = self.tail parent = self.parent @@ -57,7 +66,7 @@ class ConfigFileFinder(object): filename for filename in self.generate_possible_local_config_files() if os.path.exists(filename) - ] + ] + self.extra_config_files def local_configs(self): config, found_files = self._read_config(self.local_config_files()) @@ -81,20 +90,33 @@ class MergedConfigParser(object): GETINT_METHODS = set(['int', 'count']) GETBOOL_METHODS = set(['store_true', 'store_false']) - def __init__(self, option_manager, args=None): + def __init__(self, option_manager, extra_config_files=None, args=None): + # Our instance of flake8.options.manager.OptionManager self.option_manager = option_manager + # The prog value for the cli parser self.program_name = option_manager.program_name + # Parsed extra arguments self.args = args + # Our instance of our ConfigFileFinder self.config_finder = ConfigFileFinder(self.program_name, self.args) + # Mapping of configuration option names to + # flake8.options.manager.Option instances self.config_options = option_manager.config_options_dict + self.extra_config_files = extra_config_files or [] + + @staticmethod + def _normalize_value(option, value): + if option.normalize_paths: + final_value = utils.normalize_paths(value) + elif option.comma_separated_list: + final_value = utils.parse_comma_separated_list(value) + else: + final_value = value + LOG.debug('%r has been normalized to %r for option "%s"', + value, final_value, option.config_name) + return final_value def _parse_config(self, config_parser): - config = self.config_finder.local_configs() - if not config.has_section(self.program_name): - LOG.debug('Local configuration files have no %s section', - self.program_name) - return {} - config_dict = {} for option_name in config_parser.options(self.program_name): if option_name not in self.config_options: @@ -113,10 +135,7 @@ class MergedConfigParser(object): value = method(self.program_name, option_name) LOG.debug('Option "%s" returned value: %r', option_name, value) - final_value = value - if option.comma_separated_list: - final_value = utils.parse_comma_separated_list(value) - + final_value = self._normalize_value(value) config_dict[option_name] = final_value def is_configured_by(self, config): @@ -129,7 +148,7 @@ class MergedConfigParser(object): if not self.is_configured_by(config): LOG.debug('Local configuration files have no %s section', self.program_name) - return + return {} LOG.debug('Parsing local configuration files.') return self._parse_config(config) @@ -140,18 +159,51 @@ class MergedConfigParser(object): if not self.is_configured_by(config): LOG.debug('User configuration files have no %s section', self.program_name) - return + return {} LOG.debug('Parsing user configuration files.') return self._parse_config(config) - def parse(self): + def parse_cli_config(self, config_path): + """Parse and return the file specified by --config.""" + config = self.config_finder.cli_config(config_path) + if not self.is_configured_by(config): + LOG.debug('CLI configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing CLI configuration files.') + return self._parse_config(config) + + def parse(self, cli_config=None, isolated=False): """Parse and return the local and user config files. First this copies over the parsed local configuration and then iterates over the options in the user configuration and sets them if they were not set by the local configuration file. + + :param str cli_config: + Value of --config when specified at the command-line. Overrides + all other config files. + :param bool isolated: + Determines if we should parse configuration files at all or not. + If running in isolated mode, we ignore all configuration files + :returns: + Dictionary of parsed configuration options + :rtype: + dict """ + if isolated: + LOG.debug('Refusing to parse configuration files due to user-' + 'requested isolation') + return {} + + if cli_config: + LOG.debug('Ignoring user and locally found configuration files. ' + 'Reading only configuration from "%s" specified via ' + '--config by the user', cli_config) + return self.parse_cli_config(cli_config) + user_config = self.parse_user_config() config = self.parse_local_config() diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 4b32234..034444b 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -5,6 +5,7 @@ LOG = logging.getLogger(__name__) class Option(object): + """Our wrapper around an optparse.Option object to add features.""" def __init__(self, short_option_name=None, long_option_name=None, # Options below here are taken from the optparse.Option class action=None, default=None, type=None, dest=None, @@ -13,6 +14,7 @@ class Option(object): metavar=None, # Options below here are specific to Flake8 parse_from_config=False, comma_separated_list=False, + normalize_paths=False, ): """Initialize an Option instance wrapping optparse.Option. @@ -57,6 +59,9 @@ class Option(object): :param bool comma_separated_list: Whether the option is a comma separated list when parsing from a config file. + :param bool normalize_paths: + Whether the option is expecting a path or list of paths and should + attempt to normalize the paths to absolute paths. """ self.short_option_name = short_option_name self.long_option_name = long_option_name @@ -72,11 +77,16 @@ class Option(object): 'help': help, 'metavar': metavar, } + # Set attributes for our option arguments for key, value in self.option_kwargs.items(): setattr(self, key, value) + + # Set our custom attributes self.parse_from_config = parse_from_config self.comma_separated_list = comma_separated_list + self.normalize_paths = normalize_paths + self.config_name = None if parse_from_config: if not long_option_name: raise ValueError('When specifying parse_from_config=True, ' @@ -108,6 +118,16 @@ class OptionManager(object): self.version = version def add_option(self, *args, **kwargs): + """Create and register a new option. + + See parameters for :class:`~flake8.options.manager.Option` for + acceptable arguments to this method. + + .. note:: + + ``short_option_name`` and ``long_option_name`` may be specified + positionally as they are with optparse normally. + """ option = Option(*args, **kwargs) self.parser.add_option(option.to_optparse()) self.options.append(option) @@ -116,4 +136,11 @@ class OptionManager(object): LOG.debug('Registered option "%s".', option) def parse_args(self, args=None, values=None): + """Simple proxy to calling the OptionParser's parse_args method. + + .. todo:: + + Normalize values based on our extra attributes from + :class:`~flake8.options.manager.OptionManager`. + """ return self.parser.parse_args(args, values) From 93369e112f290e233c030ba9d0b36360a65649db Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 12:40:26 -0600 Subject: [PATCH 018/204] Add empty module to start testing options submodule --- flake8/utils.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 flake8/utils.py diff --git a/flake8/utils.py b/flake8/utils.py new file mode 100644 index 0000000..e69de29 From 1e9878611a93b3af583e222a15c0b4575fc4f027 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 12:55:31 -0600 Subject: [PATCH 019/204] Move unit tests into tests/unit --- tests/unit/__init__.py | 0 tests/{ => unit}/test_notifier.py | 0 tests/unit/test_option.py | 52 +++++++++++++++++++++++++++++++ tests/{ => unit}/test_trie.py | 0 4 files changed, 52 insertions(+) create mode 100644 tests/unit/__init__.py rename tests/{ => unit}/test_notifier.py (100%) create mode 100644 tests/unit/test_option.py rename tests/{ => unit}/test_trie.py (100%) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_notifier.py b/tests/unit/test_notifier.py similarity index 100% rename from tests/test_notifier.py rename to tests/unit/test_notifier.py diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py new file mode 100644 index 0000000..d1491eb --- /dev/null +++ b/tests/unit/test_option.py @@ -0,0 +1,52 @@ +"""Unit tests for flake8.options.manager.Option.""" +import mock + +from flake8.options import manager + + +def test_to_optparse(): + """Test conversion to an optparse.Option class.""" + opt = manager.Option( + short_option_name='-t', + long_option_name='--test', + action='count', + parse_from_config=True, + normalize_paths=True, + ) + assert opt.normalize_paths is True + assert opt.parse_from_config is True + + optparse_opt = opt.to_optparse() + assert not hasattr(optparse_opt, 'parse_from_config') + assert not hasattr(optparse_opt, 'normalize_paths') + assert optparse_opt.action == 'count' + + +@mock.patch('optparse.Option') +def test_to_optparse_creates_an_option_as_we_expect(Option): + """Show that we pass all keyword args to optparse.Option.""" + opt = manager.Option('-t', '--test', action='count') + opt.to_optparse() + option_kwargs = { + 'action': 'count', + 'default': None, + 'type': None, + 'dest': None, + 'callback': None, + 'callback_args': None, + 'callback_kwargs': None, + 'help': None, + 'metavar': None, + } + + Option.assert_called_once_with( + '-t', '--test', **option_kwargs + ) + + +def test_config_name_generation(): + """Show that we generate the config name deterministically.""" + opt = manager.Option(long_option_name='--some-very-long-option-name', + parse_from_config=True) + + assert opt.config_name == 'some_very_long_option_name' diff --git a/tests/test_trie.py b/tests/unit/test_trie.py similarity index 100% rename from tests/test_trie.py rename to tests/unit/test_trie.py From 19e5ea767d8a087fb63e0236252c9c316ff66904 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 12:55:43 -0600 Subject: [PATCH 020/204] Add the options submodule to the package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fb0585e..d0650f4 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( maintainer="Ian Cordasco", maintainer_email="graffatcolmingov@gmail.com", url="https://gitlab.com/pycqa/flake8", - packages=["flake8"], + packages=["flake8", "flake8.options"], install_requires=[ "pyflakes >= 0.8.1, < 1.1", "pep8 >= 1.5.7, != 1.6.0, != 1.6.1, != 1.6.2", From 3a8f99b4580fea358517e749bbba6599c40ba9e4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 12:56:03 -0600 Subject: [PATCH 021/204] Add mock to list of test dependencies --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index de47800..ecaae55 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = py26,py27,py32,py33,py34,py35 [testenv] deps = + mock pytest commands = py.test From 7361603d7a5b41b4e878ba1c2dad0890fb888d68 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 12:56:16 -0600 Subject: [PATCH 022/204] Flesh out OptionManager#parse_args --- flake8/options/manager.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 034444b..baccd95 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -1,6 +1,8 @@ import logging import optparse +from flake8 import utils + LOG = logging.getLogger(__name__) @@ -136,11 +138,16 @@ class OptionManager(object): LOG.debug('Registered option "%s".', option) def parse_args(self, args=None, values=None): - """Simple proxy to calling the OptionParser's parse_args method. + """Simple proxy to calling the OptionParser's parse_args method.""" + options, xargs = self.parser.parse_args(args, values) + for config_name, option in self.config_options_dict.items(): + dest = option.dest or config_name + if self.normalize_paths: + old_value = getattr(options, dest) + setattr(options, dest, utils.normalize_paths(old_value)) + elif self.comma_separated_list: + old_value = getattr(options, dest) + setattr(options, dest, + utils.parse_comma_separated_list(old_value)) - .. todo:: - - Normalize values based on our extra attributes from - :class:`~flake8.options.manager.OptionManager`. - """ - return self.parser.parse_args(args, values) + return options, xargs From a7ce4f00ec1f002780184c724fdce3b6fdc74ce3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 13:53:59 -0600 Subject: [PATCH 023/204] Start testing the OptionManager --- flake8/options/manager.py | 8 ++++-- tests/unit/test_option_manager.py | 46 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_option_manager.py diff --git a/flake8/options/manager.py b/flake8/options/manager.py index baccd95..a471960 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -105,8 +105,10 @@ class Option(object): def to_optparse(self): """Convert a Flake8 Option to an optparse Option.""" - return optparse.Option(*self.option_args, - **self.option_kwargs) + if not hasattr(self, '_opt'): + self._opt = optparse.Option(*self.option_args, + **self.option_kwargs) + return self._opt class OptionManager(object): @@ -130,6 +132,8 @@ class OptionManager(object): ``short_option_name`` and ``long_option_name`` may be specified positionally as they are with optparse normally. """ + if len(args) == 1 and args[0].startswith('--'): + args = (None, args[0]) option = Option(*args, **kwargs) self.parser.add_option(option.to_optparse()) self.options.append(option) diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py new file mode 100644 index 0000000..27c47d6 --- /dev/null +++ b/tests/unit/test_option_manager.py @@ -0,0 +1,46 @@ +"""Unit tests for flake.options.manager.OptionManager.""" +import optparse + +import pytest + +from flake8.options import manager + + +@pytest.fixture +def optmanager(): + return manager.OptionManager(prog='flake8', version='3.0.0b1') + + +def test_option_manager_creates_option_parser(optmanager): + """Verify that a new manager creates a new parser.""" + assert optmanager.parser is not None + assert isinstance(optmanager.parser, optparse.OptionParser) is True + + +def test_add_option_short_option_only(optmanager): + """Verify the behaviour of adding a short-option only.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('-s', help='Test short opt') + assert optmanager.options[0].short_option_name == '-s' + + +def test_add_option_long_option_only(optmanager): + """Verify the behaviour of adding a long-option only.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--long', help='Test long opt') + assert optmanager.options[0].short_option_name is None + assert optmanager.options[0].long_option_name == '--long' + + +def test_add_short_and_long_option_names(optmanager): + """Verify the behaviour of using both short and long option names.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('-b', '--both', help='Test both opts') + assert optmanager.options[0].short_option_name == '-b' + assert optmanager.options[0].long_option_name == '--both' From 5a6656d78de3a2f9051fc053bb6294971abfcdd1 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 15:22:03 -0600 Subject: [PATCH 024/204] Test parse_from_config behaviour of Options --- tests/unit/test_option.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py index d1491eb..51b5dcf 100644 --- a/tests/unit/test_option.py +++ b/tests/unit/test_option.py @@ -1,5 +1,6 @@ """Unit tests for flake8.options.manager.Option.""" import mock +import pytest from flake8.options import manager @@ -50,3 +51,9 @@ def test_config_name_generation(): parse_from_config=True) assert opt.config_name == 'some_very_long_option_name' + + +def test_config_name_needs_long_option_name(): + """Show that we error out if the Option should be parsed from config.""" + with pytest.raises(ValueError): + manager.Option('-s', parse_from_config=True) From 09bc9e2720546fb5bbbc0015148e8e108ec88d24 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 15:22:22 -0600 Subject: [PATCH 025/204] Add test for custom arguments to add_option --- tests/unit/test_option_manager.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index 27c47d6..4983c1d 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -44,3 +44,16 @@ def test_add_short_and_long_option_names(optmanager): optmanager.add_option('-b', '--both', help='Test both opts') assert optmanager.options[0].short_option_name == '-b' assert optmanager.options[0].long_option_name == '--both' + + +def test_add_option_with_custom_args(optmanager): + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--parse', parse_from_config=True) + optmanager.add_option('--commas', comma_separated_list=True) + optmanager.add_option('--files', normalize_paths=True) + + attrs = ['parse_from_config', 'comma_separated_list', 'normalize_paths'] + for option, attr in zip(optmanager.options, attrs): + assert getattr(option, attr) is True From 36cc2545426d788c93443c24265b7b51cb8494fd Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 15:32:44 -0600 Subject: [PATCH 026/204] Generate dest for Options always --- flake8/options/manager.py | 9 +++++++-- tests/unit/test_option.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index a471960..f4bd1d8 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -72,7 +72,7 @@ class Option(object): 'action': action, 'default': default, 'type': type, - 'dest': dest, + 'dest': self._make_dest(dest), 'callback': callback, 'callback_args': callback_args, 'callback_kwargs': callback_kwargs, @@ -93,7 +93,7 @@ class Option(object): if not long_option_name: raise ValueError('When specifying parse_from_config=True, ' 'a long_option_name must also be specified.') - self.config_name = long_option_name.strip('-').replace('-', '_') + self.config_name = self.dest def __repr__(self): return ( @@ -103,6 +103,11 @@ class Option(object): 'callback_kwargs={callback_kwargs}, metavar={metavar})' ).format(*self.option_args, **self.option_kwargs) + def _make_dest(self, dest): + if self.long_option_name: + return self.long_option_name[2:].replace('-', '_') + return self.short_option_name[1] + def to_optparse(self): """Convert a Flake8 Option to an optparse Option.""" if not hasattr(self, '_opt'): diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py index 51b5dcf..c6fc98c 100644 --- a/tests/unit/test_option.py +++ b/tests/unit/test_option.py @@ -32,7 +32,7 @@ def test_to_optparse_creates_an_option_as_we_expect(Option): 'action': 'count', 'default': None, 'type': None, - 'dest': None, + 'dest': 'test', 'callback': None, 'callback_args': None, 'callback_kwargs': None, From ad0200e792361d2909955135bca49117ab7ad4e0 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 15:33:01 -0600 Subject: [PATCH 027/204] Add failing test for OptionManager.parse_args --- flake8/options/manager.py | 8 ++++---- tests/unit/test_option_manager.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index f4bd1d8..6c0d036 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -149,12 +149,12 @@ class OptionManager(object): def parse_args(self, args=None, values=None): """Simple proxy to calling the OptionParser's parse_args method.""" options, xargs = self.parser.parse_args(args, values) - for config_name, option in self.config_options_dict.items(): - dest = option.dest or config_name - if self.normalize_paths: + for option in self.options: + dest = option.dest + if option.normalize_paths: old_value = getattr(options, dest) setattr(options, dest, utils.normalize_paths(old_value)) - elif self.comma_separated_list: + elif option.comma_separated_list: old_value = getattr(options, dest) setattr(options, dest, utils.parse_comma_separated_list(old_value)) diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index 4983c1d..37c9570 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -1,5 +1,6 @@ """Unit tests for flake.options.manager.OptionManager.""" import optparse +import os import pytest @@ -57,3 +58,19 @@ def test_add_option_with_custom_args(optmanager): attrs = ['parse_from_config', 'comma_separated_list', 'normalize_paths'] for option, attr in zip(optmanager.options, attrs): assert getattr(option, attr) is True + + +def test_parse_args(optmanager): + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('-v', '--verbose', action='count') + optmanager.add_option('--config', normalize_paths=True) + optmanager.add_option('--exclude', default='E123,W234', + comma_separated_list=True) + + options, args = optmanager.parse_args( + ['-v', '-v', '-v', '--config', '../config.ini'] + ) + assert options.verbose == 3 + assert options.config == os.path.abspath('../config.ini') From a4042d6d692354f486d9f3e3f810d08ff3d7339e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 15:58:26 -0600 Subject: [PATCH 028/204] Start fleshing out flake8.utils Add parse_comma_separated_list, normalize_path, and normalize_paths and add logic in OptionManager.parse_args to use the right normalize_path* function based on comma_separated_list value of the option. --- flake8/options/manager.py | 25 +++++++++++++------- flake8/utils.py | 50 +++++++++++++++++++++++++++++++++++++++ tests/unit/test_utils.py | 14 +++++++++++ 3 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_utils.py diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 6c0d036..3e4c128 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -150,13 +150,22 @@ class OptionManager(object): """Simple proxy to calling the OptionParser's parse_args method.""" options, xargs = self.parser.parse_args(args, values) for option in self.options: - dest = option.dest - if option.normalize_paths: - old_value = getattr(options, dest) - setattr(options, dest, utils.normalize_paths(old_value)) - elif option.comma_separated_list: - old_value = getattr(options, dest) - setattr(options, dest, - utils.parse_comma_separated_list(old_value)) + _normalize_option(options, option) return options, xargs + + +def _normalize_option(options, option): + dest = option.dest + if option.normalize_paths: + old_value = getattr(options, dest) + # Decide whether to parse a list of paths or a single path + normalize = utils.normalize_path + if option.comma_separated_list: + normalize = utils.normalize_paths + setattr(options, dest, normalize(old_value)) + + elif option.comma_separated_list: + old_value = getattr(options, dest) + setattr(options, dest, + utils.parse_comma_separated_list(old_value)) diff --git a/flake8/utils.py b/flake8/utils.py index e69de29..72c7852 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -0,0 +1,50 @@ +"""Utility methods for flake8.""" +import os + + +def parse_comma_separated_list(value): + """Parse a comma-separated list. + + :param value: + String or list of strings to be parsed and normalized. + :returns: + List of values with whitespace stripped. + :rtype: + list + """ + if not value: + return [] + + if not isinstance(value, (list, tuple)): + value = value.split(',') + + return [item.strip() for item in value] + + +def normalize_paths(paths, parent=os.curdir): + """Parse a comma-separated list of paths. + + :returns: + The normalized paths. + :rtype: + [str] + """ + paths = [] + for path in parse_comma_separated_list(paths): + if '/' in path: + path = os.path.abspath(os.path.join(parent, path)) + paths.append(path.rstrip('/')) + return paths + + +def normalize_path(path, parent=os.curdir): + """Normalize a single-path. + + :returns: + The normalized path. + :rtype: + str + """ + if '/' in path: + path = os.path.abspath(os.path.join(parent, path)) + return path diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..28c47c8 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,14 @@ +"""Tests for flake8's utils module.""" +import pytest + +from flake8 import utils + + +@pytest.mark.parametrize("value,expected", [ + ("E123,\n\tW234,\n E206", ["E123", "W234", "E206"]), + ("E123,W234,E206", ["E123", "W234", "E206"]), + (["E123", "W234", "E206"], ["E123", "W234", "E206"]), + (["E123", "\n\tW234", "\n E206"], ["E123", "W234", "E206"]), +]) +def test_parse_comma_separated_list(value, expected): + assert utils.parse_comma_separated_list(value) == expected From a7f7bbbeb2002ba0f581d6f77bca664c2057a4a6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 20:29:09 -0600 Subject: [PATCH 029/204] Handle when dest is provided to Option --- flake8/options/manager.py | 5 ++++- tests/unit/test_option.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 3e4c128..38eb605 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -93,7 +93,7 @@ class Option(object): if not long_option_name: raise ValueError('When specifying parse_from_config=True, ' 'a long_option_name must also be specified.') - self.config_name = self.dest + self.config_name = long_option_name[2:].replace('-', '_') def __repr__(self): return ( @@ -104,6 +104,9 @@ class Option(object): ).format(*self.option_args, **self.option_kwargs) def _make_dest(self, dest): + if dest: + return dest + if self.long_option_name: return self.long_option_name[2:].replace('-', '_') return self.short_option_name[1] diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py index c6fc98c..55cae5c 100644 --- a/tests/unit/test_option.py +++ b/tests/unit/test_option.py @@ -57,3 +57,8 @@ def test_config_name_needs_long_option_name(): """Show that we error out if the Option should be parsed from config.""" with pytest.raises(ValueError): manager.Option('-s', parse_from_config=True) + + +def test_dest_is_not_overridden(): + opt = manager.Option('-s', '--short', dest='something_not_short') + assert opt.dest == 'something_not_short' From 13ca3dfe37885cb3cf384a9fcc10fe16866cc223 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 20:30:28 -0600 Subject: [PATCH 030/204] Show that even defaults are properly normalized --- tests/unit/test_option_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index 37c9570..cc23669 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -60,7 +60,7 @@ def test_add_option_with_custom_args(optmanager): assert getattr(option, attr) is True -def test_parse_args(optmanager): +def test_parse_args_normalize_path(optmanager): assert optmanager.options == [] assert optmanager.config_options_dict == {} @@ -74,3 +74,4 @@ def test_parse_args(optmanager): ) assert options.verbose == 3 assert options.config == os.path.abspath('../config.ini') + assert options.exclude == ['E123', 'W234'] From ec26d46dffac5ace86222e99b465a8d1d73dbc8e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:04:11 -0600 Subject: [PATCH 031/204] Add tests for normalize_path and normalize_paths --- tests/unit/test_utils.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 28c47c8..b0e243c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,9 +1,14 @@ """Tests for flake8's utils module.""" +import os + import pytest from flake8 import utils +RELATIVE_PATHS = ["flake8", "pep8", "pyflakes", "mccabe"] + + @pytest.mark.parametrize("value,expected", [ ("E123,\n\tW234,\n E206", ["E123", "W234", "E206"]), ("E123,W234,E206", ["E123", "W234", "E206"]), @@ -11,4 +16,26 @@ from flake8 import utils (["E123", "\n\tW234", "\n E206"], ["E123", "W234", "E206"]), ]) def test_parse_comma_separated_list(value, expected): + """Verify that similar inputs produce identical outputs.""" assert utils.parse_comma_separated_list(value) == expected + + +@pytest.mark.parametrize("value,expected", [ + ("flake8", "flake8"), + ("../flake8", os.path.abspath("../flake8")), + ("flake8/", os.path.abspath("flake8")), +]) +def test_normalize_path(value, expected): + """Verify that we normalize paths provided to the tool.""" + assert utils.normalize_path(value) == expected + + +@pytest.mark.parametrize("value,expected", [ + ("flake8,pep8,pyflakes,mccabe", ["flake8", "pep8", "pyflakes", "mccabe"]), + ("flake8,\n\tpep8,\n pyflakes,\n\n mccabe", + ["flake8", "pep8", "pyflakes", "mccabe"]), + ("../flake8,../pep8,../pyflakes,../mccabe", + [os.path.abspath("../" + p) for p in RELATIVE_PATHS]), +]) +def test_normalize_paths(value, expected): + assert utils.normalize_paths(value) == expected From 237a35fe8841475b944374b45039f040d75ade09 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:14:36 -0600 Subject: [PATCH 032/204] Simplify normalize_paths --- flake8/options/manager.py | 1 - flake8/utils.py | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 38eb605..723a700 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -167,7 +167,6 @@ def _normalize_option(options, option): if option.comma_separated_list: normalize = utils.normalize_paths setattr(options, dest, normalize(old_value)) - elif option.comma_separated_list: old_value = getattr(options, dest) setattr(options, dest, diff --git a/flake8/utils.py b/flake8/utils.py index 72c7852..0dc16e2 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -29,12 +29,7 @@ def normalize_paths(paths, parent=os.curdir): :rtype: [str] """ - paths = [] - for path in parse_comma_separated_list(paths): - if '/' in path: - path = os.path.abspath(os.path.join(parent, path)) - paths.append(path.rstrip('/')) - return paths + return [normalize_path(p) for p in parse_comma_separated_list(paths)] def normalize_path(path, parent=os.curdir): @@ -47,4 +42,4 @@ def normalize_path(path, parent=os.curdir): """ if '/' in path: path = os.path.abspath(os.path.join(parent, path)) - return path + return path.rstrip('/') From cc3a9210fd558340d3d50192682bc3df1cbd40a7 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:15:03 -0600 Subject: [PATCH 033/204] Add more tests around parse_args --- tests/unit/test_option_manager.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index cc23669..9b1dbba 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -48,6 +48,7 @@ def test_add_short_and_long_option_names(optmanager): def test_add_option_with_custom_args(optmanager): + """Verify that add_option handles custom Flake8 parameters.""" assert optmanager.options == [] assert optmanager.config_options_dict == {} @@ -61,17 +62,35 @@ def test_add_option_with_custom_args(optmanager): def test_parse_args_normalize_path(optmanager): + """Show that parse_args handles path normalization.""" assert optmanager.options == [] assert optmanager.config_options_dict == {} - optmanager.add_option('-v', '--verbose', action='count') optmanager.add_option('--config', normalize_paths=True) + + options, args = optmanager.parse_args(['--config', '../config.ini']) + assert options.config == os.path.abspath('../config.ini') + + +def test_parse_args_handles_comma_separated_defaults(optmanager): + """Show that parse_args handles defaults that are comma separated.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + optmanager.add_option('--exclude', default='E123,W234', comma_separated_list=True) - options, args = optmanager.parse_args( - ['-v', '-v', '-v', '--config', '../config.ini'] - ) - assert options.verbose == 3 - assert options.config == os.path.abspath('../config.ini') + options, args = optmanager.parse_args([]) assert options.exclude == ['E123', 'W234'] + + +def test_parse_args_handles_comma_separated_lists(optmanager): + """Show that parse_args handles user-specified comma separated lists.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--exclude', default='E123,W234', + comma_separated_list=True) + + options, args = optmanager.parse_args(['--exclude', 'E201,W111,F280']) + assert options.exclude == ['E201', 'W111', 'F280'] From 684f19f5e3f7f62e58fe87adfaf6c44dc50ff43f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:17:22 -0600 Subject: [PATCH 034/204] Re-factor normalization in parse_args --- flake8/options/manager.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 723a700..353ab55 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -111,6 +111,20 @@ class Option(object): return self.long_option_name[2:].replace('-', '_') return self.short_option_name[1] + def normalize(self, options): + dest = self.dest + if self.normalize_paths: + old_value = getattr(options, dest) + # Decide whether to parse a list of paths or a single path + normalize = utils.normalize_path + if self.comma_separated_list: + normalize = utils.normalize_paths + setattr(options, dest, normalize(old_value)) + elif self.comma_separated_list: + old_value = getattr(options, dest) + setattr(options, dest, + utils.parse_comma_separated_list(old_value)) + def to_optparse(self): """Convert a Flake8 Option to an optparse Option.""" if not hasattr(self, '_opt'): @@ -153,21 +167,6 @@ class OptionManager(object): """Simple proxy to calling the OptionParser's parse_args method.""" options, xargs = self.parser.parse_args(args, values) for option in self.options: - _normalize_option(options, option) + option.normalize(options) return options, xargs - - -def _normalize_option(options, option): - dest = option.dest - if option.normalize_paths: - old_value = getattr(options, dest) - # Decide whether to parse a list of paths or a single path - normalize = utils.normalize_path - if option.comma_separated_list: - normalize = utils.normalize_paths - setattr(options, dest, normalize(old_value)) - elif option.comma_separated_list: - old_value = getattr(options, dest) - setattr(options, dest, - utils.parse_comma_separated_list(old_value)) From 9fe9ef06f7d00a9082aa5a43e80b97e8a5f07362 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:20:20 -0600 Subject: [PATCH 035/204] Add test for comma-separated paths normalization --- tests/unit/test_option_manager.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index 9b1dbba..a9e641d 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -73,7 +73,7 @@ def test_parse_args_normalize_path(optmanager): def test_parse_args_handles_comma_separated_defaults(optmanager): - """Show that parse_args handles defaults that are comma separated.""" + """Show that parse_args handles defaults that are comma-separated.""" assert optmanager.options == [] assert optmanager.config_options_dict == {} @@ -85,7 +85,7 @@ def test_parse_args_handles_comma_separated_defaults(optmanager): def test_parse_args_handles_comma_separated_lists(optmanager): - """Show that parse_args handles user-specified comma separated lists.""" + """Show that parse_args handles user-specified comma-separated lists.""" assert optmanager.options == [] assert optmanager.config_options_dict == {} @@ -94,3 +94,21 @@ def test_parse_args_handles_comma_separated_lists(optmanager): options, args = optmanager.parse_args(['--exclude', 'E201,W111,F280']) assert options.exclude == ['E201', 'W111', 'F280'] + + +def test_parse_args_normalize_paths(optmanager): + """Verify parse_args normalizes a comma-separated list of paths.""" + assert optmanager.options == [] + assert optmanager.config_options_dict == {} + + optmanager.add_option('--extra-config', normalize_paths=True, + comma_separated_list=True) + + options, args = optmanager.parse_args([ + '--extra-config', '../config.ini,tox.ini,flake8/some-other.cfg' + ]) + assert options.extra_config == [ + os.path.abspath('../config.ini'), + 'tox.ini', + os.path.abspath('flake8/some-other.cfg'), + ] From e076fec666b087c86d392545e949ec9aea62fe4c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:28:49 -0600 Subject: [PATCH 036/204] Add some docstrings to manager module --- flake8/options/manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 353ab55..e249ffb 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -1,3 +1,4 @@ +"""Option handling and Option management logic.""" import logging import optparse @@ -8,6 +9,7 @@ LOG = logging.getLogger(__name__) class Option(object): """Our wrapper around an optparse.Option object to add features.""" + def __init__(self, short_option_name=None, long_option_name=None, # Options below here are taken from the optparse.Option class action=None, default=None, type=None, dest=None, @@ -96,6 +98,7 @@ class Option(object): self.config_name = long_option_name[2:].replace('-', '_') def __repr__(self): + """Simple representation of an Option class.""" return ( 'Option({0}, {1}, action={action}, default={default}, ' 'dest={dest}, type={type}, callback={callback}, help={help},' @@ -134,8 +137,19 @@ class Option(object): class OptionManager(object): + """Manage Options and OptionParser while adding post-processing.""" + def __init__(self, prog=None, version=None, usage='%prog [options] input'): + """Initialize an instance of an OptionManager. + + :param str prog: + Name of the actual program (e.g., flake8). + :param str version: + Version string for the program. + :param str usage: + Basic usage string used by the OptionParser. + """ self.parser = optparse.OptionParser(prog=prog, version=version, usage=usage) self.config_options_dict = {} From b4b59a96279da897d2a0fc545682051d46c0dc43 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 21:29:08 -0600 Subject: [PATCH 037/204] Make Option#normalize more reusable --- flake8/options/manager.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index e249ffb..fafc1e9 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -114,19 +114,17 @@ class Option(object): return self.long_option_name[2:].replace('-', '_') return self.short_option_name[1] - def normalize(self, options): - dest = self.dest + def normalize(self, value): + """Normalize the value based on the option configuration.""" if self.normalize_paths: - old_value = getattr(options, dest) # Decide whether to parse a list of paths or a single path normalize = utils.normalize_path if self.comma_separated_list: normalize = utils.normalize_paths - setattr(options, dest, normalize(old_value)) + return normalize(value) elif self.comma_separated_list: - old_value = getattr(options, dest) - setattr(options, dest, - utils.parse_comma_separated_list(old_value)) + return utils.parse_comma_separated_list(value) + return value def to_optparse(self): """Convert a Flake8 Option to an optparse Option.""" @@ -181,6 +179,7 @@ class OptionManager(object): """Simple proxy to calling the OptionParser's parse_args method.""" options, xargs = self.parser.parse_args(args, values) for option in self.options: - option.normalize(options) + old_value = getattr(options, option.dest) + setattr(options, option.dest, option.normalize(old_value)) return options, xargs From 30f4ccda0e6d82a526fee883cc86aaefb5dd2b10 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 22:34:13 -0600 Subject: [PATCH 038/204] Add docstrings to config module --- flake8/options/config.py | 51 ++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/flake8/options/config.py b/flake8/options/config.py index 9317e7a..666f9d9 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -1,3 +1,4 @@ +"""Config handling logic for Flake8.""" import logging import os.path import sys @@ -7,17 +8,26 @@ if sys.version_info < (3, 0): else: import configparser -from . import utils - LOG = logging.getLogger(__name__) -__all__('ConfigFileFinder', 'MergedConfigParser') +__all__ = ('ConfigFileFinder', 'MergedConfigParser') class ConfigFileFinder(object): + """Encapsulate the logic for finding and reading config files.""" + PROJECT_FILENAMES = ('setup.cfg', 'tox.ini') def __init__(self, program_name, args, extra_config_files): + """Initialize object to find config files. + + :param str program_name: + Name of the current program (e.g., flake8). + :param list args: + The extra arguments passed on the command-line. + :param list extra_config_files: + Extra configuration files specified by the user to read. + """ # The values of --append-config from the CLI self.extra_config_files = extra_config_files @@ -46,12 +56,14 @@ class ConfigFileFinder(object): return (config, found_files) def cli_config(self, files): + """Read and parse the config file specified on the command-line.""" config, found_files = self._read_config(files) if found_files: LOG.debug('Found cli configuration files: %s', found_files) return config def generate_possible_local_config_files(self): + """Find and generate all local config files.""" tail = self.tail parent = self.parent while tail: @@ -62,6 +74,7 @@ class ConfigFileFinder(object): (parent, tail) = os.path.split(parent) def local_config_files(self): + """Find all local config files.""" return [ filename for filename in self.generate_possible_local_config_files() @@ -69,17 +82,20 @@ class ConfigFileFinder(object): ] + self.extra_config_files def local_configs(self): + """Parse all local config files into one config object.""" config, found_files = self._read_config(self.local_config_files()) if found_files: LOG.debug('Found local configuration files: %s', found_files) return config def user_config_file(self): + """Find the user-level config file.""" if self.is_windows: return os.path.expanduser('~\\' + self.program_config) return os.path.join(self.xdg_home, self.program_name) def user_config(self): + """Parse the user config file into a config object.""" config, found_files = self._read_config(self.user_config_file()) if found_files: LOG.debug('Found user configuration files: %s', found_files) @@ -87,10 +103,26 @@ class ConfigFileFinder(object): class MergedConfigParser(object): + """Encapsulate merging different types of configuration files. + + This parses out the options registered that were specified in the + configuration files, handles extra configuration files, and returns + dictionaries with the parsed values. + """ + GETINT_METHODS = set(['int', 'count']) GETBOOL_METHODS = set(['store_true', 'store_false']) def __init__(self, option_manager, extra_config_files=None, args=None): + """Initialize the MergedConfigParser instance. + + :param flake8.option.manager.OptionManager option_manager: + Initialized OptionManager. + :param list extra_config_files: + List of extra config files to parse. + :params list args: + The extra parsed arguments from the command-line. + """ # Our instance of flake8.options.manager.OptionManager self.option_manager = option_manager # The prog value for the cli parser @@ -106,12 +138,7 @@ class MergedConfigParser(object): @staticmethod def _normalize_value(option, value): - if option.normalize_paths: - final_value = utils.normalize_paths(value) - elif option.comma_separated_list: - final_value = utils.parse_comma_separated_list(value) - else: - final_value = value + final_value = option.normalize(value) LOG.debug('%r has been normalized to %r for option "%s"', value, final_value, option.config_name) return final_value @@ -126,11 +153,11 @@ class MergedConfigParser(object): option = self.config_options[option_name] # Use the appropriate method to parse the config value - method = config.get + method = config_parser.get if option.type in self.GETINT_METHODS: - method = config.getint + method = config_parser.getint elif option.action in self.GETBOOL_METHODS: - method = config.getboolean + method = config_parser.getboolean value = method(self.program_name, option_name) LOG.debug('Option "%s" returned value: %r', option_name, value) From 8dd160c98efc8fcfef74514fabc506042f6881b2 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 22:34:39 -0600 Subject: [PATCH 039/204] Start writing tests for the config finder --- tests/unit/test_config_file_finder.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/unit/test_config_file_finder.py diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py new file mode 100644 index 0000000..2df7694 --- /dev/null +++ b/tests/unit/test_config_file_finder.py @@ -0,0 +1,11 @@ +"""Tests for the ConfigFileFinder.""" +import os + +from flake8.options import config + + +def test_uses_default_args(): + """Show that we default the args value.""" + finder = config.ConfigFileFinder('flake8', None, []) + assert finder.args == ['.'] + assert finder.parent == os.path.abspath('.') From dd46f02b5863fb0850d3421b8cd698d1d3ecdfba Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 9 Jan 2016 23:02:20 -0600 Subject: [PATCH 040/204] Add config file fixtures Add more ConfigFileFinder tests --- tests/fixtures/config_files/cli-specified.ini | 9 +++++++++ tests/unit/test_config_file_finder.py | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/config_files/cli-specified.ini diff --git a/tests/fixtures/config_files/cli-specified.ini b/tests/fixtures/config_files/cli-specified.ini new file mode 100644 index 0000000..753604a --- /dev/null +++ b/tests/fixtures/config_files/cli-specified.ini @@ -0,0 +1,9 @@ +[flake8] +ignore = + E123, + W234, + E111 +exclude = + foo/, + bar/, + bogus/ diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 2df7694..8ed940d 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -1,5 +1,8 @@ """Tests for the ConfigFileFinder.""" import os +import sys + +import mock from flake8.options import config @@ -7,5 +10,19 @@ from flake8.options import config def test_uses_default_args(): """Show that we default the args value.""" finder = config.ConfigFileFinder('flake8', None, []) - assert finder.args == ['.'] assert finder.parent == os.path.abspath('.') + + +@mock.patch.object(sys, 'platform', 'win32') +def test_windows_detection(): + """Verify we detect Windows to the best of our knowledge.""" + finder = config.ConfigFileFinder('flake8', None, []) + assert finder.is_windows is True + + +def test_cli_config(): + cli_filepath = 'tests/fixtures/config_files/cli-specified.ini' + finder = config.ConfigFileFinder('flake8', None, []) + + parsed_config = finder.cli_config(cli_filepath) + assert parsed_config.has_section('flake8') From 949d3e48fe094d8c5629f7e9f15e1b6503ee9d49 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 08:40:35 -0600 Subject: [PATCH 041/204] Add tests for generate_possible_local_config_files Fix a bug found via testing where we traverse all the way to the root directory looking for config files. --- flake8/options/config.py | 5 ++++- tests/unit/test_config_file_finder.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/flake8/options/config.py b/flake8/options/config.py index 666f9d9..e82cb62 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -43,7 +43,7 @@ class ConfigFileFinder(object): # List of filenames to find in the local/project directory self.project_filenames = ('setup.cfg', 'tox.ini', self.program_config) - self.local_directory = os.curdir + self.local_directory = os.path.abspath(os.curdir) if not args: args = ['.'] @@ -66,11 +66,14 @@ class ConfigFileFinder(object): """Find and generate all local config files.""" tail = self.tail parent = self.parent + local_dir = self.local_directory while tail: for project_filename in self.project_filenames: filename = os.path.abspath(os.path.join(parent, project_filename)) yield filename + if parent == local_dir: + break (parent, tail) = os.path.split(parent) def local_config_files(self): diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 8ed940d..8904316 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -3,6 +3,7 @@ import os import sys import mock +import pytest from flake8.options import config @@ -21,8 +22,30 @@ def test_windows_detection(): def test_cli_config(): + """Verify opening and reading the file specified via the cli.""" cli_filepath = 'tests/fixtures/config_files/cli-specified.ini' finder = config.ConfigFileFinder('flake8', None, []) parsed_config = finder.cli_config(cli_filepath) assert parsed_config.has_section('flake8') + + +@pytest.mark.parametrize('args,expected', [ + ([], # No arguments + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath('.flake8')]), + (['flake8/options', 'flake8/'], # Common prefix of "flake8/" + [os.path.abspath('flake8/setup.cfg'), + os.path.abspath('flake8/tox.ini'), + os.path.abspath('flake8/.flake8'), + os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath('.flake8')]), +]) +def test_generate_possible_local_config_files(args, expected): + """Verify generation of all possible config paths.""" + finder = config.ConfigFileFinder('flake8', args, []) + + assert (list(finder.generate_possible_local_config_files()) == + expected) From cb276d63e37b16a07c6f5e4897bd6c3bc8add445 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:04:39 -0600 Subject: [PATCH 042/204] Parametrize windows detection unit test --- tests/unit/test_config_file_finder.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 8904316..3a8f8f2 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -14,11 +14,16 @@ def test_uses_default_args(): assert finder.parent == os.path.abspath('.') -@mock.patch.object(sys, 'platform', 'win32') -def test_windows_detection(): +@pytest.mark.parametrize('platform,is_windows', [ + ('win32', True), + ('linux', False), + ('darwin', False), +]) +def test_windows_detection(platform, is_windows): """Verify we detect Windows to the best of our knowledge.""" - finder = config.ConfigFileFinder('flake8', None, []) - assert finder.is_windows is True + with mock.patch.object(sys, 'platform', platform): + finder = config.ConfigFileFinder('flake8', None, []) + assert finder.is_windows is is_windows def test_cli_config(): From 204a367095a6d63bcf6dcc690cd51d2db5fe8ff9 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:05:09 -0600 Subject: [PATCH 043/204] Add more parameters to file location generation test --- tests/unit/test_config_file_finder.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 3a8f8f2..e723397 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -36,17 +36,30 @@ def test_cli_config(): @pytest.mark.parametrize('args,expected', [ - ([], # No arguments + # No arguments, common prefix of abspath('.') + ([], [os.path.abspath('setup.cfg'), os.path.abspath('tox.ini'), os.path.abspath('.flake8')]), - (['flake8/options', 'flake8/'], # Common prefix of "flake8/" + # Common prefix of "flake8/" + (['flake8/options', 'flake8/'], [os.path.abspath('flake8/setup.cfg'), os.path.abspath('flake8/tox.ini'), os.path.abspath('flake8/.flake8'), os.path.abspath('setup.cfg'), os.path.abspath('tox.ini'), os.path.abspath('.flake8')]), + # Common prefix of "flake8/options" + (['flake8/options', 'flake8/options/sub'], + [os.path.abspath('flake8/options/setup.cfg'), + os.path.abspath('flake8/options/tox.ini'), + os.path.abspath('flake8/options/.flake8'), + os.path.abspath('flake8/setup.cfg'), + os.path.abspath('flake8/tox.ini'), + os.path.abspath('flake8/.flake8'), + os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath('.flake8')]), ]) def test_generate_possible_local_config_files(args, expected): """Verify generation of all possible config paths.""" From fdff0f705e2dd2dbdb9427d277b181068644d2f6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:05:21 -0600 Subject: [PATCH 044/204] Allow users to pass args to py.test via tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ecaae55..650c16e 100644 --- a/tox.ini +++ b/tox.ini @@ -6,4 +6,4 @@ deps = mock pytest commands = - py.test + py.test {posargs} From b8dfc9a8e2aef52a52ac753d617f2e44c76a49d5 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:39:37 -0600 Subject: [PATCH 045/204] Start testing ConfigFileFinder.local_config_files Make sure extra_config_files are all absolute paths and filter them in local_config_files to ensure they exist. --- flake8/options/config.py | 21 ++++++++++++++++++--- tests/unit/test_config_file_finder.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/flake8/options/config.py b/flake8/options/config.py index e82cb62..d4f64d8 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -29,7 +29,11 @@ class ConfigFileFinder(object): Extra configuration files specified by the user to read. """ # The values of --append-config from the CLI - self.extra_config_files = extra_config_files + extra_config_files = extra_config_files or [] + self.extra_config_files = [ + # Ensure the paths are absolute paths for local_config_files + os.path.abspath(f) for f in extra_config_files + ] # Platform specific settings self.is_windows = sys.platform == 'win32' @@ -77,12 +81,23 @@ class ConfigFileFinder(object): (parent, tail) = os.path.split(parent) def local_config_files(self): - """Find all local config files.""" + """Find all local config files which actually exist. + + Filter results from + :meth:`~ConfigFileFinder.generate_possible_local_config_files` based + on whether the filename exists or not. + + :returns: + List of files that exist that are local project config files with + extra config files appended to that list (which also exist). + :rtype: + [str] + """ return [ filename for filename in self.generate_possible_local_config_files() if os.path.exists(filename) - ] + self.extra_config_files + ] + list(filter(os.path.exists, self.extra_config_files)) def local_configs(self): """Parse all local config files into one config object.""" diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index e723397..4b73979 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -67,3 +67,27 @@ def test_generate_possible_local_config_files(args, expected): assert (list(finder.generate_possible_local_config_files()) == expected) + + +@pytest.mark.parametrize('args,extra_config_files,expected', [ + # No arguments, common prefix of abspath('.') + ([], + [], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini')]), + # Common prefix of "flake8/" + (['flake8/options', 'flake8/'], + [], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini')]), + # Common prefix of "flake8/options" + (['flake8/options', 'flake8/options/sub'], + [], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini')]), +]) +def test_local_config_files(args, extra_config_files, expected): + """Verify discovery of local config files.""" + finder = config.ConfigFileFinder('flake8', args, extra_config_files) + + assert list(finder.local_config_files()) == expected From 91d20207fbb44993b5baf763e1fb9202ea9a1a46 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:40:09 -0600 Subject: [PATCH 046/204] Add setup.cfg for testing purposes --- setup.cfg | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e69de29 From 7269d84845f9aa4896a418d6d594682ab0b6d074 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:45:37 -0600 Subject: [PATCH 047/204] Add test parameters with extra_config_files --- tests/unit/test_config_file_finder.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 4b73979..35fa4a6 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -7,6 +7,8 @@ import pytest from flake8.options import config +CLI_SPECIFIED_FILEPATH = 'tests/fixtures/config_files/cli-specified.ini' + def test_uses_default_args(): """Show that we default the args value.""" @@ -28,7 +30,7 @@ def test_windows_detection(platform, is_windows): def test_cli_config(): """Verify opening and reading the file specified via the cli.""" - cli_filepath = 'tests/fixtures/config_files/cli-specified.ini' + cli_filepath = CLI_SPECIFIED_FILEPATH finder = config.ConfigFileFinder('flake8', None, []) parsed_config = finder.cli_config(cli_filepath) @@ -85,6 +87,12 @@ def test_generate_possible_local_config_files(args, expected): [], [os.path.abspath('setup.cfg'), os.path.abspath('tox.ini')]), + # Common prefix of "flake8/" with extra config files specified + (['flake8/'], + [CLI_SPECIFIED_FILEPATH], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath(CLI_SPECIFIED_FILEPATH)]), ]) def test_local_config_files(args, extra_config_files, expected): """Verify discovery of local config files.""" From 9231aae1d6ecfca856e97fa4279cf06e83f42e57 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:50:16 -0600 Subject: [PATCH 048/204] Add README to fixtures directory --- tests/fixtures/config_files/README.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/fixtures/config_files/README.rst diff --git a/tests/fixtures/config_files/README.rst b/tests/fixtures/config_files/README.rst new file mode 100644 index 0000000..b69e080 --- /dev/null +++ b/tests/fixtures/config_files/README.rst @@ -0,0 +1,22 @@ +About this directory +==================== + +The files in this directory are test fixtures for unit and integration tests. +Their purpose is described below. Please note the list of file names that can +not be created as they are already used by tests. + +New fixtures are preferred over updating existing features unless existing +tests will fail. + +Files that should not be created +-------------------------------- + +- ``tests/fixtures/config_files/missing.ini`` + +Purposes of existing fixtures +----------------------------- + +``tests/fixtures/config_files/cli-specified.ini`` + + This should only be used when providing config file(s) specified by the + user on the command-line. From de96b24bad8a1b099749a8b8473de9e885b4718e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 09:50:32 -0600 Subject: [PATCH 049/204] Add parameter for missing extra config file --- tests/unit/test_config_file_finder.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 35fa4a6..948e741 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -93,6 +93,13 @@ def test_generate_possible_local_config_files(args, expected): [os.path.abspath('setup.cfg'), os.path.abspath('tox.ini'), os.path.abspath(CLI_SPECIFIED_FILEPATH)]), + # Common prefix of "flake8/" with missing extra config files specified + (['flake8/'], + [CLI_SPECIFIED_FILEPATH, + 'tests/fixtures/config_files/missing.ini'], + [os.path.abspath('setup.cfg'), + os.path.abspath('tox.ini'), + os.path.abspath(CLI_SPECIFIED_FILEPATH)]), ]) def test_local_config_files(args, extra_config_files, expected): """Verify discovery of local config files.""" From 6553198074593841d720c9978103fb0d5f6e6915 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 12:01:22 -0600 Subject: [PATCH 050/204] Update setup.py --- setup.cfg | 5 +++++ setup.py | 20 ++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/setup.cfg b/setup.cfg index e69de29..f05f933 100644 --- a/setup.cfg +++ b/setup.cfg @@ -0,0 +1,5 @@ +[aliases] +test=pytest + +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index d0650f4..92ea08f 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import with_statement -from setuptools import setup +import setuptools + +import flake8 + try: # Work around a traceback with Nose on Python 2.6 # http://bugs.python.org/issue15881#msg170215 @@ -16,18 +19,11 @@ except ImportError: mock = None -tests_require = ['nose'] +tests_require = ['pytest'] if mock is None: tests_require += ['mock'] -def get_version(fname='flake8/__init__.py'): - with open(fname) as f: - for line in f: - if line.startswith('__version__'): - return eval(line.split('=')[-1]) - - def get_long_description(): descr = [] for fname in ('README.rst', 'CHANGES.rst'): @@ -36,10 +32,10 @@ def get_long_description(): return '\n\n'.join(descr) -setup( +setuptools.setup( name="flake8", license="MIT", - version=get_version(), + version=flake8.__version__, description="the modular source code checker: pep8, pyflakes and co", # long_description=get_long_description(), author="Tarek Ziade", @@ -71,5 +67,5 @@ setup( "Topic :: Software Development :: Quality Assurance", ], tests_require=tests_require, - test_suite='nose.collector', + setup_requires=['pytest-runner'], ) From 8b85e93e21fc44fcc1501b799141d16c4201381f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 12:52:29 -0600 Subject: [PATCH 051/204] Add test for local_configs method --- tests/unit/test_config_file_finder.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 948e741..200f4e5 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -1,4 +1,9 @@ """Tests for the ConfigFileFinder.""" +try: + import ConfigParser as configparser +except ImportError: + import configparser + import os import sys @@ -106,3 +111,10 @@ def test_local_config_files(args, extra_config_files, expected): finder = config.ConfigFileFinder('flake8', args, extra_config_files) assert list(finder.local_config_files()) == expected + + +def test_local_configs(): + """Verify we return a ConfigParser.""" + finder = config.ConfigFileFinder('flake8', None, []) + + assert isinstance(finder.local_configs(), configparser.RawConfigParser) From 66da9160b4f0ff730a6146f9504c4d947be3fab2 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:05:35 -0600 Subject: [PATCH 052/204] Start testing MergedConfigParser Rename the GETINT/GETBOOL sets on the MergedConfigParser to use better names. --- flake8/options/config.py | 28 ++++++++++------- tests/unit/test_merged_config_parser.py | 41 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_merged_config_parser.py diff --git a/flake8/options/config.py b/flake8/options/config.py index d4f64d8..558ec93 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -128,8 +128,12 @@ class MergedConfigParser(object): dictionaries with the parsed values. """ - GETINT_METHODS = set(['int', 'count']) - GETBOOL_METHODS = set(['store_true', 'store_false']) + #: Set of types that should use the + #: :meth:`~configparser.RawConfigParser.getint` method. + GETINT_TYPES = set(['int', 'count']) + #: Set of actions that should use the + #: :meth:`~configparser.RawConfigParser.getbool` method. + GETBOOL_ACTIONS = set(['store_true', 'store_false']) def __init__(self, option_manager, extra_config_files=None, args=None): """Initialize the MergedConfigParser instance. @@ -141,18 +145,20 @@ class MergedConfigParser(object): :params list args: The extra parsed arguments from the command-line. """ - # Our instance of flake8.options.manager.OptionManager + #: Our instance of flake8.options.manager.OptionManager self.option_manager = option_manager - # The prog value for the cli parser + #: The prog value for the cli parser self.program_name = option_manager.program_name - # Parsed extra arguments + #: Parsed extra arguments self.args = args - # Our instance of our ConfigFileFinder - self.config_finder = ConfigFileFinder(self.program_name, self.args) - # Mapping of configuration option names to - # flake8.options.manager.Option instances + #: Mapping of configuration option names to + #: :class:`~flake8.options.manager.Option` instances self.config_options = option_manager.config_options_dict + #: List of extra config files self.extra_config_files = extra_config_files or [] + #: Our instance of our :class:`~ConfigFileFinder` + self.config_finder = ConfigFileFinder(self.program_name, self.args, + self.extra_config_files) @staticmethod def _normalize_value(option, value): @@ -172,9 +178,9 @@ class MergedConfigParser(object): # Use the appropriate method to parse the config value method = config_parser.get - if option.type in self.GETINT_METHODS: + if option.type in self.GETINT_TYPES: method = config_parser.getint - elif option.action in self.GETBOOL_METHODS: + elif option.action in self.GETBOOL_ACTIONS: method = config_parser.getboolean value = method(self.program_name, option_name) diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py new file mode 100644 index 0000000..2e79b4a --- /dev/null +++ b/tests/unit/test_merged_config_parser.py @@ -0,0 +1,41 @@ +"""Unit tests for flake8.options.config.MergedConfigParser.""" +import mock +import pytest + +from flake8.options import config +from flake8.options import manager + + +@pytest.fixture +def optmanager(): + return manager.OptionManager(prog='flake8', version='3.0.0a1') + + +@pytest.mark.parametrize('args,extra_config_files', [ + (None, None), + (None, []), + (None, ['foo.ini']), + ('flake8/', []), + ('flake8/', ['foo.ini']), +]) +def test_creates_its_own_config_file_finder(args, extra_config_files, + optmanager): + """Verify we create a ConfigFileFinder correctly.""" + class_path = 'flake8.options.config.ConfigFileFinder' + with mock.patch(class_path) as ConfigFileFinder: + parser = config.MergedConfigParser( + option_manager=optmanager, + extra_config_files=extra_config_files, + args=args, + ) + + assert parser.program_name == 'flake8' + ConfigFileFinder.assert_called_once_with( + 'flake8', + args, + extra_config_files or [], + ) + + +def test_parse_cli_config(optmanager): + pass From c33b9d24b102b1dd845886e3434bd079b1d58cb4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:13:08 -0600 Subject: [PATCH 053/204] Add top-level function to configure logging --- flake8/__init__.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index 49cbabd..6f33c82 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -1,4 +1,5 @@ import logging +import sys try: from logging import NullHandler @@ -7,6 +8,48 @@ except ImportError: def emit(self, record): pass -logging.getLogger(__name__).addHandler(NullHandler()) +LOG = logging.getLogger(__name__) +LOG.addHandler(NullHandler()) + +# Clean up after LOG config +del NullHandler __version__ = '3.0.0a1' + + +_VERBOSITY_TO_LOG_LEVEL = { + # output more than warnings but not debugging info + 1: logging.INFO, + # output debugging information and everything else + 2: logging.DEBUG, +} + + +def configure_logging(verbosity, filename=None): + """Configure logging for flake8. + + :param int verbosity: + How verbose to be in logging information. + :param str filename: + Name of the file to append log information to. + If ``None`` this will log to ``sys.stderr``. + If the name is "stdout" or "stderr" this will log to the appropriate + stream. + """ + global LOG + if verbosity <= 0: + return + if verbosity > 2: + verbosity = 2 + + log_level = _VERBOSITY_TO_LOG_LEVEL[verbosity] + + if not filename or filename in ('stderr', 'stdout'): + handler = logging.StreamHandler(getattr(sys, filename)) + else: + handler = logging.FileHandler(filename) + + LOG.addHandler(handler) + LOG.setLevel(log_level) + LOG.debug('Added a %s logging handler to logger root at %s', + filename, __name__) From 16669a6c4567d870121217407e20ae9b7d5d326b Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:13:33 -0600 Subject: [PATCH 054/204] Add test configuration --- tests/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7fa72ff --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import sys + +import flake8 + +flake8.configure_logging(2, 'test-logs-%s.%s.log' % sys.version_info[0:2]) From 8b35e29f4ff327b42ea990540e4b4ff75fc4133a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:14:09 -0600 Subject: [PATCH 055/204] Ignore .eggs/ directory --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a41da07..dfc3773 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ *.pyc *.sw* *.egg-info + +.eggs/ From 4d13923bfbe4b1c6dae24693199af73d084420b2 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:15:14 -0600 Subject: [PATCH 056/204] Ignore test log files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dfc3773..612baf8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ *.pyc *.sw* *.egg-info - +*.log .eggs/ From 03fd22c23073cf2faffb3d9577c2ef134acf5e4d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:15:25 -0600 Subject: [PATCH 057/204] Add a docstring to conftest --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 7fa72ff..9bf4f95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +"""Test configuration for py.test.""" import sys import flake8 From a8576aff1239c3eb3e4d3e3ab28e82d045665e8e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 13:22:18 -0600 Subject: [PATCH 058/204] Fix bugs found while writing parse_cli_config test --- flake8/options/config.py | 4 +++- flake8/options/manager.py | 3 ++- tests/unit/test_merged_config_parser.py | 21 ++++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/flake8/options/config.py b/flake8/options/config.py index 558ec93..7acddd5 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -186,9 +186,11 @@ class MergedConfigParser(object): value = method(self.program_name, option_name) LOG.debug('Option "%s" returned value: %r', option_name, value) - final_value = self._normalize_value(value) + final_value = self._normalize_value(option, value) config_dict[option_name] = final_value + return config_dict + def is_configured_by(self, config): """Check if the specified config parser has an appropriate section.""" return config.has_section(self.program_name) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index fafc1e9..c4da8e0 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -104,7 +104,8 @@ class Option(object): 'dest={dest}, type={type}, callback={callback}, help={help},' ' callback={callback}, callback_args={callback_args}, ' 'callback_kwargs={callback_kwargs}, metavar={metavar})' - ).format(*self.option_args, **self.option_kwargs) + ).format(self.short_option_name, self.long_option_name, + **self.option_kwargs) def _make_dest(self, dest): if dest: diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index 2e79b4a..028ad18 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -1,4 +1,6 @@ """Unit tests for flake8.options.config.MergedConfigParser.""" +import os + import mock import pytest @@ -38,4 +40,21 @@ def test_creates_its_own_config_file_finder(args, extra_config_files, def test_parse_cli_config(optmanager): - pass + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + + parsed_config = parser.parse_cli_config( + 'tests/fixtures/config_files/cli-specified.ini' + ) + assert parsed_config == { + 'ignore': ['E123', 'W234', 'E111'], + 'exclude': [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] + } From 59035767329af1a6e29b9b16f172064102d4bdee Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 14:36:52 -0600 Subject: [PATCH 059/204] Ignore only D203 and fix Flake8 errors --- flake8/__init__.py | 14 ++++++++++++++ flake8/_trie.py | 2 +- flake8/notifier.py | 3 +++ flake8/options/__init__.py | 12 ++++++++++++ flake8/options/manager.py | 4 ++-- flake8/style_guide.py | 1 + tests/unit/test_merged_config_parser.py | 1 + tox.ini | 3 +++ 8 files changed, 37 insertions(+), 3 deletions(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index 6f33c82..50dc71c 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -1,3 +1,14 @@ +"""Top-level module for Flake8. + +This module + +- initializes logging for the command-line tool +- tracks the version of the package +- provides a way to configure logging for the command-line tool + +.. autofunction:: flake8.configure_logging + +""" import logging import sys @@ -5,7 +16,10 @@ try: from logging import NullHandler except ImportError: class NullHandler(logging.Handler): + """Shim for version of Python < 2.7.""" + def emit(self, record): + """Do nothing.""" pass LOG = logging.getLogger(__name__) diff --git a/flake8/_trie.py b/flake8/_trie.py index 858c067..4871abb 100644 --- a/flake8/_trie.py +++ b/flake8/_trie.py @@ -5,7 +5,7 @@ __all__ = ('Trie', 'TrieNode') def _iterate_stringlike_objects(string): for i in range(len(string)): - yield string[i:i+1] + yield string[i:i + 1] class Trie(object): diff --git a/flake8/notifier.py b/flake8/notifier.py index 6bf51ef..e7bf3ba 100644 --- a/flake8/notifier.py +++ b/flake8/notifier.py @@ -1,7 +1,10 @@ +"""Implementation of the class that registers and notifies listeners.""" from flake8 import _trie class Notifier(object): + """Object that tracks and notifies listener objects.""" + def __init__(self): """Initialize an empty notifier object.""" self.listeners = _trie.Trie() diff --git a/flake8/options/__init__.py b/flake8/options/__init__.py index e69de29..cc20daa 100644 --- a/flake8/options/__init__.py +++ b/flake8/options/__init__.py @@ -0,0 +1,12 @@ +"""Package containing the option manager and config management logic. + +- :mod:`flake8.options.config` contains the logic for finding, parsing, and + merging configuration files. + +- :mod:`flake8.options.manager` contains the logic for managing customized + Flake8 command-line and configuration options. + +- :mod:`flake8.options.aggregator` uses objects from both of the above modules + to aggregate configuration into one object used by plugins and Flake8. + +""" diff --git a/flake8/options/manager.py b/flake8/options/manager.py index c4da8e0..d328a1f 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -104,8 +104,8 @@ class Option(object): 'dest={dest}, type={type}, callback={callback}, help={help},' ' callback={callback}, callback_args={callback_args}, ' 'callback_kwargs={callback_kwargs}, metavar={metavar})' - ).format(self.short_option_name, self.long_option_name, - **self.option_kwargs) + ).format(self.short_option_name, self.long_option_name, + **self.option_kwargs) def _make_dest(self, dest): if dest: diff --git a/flake8/style_guide.py b/flake8/style_guide.py index e69de29..a3f63eb 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -0,0 +1 @@ +"""Implementation of the StyleGuide used by Flake8.""" diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index 028ad18..96f6aae 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -40,6 +40,7 @@ def test_creates_its_own_config_file_finder(args, extra_config_files, def test_parse_cli_config(optmanager): + """Parse the specified config file as a cli config file.""" optmanager.add_option('--exclude', parse_from_config=True, comma_separated_list=True, normalize_paths=True) diff --git a/tox.ini b/tox.ini index 650c16e..8d9d425 100644 --- a/tox.ini +++ b/tox.ini @@ -7,3 +7,6 @@ deps = pytest commands = py.test {posargs} + +[flake8] +ignore = D203 From 8c872c4bb99c13d82b7b82db037eb3cb29a243e4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 14:47:06 -0600 Subject: [PATCH 060/204] Add more docstrings and fix lint errors --- setup.py | 2 ++ tests/unit/__init__.py | 0 tests/unit/test_merged_config_parser.py | 1 + tests/unit/test_notifier.py | 5 +++++ tests/unit/test_option.py | 1 + tests/unit/test_option_manager.py | 1 + tests/unit/test_trie.py | 1 + tests/unit/test_utils.py | 1 + tox.ini | 1 + 9 files changed, 13 insertions(+) delete mode 100644 tests/unit/__init__.py diff --git a/setup.py b/setup.py index 92ea08f..064b28e 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +"""Packaging logic for Flake8.""" # -*- coding: utf-8 -*- from __future__ import with_statement import setuptools @@ -25,6 +26,7 @@ if mock is None: def get_long_description(): + """Generate a long description from the README and CHANGES files.""" descr = [] for fname in ('README.rst', 'CHANGES.rst'): with open(fname) as f: diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index 96f6aae..8287d1b 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -10,6 +10,7 @@ from flake8.options import manager @pytest.fixture def optmanager(): + """Generate an OptionManager with simple values.""" return manager.OptionManager(prog='flake8', version='3.0.0a1') diff --git a/tests/unit/test_notifier.py b/tests/unit/test_notifier.py index b0365f3..84b8a45 100644 --- a/tests/unit/test_notifier.py +++ b/tests/unit/test_notifier.py @@ -1,7 +1,9 @@ +"""Unit tests for the Notifier object.""" import pytest from flake8 import notifier + class _Listener(object): def __init__(self, error_code): self.error_code = error_code @@ -13,8 +15,11 @@ class _Listener(object): class TestNotifier(object): + """Notifier unit tests.""" + @pytest.fixture(autouse=True) def setup(self): + """Set up each TestNotifier instance.""" self.notifier = notifier.Notifier() self.listener_map = {} diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py index 55cae5c..5facf2a 100644 --- a/tests/unit/test_option.py +++ b/tests/unit/test_option.py @@ -60,5 +60,6 @@ def test_config_name_needs_long_option_name(): def test_dest_is_not_overridden(): + """Show that we do not override custom destinations.""" opt = manager.Option('-s', '--short', dest='something_not_short') assert opt.dest == 'something_not_short' diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index a9e641d..b0e4ee7 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -9,6 +9,7 @@ from flake8.options import manager @pytest.fixture def optmanager(): + """Generate a simple OptionManager with default test arguments.""" return manager.OptionManager(prog='flake8', version='3.0.0b1') diff --git a/tests/unit/test_trie.py b/tests/unit/test_trie.py index 0a5a0a9..4cbd8c5 100644 --- a/tests/unit/test_trie.py +++ b/tests/unit/test_trie.py @@ -1,3 +1,4 @@ +"""Unit test for the _trie module.""" from flake8 import _trie as trie diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b0e243c..a6096d4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -38,4 +38,5 @@ def test_normalize_path(value, expected): [os.path.abspath("../" + p) for p in RELATIVE_PATHS]), ]) def test_normalize_paths(value, expected): + """Verify we normalize comma-separated paths provided to the tool.""" assert utils.normalize_paths(value) == expected diff --git a/tox.ini b/tox.ini index 8d9d425..2267e15 100644 --- a/tox.ini +++ b/tox.ini @@ -9,4 +9,5 @@ commands = py.test {posargs} [flake8] +# Ignore some flake8-docstrings errors ignore = D203 From 629f796fc17820a28a5d08e9f039c636fa529a29 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 14:53:26 -0600 Subject: [PATCH 061/204] Add flake8 linting as a testenv --- tox.ini | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2267e15..15218a1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = py26,py27,py32,py33,py34,py35 +minversion=2.3.1 +envlist = py26,py27,py32,py33,py34,py35,flake8 [testenv] deps = @@ -8,6 +9,16 @@ deps = commands = py.test {posargs} +[testenv:flake8] +skipsdist = true +skip_install = true +use_develop = false +deps = + flake8 + flake8-docstrings +commands = + flake8 + [flake8] # Ignore some flake8-docstrings errors ignore = D203 From 53d5f0e49af43a3598d261d9e9904aeb21f08c0e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 15:19:34 -0600 Subject: [PATCH 062/204] Add a pre-specified log format --- flake8/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake8/__init__.py b/flake8/__init__.py index 50dc71c..a0991cc 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -63,6 +63,9 @@ def configure_logging(verbosity, filename=None): else: handler = logging.FileHandler(filename) + handler.setFormatter( + logging.Formatter('%(asctime)s %(levelname)s %(message)s') + ) LOG.addHandler(handler) LOG.setLevel(log_level) LOG.debug('Added a %s logging handler to logger root at %s', From d04cb245d3d9a1f61a5a8cc60d1fbf742d7b46d3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 15:19:57 -0600 Subject: [PATCH 063/204] Move user and local config merging to own method --- flake8/options/config.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/flake8/options/config.py b/flake8/options/config.py index 7acddd5..2484cdd 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -228,6 +228,22 @@ class MergedConfigParser(object): LOG.debug('Parsing CLI configuration files.') return self._parse_config(config) + def merge_user_and_local_config(self): + """Merge the parsed user and local configuration files. + + :returns: + Dictionary of the parsed and merged configuration options. + :rtype: + dict + """ + user_config = self.parse_user_config() + config = self.parse_local_config() + + for option, value in user_config.items(): + config.setdefault(option, value) + + return config + def parse(self, cli_config=None, isolated=False): """Parse and return the local and user config files. @@ -257,10 +273,4 @@ class MergedConfigParser(object): '--config by the user', cli_config) return self.parse_cli_config(cli_config) - user_config = self.parse_user_config() - config = self.parse_local_config() - - for option, value in user_config.items(): - config.setdefault(option, value) - - return config + return self.merge_user_and_local_config() From 18bfa1f8a2b4232620200c738d6bbda7b17d0a6f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 15:20:08 -0600 Subject: [PATCH 064/204] Add test for is_configured_by --- tests/unit/test_merged_config_parser.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index 8287d1b..ba3100f 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -60,3 +60,15 @@ def test_parse_cli_config(optmanager): os.path.abspath('bogus/'), ] } + + +@pytest.mark.parametrize('filename,is_configured_by', [ + ('tests/fixtures/config_files/cli-specified.ini', True), + ('tests/fixtures/config_files/no-flake8-section.ini', False), +]) +def test_is_configured_by(filename, is_configured_by, optmanager): + """Verify the behaviour of the is_configured_by method.""" + parsed_config, _ = config.ConfigFileFinder._read_config(filename) + parser = config.MergedConfigParser(optmanager) + + assert parser.is_configured_by(parsed_config) is is_configured_by From ebe47ec7407e309f5b39ed6d0560aaac8b0b2853 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 21:05:51 -0600 Subject: [PATCH 065/204] Add fixture file without a flake8 section --- .../config_files/no-flake8-section.ini | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/fixtures/config_files/no-flake8-section.ini diff --git a/tests/fixtures/config_files/no-flake8-section.ini b/tests/fixtures/config_files/no-flake8-section.ini new file mode 100644 index 0000000..a85b709 --- /dev/null +++ b/tests/fixtures/config_files/no-flake8-section.ini @@ -0,0 +1,20 @@ +[tox] +minversion=2.3.1 +envlist = py26,py27,py32,py33,py34,py35,flake8 + +[testenv] +deps = + mock + pytest +commands = + py.test {posargs} + +[testenv:flake8] +skipsdist = true +skip_install = true +use_develop = false +deps = + flake8 + flake8-docstrings +commands = + flake8 From e5c3ae5cd217bb83545114a2bcbe35a0a336e39d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 21:43:00 -0600 Subject: [PATCH 066/204] Add tests for parse_{user,local}_config methods --- tests/unit/test_merged_config_parser.py | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index ba3100f..d0f53fd 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -72,3 +72,52 @@ def test_is_configured_by(filename, is_configured_by, optmanager): parser = config.MergedConfigParser(optmanager) assert parser.is_configured_by(parsed_config) is is_configured_by + + +def test_parse_user_config(optmanager): + """Verify parsing of user config files.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + + with mock.patch.object(parser.config_finder, 'user_config_file') as usercf: + usercf.return_value = 'tests/fixtures/config_files/cli-specified.ini' + parsed_config = parser.parse_user_config() + + assert parsed_config == { + 'ignore': ['E123', 'W234', 'E111'], + 'exclude': [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] + } + + +def test_parse_local_config(optmanager): + """Verify parsing of local config files.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + config_finder = parser.config_finder + + with mock.patch.object(config_finder, 'local_config_files') as localcfs: + localcfs.return_value = [ + 'tests/fixtures/config_files/cli-specified.ini' + ] + parsed_config = parser.parse_local_config() + + assert parsed_config == { + 'ignore': ['E123', 'W234', 'E111'], + 'exclude': [ + os.path.abspath('foo/'), + os.path.abspath('bar/'), + os.path.abspath('bogus/'), + ] + } From cb9231eb9ba88a01ccbd47910d4617405ef5e2dc Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 10 Jan 2016 22:07:30 -0600 Subject: [PATCH 067/204] Add tests around merge_user_and_local_config and parse --- tests/unit/test_merged_config_parser.py | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index d0f53fd..2c4c5fd 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -121,3 +121,53 @@ def test_parse_local_config(optmanager): os.path.abspath('bogus/'), ] } + + +def test_merge_user_and_local_config(optmanager): + """Verify merging of parsed user and local config files.""" + optmanager.add_option('--exclude', parse_from_config=True, + comma_separated_list=True, + normalize_paths=True) + optmanager.add_option('--ignore', parse_from_config=True, + comma_separated_list=True) + optmanager.add_option('--select', parse_from_config=True, + comma_separated_list=True) + parser = config.MergedConfigParser(optmanager) + config_finder = parser.config_finder + + with mock.patch.object(config_finder, 'local_config_files') as localcfs: + localcfs.return_value = [ + 'tests/fixtures/config_files/local-config.ini' + ] + with mock.patch.object(config_finder, + 'user_config_file') as usercf: + usercf.return_value = ('tests/fixtures/config_files/' + 'user-config.ini') + parsed_config = parser.merge_user_and_local_config() + + assert parsed_config == { + 'exclude': [ + os.path.abspath('docs/') + ], + 'ignore': ['D203'], + 'select': ['E', 'W', 'F'], + } + + +@mock.patch('flake8.options.config.ConfigFileFinder') +def test_parse_isolates_config(ConfigFileManager, optmanager): + """Verify behaviour of the parse method with isolated=True.""" + parser = config.MergedConfigParser(optmanager) + + assert parser.parse(isolated=True) == {} + assert parser.config_finder.local_configs.called is False + assert parser.config_finder.user_config.called is False + + +@mock.patch('flake8.options.config.ConfigFileFinder') +def test_parse_uses_cli_config(ConfigFileManager, optmanager): + """Verify behaviour of the parse method with a specified config.""" + parser = config.MergedConfigParser(optmanager) + + parser.parse(cli_config='foo.ini') + parser.config_finder.cli_config.assert_called_once_with('foo.ini') From 6940ef3aafa20f72a7da616aafe05c1c7df93a9f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 11 Jan 2016 23:15:11 -0600 Subject: [PATCH 068/204] Make log format configurable --- flake8/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index a0991cc..a6520e4 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -39,7 +39,8 @@ _VERBOSITY_TO_LOG_LEVEL = { } -def configure_logging(verbosity, filename=None): +def configure_logging(verbosity, filename=None, + logformat='%(asctime)s %(levelname)s %(message)s'): """Configure logging for flake8. :param int verbosity: @@ -63,9 +64,7 @@ def configure_logging(verbosity, filename=None): else: handler = logging.FileHandler(filename) - handler.setFormatter( - logging.Formatter('%(asctime)s %(levelname)s %(message)s') - ) + handler.setFormatter(logging.Formatter(logformat)) LOG.addHandler(handler) LOG.setLevel(log_level) LOG.debug('Added a %s logging handler to logger root at %s', From 40189885852ca17a7bc3c05fc9540b4721551503 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 11 Jan 2016 23:15:23 -0600 Subject: [PATCH 069/204] Add a module with defaults --- flake8/defaults.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 flake8/defaults.py diff --git a/flake8/defaults.py b/flake8/defaults.py new file mode 100644 index 0000000..ac798b5 --- /dev/null +++ b/flake8/defaults.py @@ -0,0 +1,5 @@ +"""Constants that define defaults.""" + +EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' +IGNORE = 'E121,E123,E126,E226,E24,E704' +MAX_LINE_LENGTH = 79 From 8466d1462f5a3b3b3775f229b104d275e7584da9 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 11 Jan 2016 23:15:53 -0600 Subject: [PATCH 070/204] Start bulding a bit of the cli for functional testing --- flake8/main/__init__.py | 0 flake8/main/cli.py | 122 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 flake8/main/__init__.py create mode 100644 flake8/main/cli.py diff --git a/flake8/main/__init__.py b/flake8/main/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flake8/main/cli.py b/flake8/main/cli.py new file mode 100644 index 0000000..5bd7c84 --- /dev/null +++ b/flake8/main/cli.py @@ -0,0 +1,122 @@ +"""Command-line implementation of flake8.""" +from flake8 import defaults + + +def register_default_options(option_manager): + """Register the default options on our OptionManager.""" + add_option = option_manager.add_option + + # pep8 options + add_option( + '-v', '--verbose', default=0, action='count', + parse_from_config=True, + help='Print more information about what is happening in flake8.' + ' This option is repeatable and will increase verbosity each ' + 'time it is repeated.', + ) + add_option( + '-q', '--quiet', default=0, action='count', + parse_from_config=True, + help='Report only file names, or nothing. This option is repeatable.', + ) + + add_option( + '--count', action='store_true', parse_from_config=True, + help='Print total number of errors and warnings to standard error and' + ' set the exit code to 1 if total is not empty.', + ) + + add_option( + '--diff', action='store_true', + help='Report changes only within line number ranges in the unified ' + 'diff provided on standard in by the user.', + ) + + add_option( + '--exclude', metavar='patterns', default=defaults.EXCLUDE, + comma_separated_list=True, parse_from_config=True, + normalize_paths=True, + help='Comma-separated list of files or directories to exclude.' + '(Default: %default)', + ) + + add_option( + '--filename', metavar='patterns', default='*.py', + parse_from_config=True, comma_separated_list=True, + help='Only check for filenames matching the patterns in this comma-' + 'separated list. (Default: %default)', + ) + + # TODO(sigmavirus24): Figure out --first/--repeat + + add_option( + '--format', metavar='format', default='default', choices=['default'], + parse_from_config=True, + help='Format errors according to the chosen formatter.', + ) + + add_option( + '--hang-closing', action='store_true', parse_from_config=True, + help='Hang closing bracket instead of matching indentation of opening' + " bracket's line.", + ) + + add_option( + '--ignore', metavar='errors', default=defaults.IGNORE, + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to ignore (or skip).' + ' For example, ``--ignore=E4,E51,W234``. (Default: %default)', + ) + + add_option( + '--max-line-length', type='int', metavar='n', + default=defaults.MAX_LINE_LENGTH, parse_from_config=True, + help='Maximum allowed line length for the entirety of this run. ' + '(Default: %default)', + ) + + add_option( + '--select', metavar='errors', default='', + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to enable.' + ' For example, ``--select=E4,E51,W234``. (Default: %default)', + ) + + # TODO(sigmavirus24): Decide what to do about --show-pep8 + + add_option( + '--show-source', action='store_true', parse_from_config=True, + help='Show the source generate each error or warning.', + ) + + add_option( + '--statistics', action='store_true', parse_from_config=True, + help='Count errors and warnings.', + ) + + # Flake8 options + add_option( + '--enabled-extensions', default='', parse_from_config=True, + comma_separated_list=True, type='string', + help='Enable plugins and extensions that are otherwise disabled ' + 'by default', + ) + + add_option( + '--exit-zero', action='store_true', + help='Exit with status code "0" even if there are errors.', + ) + + add_option( + '-j', '--jobs', type='string', default='auto', parse_from_config=True, + help='Number of subprocesses to use to run checks in parallel. ' + 'This is ignored on Windows. The default, "auto", will ' + 'auto-detect the number of processors available to use.' + ' (Default: %default)', + ) + + add_option( + '--output-file', default=None, type='string', parse_from_config=True, + # callback=callbacks.redirect_stdout, + help='Redirect report to a file.', + ) From 18708111372a4cbc0cd961cd8a32fc66e83df781 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 14 Jan 2016 22:25:02 -0600 Subject: [PATCH 071/204] Start working on plugin management --- flake8/plugins/__init__.py | 0 flake8/plugins/manager.py | 199 +++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 flake8/plugins/__init__.py create mode 100644 flake8/plugins/manager.py diff --git a/flake8/plugins/__init__.py b/flake8/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py new file mode 100644 index 0000000..0bc4a78 --- /dev/null +++ b/flake8/plugins/manager.py @@ -0,0 +1,199 @@ +"""Plugin loading and management logic and classes.""" +import logging + +import pkg_resources + +LOG = logging.getLogger(__name__) + + +class Plugin(object): + """Wrap an EntryPoint from setuptools and other logic.""" + + def __init__(self, name, entry_point): + """"Initialize our Plugin. + + :param str name: + Name of the entry-point as it was registered with setuptools. + :param entry_point: + EntryPoint returned by setuptools. + :type entry_point: + setuptools.EntryPoint + """ + self.name = name + self.entry_point = entry_point + self.plugin = None + + def __repr__(self): + return 'Plugin(name="{0}", entry_point="{1}")'.format( + self.name, self.entry_point + ) + + def execute(self, *args, **kwargs): + r"""Call the plugin with \*args and \*\*kwargs.""" + return self.plugin(*args, **kwargs) + + def load(self, verify_requirements=False): + """Retrieve the plugin for this entry-point. + + This loads the plugin, stores it on the instance and then returns it. + It does not reload it after the first time, it merely returns the + cached plugin. + + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + :returns: + The plugin resolved from the entry-point. + """ + if self.plugin is None: + LOG.debug('Loading plugin "%s" from entry-point.', self.name) + # Avoid relying on hasattr() here. + resolve = getattr(self.entry_point, 'resolve', None) + require = getattr(self.entry_point, 'require', None) + if resolve and require: + if verify_requirements: + LOG.debug('Verifying plugin "%s"\'s requirements.', + self.name) + require() + self.plugin = resolve() + else: + self.plugin = self.entry_point.load( + require=verify_requirements + ) + + return self.plugin + + def provide_options(self, optmanager, options, extra_args): + """Pass the parsed options and extra arguments to the plugin.""" + plugin = self.load() + parse_options = getattr(plugin, 'parse_options', None) + if parse_options is not None: + LOG.debug('Providing options to plugin "%s".', self.name) + try: + parse_options(optmanager, options, extra_args) + except TypeError: + parse_options(options) + + def register_options(self, optmanager): + """Register the plugin's command-line options on the OptionManager. + + :param optmanager: + Instantiated OptionManager to register options on. + :type optmanager: + flake8.options.manager.OptionManager + :returns: + Nothing + """ + plugin = self.load() + add_options = getattr(plugin, 'add_options', None) + if add_options is not None: + LOG.debug( + 'Registering options from plugin "%s" on OptionManager %r', + self.name, optmanager + ) + plugin.add_options(optmanager) + + +class PluginManager(object): + """Find and manage plugins consistently.""" + + def __init__(self, namespace, verify_requirements=False): + """Initialize the manager. + + :param str namespace: + Namespace of the plugins to manage, e.g., 'flake8.extension'. + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + """ + self.namespace = namespace + self.verify_requirements = verify_requirements + self.plugins = {} + self.names = [] + self._load_all_plugins() + + def __contains__(self, name): + LOG.debug('Checking for "%s" in plugin manager.', name) + return name in self.plugins + + def __getitem__(self, name): + LOG.debug('Retrieving plugin for "%s".', name) + return self.plugins[name] + + def _load_all_plugins(self): + LOG.debug('Loading entry-points for "%s".', self.namespace) + for entry_point in pkg_resources.iter_entry_points(self.namespace): + name = entry_point.name + self.plugins[name] = Plugin(name, entry_point) + self.names.append(name) + LOG.info('Loaded %r for plugin "%s".', self.plugins[name], name) + + def map(self, func, *args, **kwargs): + """Call ``func`` with the plugin and *args and **kwargs after. + + This yields the return value from ``func`` for each plugin. + + :param collections.Callable func: + Function to call with each plugin. Signature should at least be: + + .. code-block:: python + + def myfunc(plugin): + pass + + Any extra positional or keyword arguments specified with map will + be passed along to this function after the plugin. + :param args: + Positional arguments to pass to ``func`` after each plugin. + :param kwargs: + Keyword arguments to pass to ``func`` after each plugin. + """ + for name in self.names: + yield func(self.plugins[name], *args, **kwargs) + + +class Checkers(object): + """All of the checkers registered through entry-ponits.""" + + def __init__(self, namespace='flake8.extension'): + """Initialize the Checkers collection.""" + self.manager = PluginManager(namespace) + + @property + def names(self): + return self.manager.names + + def register_options(self, optmanager): + """Register all of the checkers' options to the OptionManager.""" + def call_register_options(plugin, optmanager): + return plugin.register_options(optmanager) + + list(self.map(call_register_options, optmanager)) + + def provide_options(self, optmanager, options, extra_args): + def call_provide_options(plugin, optmanager, options, extra_args): + return plugin.provide_options(optmanager, options, extra_args) + + list(self.map(call_provide_options, optmanager, options, extra_args)) + + +class Listeners(object): + """All of the listeners registered through entry-points.""" + + def __init__(self, namespace='flake8.listener'): + self.manager = PluginManager(namespace) + + @property + def names(self): + return self.manager.names + + +class ReportFormatters(object): + """All of the report formatters registered through entry-points.""" + + def __init__(self, namespace='flake8.report'): + self.manager = PluginManager(namespace) + + @property + def names(self): + return self.manager.names From f1010b77334dc8f260fd21383baaacaac74926d8 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 15 Jan 2016 10:59:51 -0600 Subject: [PATCH 072/204] Make plugin and version loading lazy Abstract plugin type management into one class and provide specific classes for each of our known types of plugins --- flake8/plugins/manager.py | 101 +++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 0bc4a78..f18b666 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -1,4 +1,5 @@ """Plugin loading and management logic and classes.""" +import collections import logging import pkg_resources @@ -21,13 +22,22 @@ class Plugin(object): """ self.name = name self.entry_point = entry_point - self.plugin = None + self._plugin = None def __repr__(self): return 'Plugin(name="{0}", entry_point="{1}")'.format( self.name, self.entry_point ) + @property + def plugin(self): + self.load_plugin() + return self._plugin + + @property + def version(self): + return self.plugin.version + def execute(self, *args, **kwargs): r"""Call the plugin with \*args and \*\*kwargs.""" return self.plugin(*args, **kwargs) @@ -45,7 +55,22 @@ class Plugin(object): :returns: The plugin resolved from the entry-point. """ - if self.plugin is None: + return self.plugin + + def load_plugin(self, verify_requirements=False): + """Retrieve the plugin for this entry-point. + + This loads the plugin, stores it on the instance and then returns it. + It does not reload it after the first time, it merely returns the + cached plugin. + + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + :returns: + Nothing + """ + if self._plugin is None: LOG.debug('Loading plugin "%s" from entry-point.', self.name) # Avoid relying on hasattr() here. resolve = getattr(self.entry_point, 'resolve', None) @@ -55,14 +80,12 @@ class Plugin(object): LOG.debug('Verifying plugin "%s"\'s requirements.', self.name) require() - self.plugin = resolve() + self._plugin = resolve() else: - self.plugin = self.entry_point.load( + self._plugin = self.entry_point.load( require=verify_requirements ) - return self.plugin - def provide_options(self, optmanager, options, extra_args): """Pass the parsed options and extra arguments to the plugin.""" plugin = self.load() @@ -152,48 +175,68 @@ class PluginManager(object): yield func(self.plugins[name], *args, **kwargs) -class Checkers(object): - """All of the checkers registered through entry-ponits.""" +class PluginTypeManager(object): + """Parent class for most of the specific plugin types.""" - def __init__(self, namespace='flake8.extension'): - """Initialize the Checkers collection.""" - self.manager = PluginManager(namespace) + def __init__(self): + """Initialize the plugin type's manager.""" + self.manager = PluginManager(self.namespace) @property def names(self): + """Proxy attribute to underlying manager.""" return self.manager.names + @property + def plugins(self): + """Proxy attribute to underlying manager.""" + return self.manager.plugins + + @staticmethod + def _generate_call_function(method_name, optmanager, *args, **kwargs): + def generated_function(plugin): + method = getattr(plugin, method_name, None) + if (method is not None and + isinstance(method, collections.Callable)): + return method(optmanager, *args, **kwargs) + + def load_plugins(self): + def load_plugin(plugin): + return plugin.load() + + return list(self.manager.map(load_plugin)) + def register_options(self, optmanager): """Register all of the checkers' options to the OptionManager.""" - def call_register_options(plugin, optmanager): - return plugin.register_options(optmanager) + call_register_options = self._generate_call_function( + 'register_options', optmanager, + ) - list(self.map(call_register_options, optmanager)) + list(self.manager.map(call_register_options, optmanager)) def provide_options(self, optmanager, options, extra_args): - def call_provide_options(plugin, optmanager, options, extra_args): - return plugin.provide_options(optmanager, options, extra_args) + """Provide parsed options and extra arguments to the plugins.""" + call_provide_options = self._generate_call_function( + 'provide_options', optmanager, options, extra_args, + ) - list(self.map(call_provide_options, optmanager, options, extra_args)) + list(self.manager.map(call_provide_options, optmanager, options, + extra_args)) + + +class Checkers(PluginTypeManager): + """All of the checkers registered through entry-ponits.""" + + namespace = 'flake8.extension' class Listeners(object): """All of the listeners registered through entry-points.""" - def __init__(self, namespace='flake8.listener'): - self.manager = PluginManager(namespace) - - @property - def names(self): - return self.manager.names + namespace = 'flake8.listen' class ReportFormatters(object): """All of the report formatters registered through entry-points.""" - def __init__(self, namespace='flake8.report'): - self.manager = PluginManager(namespace) - - @property - def names(self): - return self.manager.names + namespace = 'flake8.report' From d2f4e97c315fac01d2c91f073083e01dd122374a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 15 Jan 2016 21:39:33 -0600 Subject: [PATCH 073/204] Delete unnecessary and dead code --- flake8/plugins/manager.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index f18b666..6e71f7e 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -42,21 +42,6 @@ class Plugin(object): r"""Call the plugin with \*args and \*\*kwargs.""" return self.plugin(*args, **kwargs) - def load(self, verify_requirements=False): - """Retrieve the plugin for this entry-point. - - This loads the plugin, stores it on the instance and then returns it. - It does not reload it after the first time, it merely returns the - cached plugin. - - :param bool verify_requirements: - Whether or not to make setuptools verify that the requirements for - the plugin are satisfied. - :returns: - The plugin resolved from the entry-point. - """ - return self.plugin - def load_plugin(self, verify_requirements=False): """Retrieve the plugin for this entry-point. @@ -88,8 +73,7 @@ class Plugin(object): def provide_options(self, optmanager, options, extra_args): """Pass the parsed options and extra arguments to the plugin.""" - plugin = self.load() - parse_options = getattr(plugin, 'parse_options', None) + parse_options = getattr(self.plugin, 'parse_options', None) if parse_options is not None: LOG.debug('Providing options to plugin "%s".', self.name) try: @@ -107,14 +91,13 @@ class Plugin(object): :returns: Nothing """ - plugin = self.load() - add_options = getattr(plugin, 'add_options', None) + add_options = getattr(self.plugin, 'add_options', None) if add_options is not None: LOG.debug( 'Registering options from plugin "%s" on OptionManager %r', self.name, optmanager ) - plugin.add_options(optmanager) + add_options(optmanager) class PluginManager(object): From 6ef9089eb7aa0b7b3f302e38d0e2819571da118a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 16 Jan 2016 07:02:36 -0600 Subject: [PATCH 074/204] Add main function entry-point to flake8 * Add --isolated option * Add --append-config * Add docstring to flake8.main --- flake8/main/__init__.py | 1 + flake8/main/cli.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/flake8/main/__init__.py b/flake8/main/__init__.py index e69de29..d3aa1de 100644 --- a/flake8/main/__init__.py +++ b/flake8/main/__init__.py @@ -0,0 +1 @@ +"""Module containing the logic for the Flake8 entry-points.""" diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 5bd7c84..05af72e 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -1,5 +1,9 @@ """Command-line implementation of flake8.""" +import flake8 from flake8 import defaults +from flake8.options import aggregator +from flake8.options import manager +from flake8.plugins import manager as plugin_manager def register_default_options(option_manager): @@ -120,3 +124,39 @@ def register_default_options(option_manager): # callback=callbacks.redirect_stdout, help='Redirect report to a file.', ) + + # Config file options + + add_option( + '--isolated', default=False, action='store_true', + help='Ignore all found configuration files.', + ) + + add_option( + '--append-config', action='append', + help='Provide extra config files to parse in addition to the files ' + 'found by Flake8 by default. These files are the last ones read ' + 'and so they take the highest precedence when multiple files ' + 'provide the same option.', + ) + + +def main(argv=None): + """Main entry-point for the flake8 command-line tool.""" + option_manager = manager.OptionManager( + prog='flake8', version=flake8.__version__ + ) + # Load our plugins + check_plugins = plugin_manager.Checkers() + listening_plugins = plugin_manager.Listeners() + formatting_plugins = plugin_manager.ReportFormatters() + + # Register all command-line and config-file options + register_default_options(option_manager) + check_plugins.register_options(option_manager) + listening_plugins.register_options(option_manager) + formatting_plugins.register_options(option_manager) + + # Parse out our options from our found config files and user-provided CLI + # options + options, args = aggregator.aggregate_options(option_manager) From 45d470927c85a680fcc4a4f902fe7da7f6e7606d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 16 Jan 2016 07:03:37 -0600 Subject: [PATCH 075/204] Load plugins of each type idempotently * Do not pass too many arguments to provide_options or register_options * Sub-class PluginTypeManager for both Listeners and Formatters --- flake8/plugins/manager.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 6e71f7e..81abdc4 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -164,6 +164,7 @@ class PluginTypeManager(object): def __init__(self): """Initialize the plugin type's manager.""" self.manager = PluginManager(self.namespace) + self.plugins_loaded = False @property def names(self): @@ -182,20 +183,29 @@ class PluginTypeManager(object): if (method is not None and isinstance(method, collections.Callable)): return method(optmanager, *args, **kwargs) + return generated_function def load_plugins(self): - def load_plugin(plugin): - return plugin.load() + """Load all plugins of this type that are managed by this manager.""" + if self.plugins_loaded: + return - return list(self.manager.map(load_plugin)) + def load_plugin(plugin): + return plugin.load_plugin() + + plugins = list(self.manager.map(load_plugin)) + # Do not set plugins_loaded if we run into an exception + self.plugins_loaded = True + return plugins def register_options(self, optmanager): """Register all of the checkers' options to the OptionManager.""" + self.load_plugins() call_register_options = self._generate_call_function( 'register_options', optmanager, ) - list(self.manager.map(call_register_options, optmanager)) + list(self.manager.map(call_register_options)) def provide_options(self, optmanager, options, extra_args): """Provide parsed options and extra arguments to the plugins.""" @@ -203,8 +213,7 @@ class PluginTypeManager(object): 'provide_options', optmanager, options, extra_args, ) - list(self.manager.map(call_provide_options, optmanager, options, - extra_args)) + list(self.manager.map(call_provide_options)) class Checkers(PluginTypeManager): @@ -213,13 +222,13 @@ class Checkers(PluginTypeManager): namespace = 'flake8.extension' -class Listeners(object): +class Listeners(PluginTypeManager): """All of the listeners registered through entry-points.""" namespace = 'flake8.listen' -class ReportFormatters(object): +class ReportFormatters(PluginTypeManager): """All of the report formatters registered through entry-points.""" namespace = 'flake8.report' From 4d86e44992291487a5303d5d65d5e0c84fcf393a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 16 Jan 2016 07:04:28 -0600 Subject: [PATCH 076/204] Add pyflakes shim from previous flake8 Modify the shim plugin to work with new flake8 --- flake8/plugins/pyflakes.py | 130 +++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 flake8/plugins/pyflakes.py diff --git a/flake8/plugins/pyflakes.py b/flake8/plugins/pyflakes.py new file mode 100644 index 0000000..c801e54 --- /dev/null +++ b/flake8/plugins/pyflakes.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +try: + # The 'demandimport' breaks pyflakes and flake8.plugins.pyflakes + from mercurial import demandimport +except ImportError: + pass +else: + demandimport.disable() +import os + +import pyflakes +import pyflakes.checker + +from flake8 import utils + + +def patch_pyflakes(): + """Add error codes to Pyflakes messages.""" + codes = dict([line.split()[::-1] for line in ( + 'F401 UnusedImport', + 'F402 ImportShadowedByLoopVar', + 'F403 ImportStarUsed', + 'F404 LateFutureImport', + 'F810 Redefined', # XXX Obsolete? + 'F811 RedefinedWhileUnused', + 'F812 RedefinedInListComp', + 'F821 UndefinedName', + 'F822 UndefinedExport', + 'F823 UndefinedLocal', + 'F831 DuplicateArgument', + 'F841 UnusedVariable', + )]) + + for name, obj in vars(pyflakes.messages).items(): + if name[0].isupper() and obj.message: + obj.flake8_msg = '%s %s' % (codes.get(name, 'F999'), obj.message) +patch_pyflakes() + + +class FlakesChecker(pyflakes.checker.Checker): + """Subclass the Pyflakes checker to conform with the flake8 API.""" + name = 'pyflakes' + version = pyflakes.__version__ + + def __init__(self, tree, filename): + filename = utils.normalize_paths(filename)[0] + withDoctest = self.withDoctest + included_by = [include for include in self.include_in_doctest + if include != '' and filename.startswith(include)] + if included_by: + withDoctest = True + + for exclude in self.exclude_from_doctest: + if exclude != '' and filename.startswith(exclude): + withDoctest = False + overlaped_by = [include for include in included_by + if include.startswith(exclude)] + + if overlaped_by: + withDoctest = True + + super(FlakesChecker, self).__init__(tree, filename, + withDoctest=withDoctest) + + @classmethod + def add_options(cls, parser): + parser.add_option( + '--builtins', parse_from_config=True, comma_separated_list=True, + help="define more built-ins, comma separated", + ) + parser.add_option( + '--doctests', default=False, action='store_true', + parse_from_config=True, + help="check syntax of the doctests", + ) + parser.add_option( + '--include-in-doctest', default='', + dest='include_in_doctest', parse_from_config=True, + comma_separated_list=True, normalize_paths=True, + help='Run doctests only on these files', + type='string', + ) + parser.add_option( + '--exclude-from-doctest', default='', + dest='exclude_from_doctest', parse_from_config=True, + comma_separated_list=True, normalize_paths=True, + help='Skip these files when running doctests', + type='string', + ) + + @classmethod + def parse_options(cls, options): + if options.builtins: + cls.builtIns = cls.builtIns.union(options.builtins.split(',')) + cls.withDoctest = options.doctests + + included_files = [] + for included_file in options.include_in_doctest: + if included_file == '': + continue + if not included_file.startswith((os.sep, './', '~/')): + included_files.append('./' + included_file) + else: + included_files.append(included_file) + cls.include_in_doctest = utils.normalize_paths(included_files) + + excluded_files = [] + for excluded_file in options.exclude_from_doctest.split(','): + if excluded_file == '': + continue + if not excluded_file.startswith((os.sep, './', '~/')): + excluded_files.append('./' + excluded_file) + else: + excluded_files.append(excluded_file) + cls.exclude_from_doctest = utils.normalize_paths(excluded_files) + + inc_exc = set(cls.include_in_doctest).intersection( + cls.exclude_from_doctest + ) + if inc_exc: + raise ValueError('"%s" was specified in both the ' + 'include-in-doctest and exclude-from-doctest ' + 'options. You are not allowed to specify it in ' + 'both for doctesting.' % inc_exc) + + def run(self): + for m in self.messages: + col = getattr(m, 'col', 0) + yield m.lineno, col, (m.flake8_msg % m.message_args), m.__class__ From ef0e018fa2367188ba0aebbbc32897c63e6d4313 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 16 Jan 2016 07:04:55 -0600 Subject: [PATCH 077/204] Update setup.py for new entry-points and packages Comment out mccabe because it is currently broken with flake8 3.0 --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 064b28e..93e4fdb 100644 --- a/setup.py +++ b/setup.py @@ -45,17 +45,22 @@ setuptools.setup( maintainer="Ian Cordasco", maintainer_email="graffatcolmingov@gmail.com", url="https://gitlab.com/pycqa/flake8", - packages=["flake8", "flake8.options"], + packages=[ + "flake8", + "flake8.main", + "flake8.options", + "flake8.plugins", + ], install_requires=[ "pyflakes >= 0.8.1, < 1.1", "pep8 >= 1.5.7, != 1.6.0, != 1.6.1, != 1.6.2", - "mccabe >= 0.2.1, < 0.4", + # "mccabe >= 0.2.1, < 0.4", ], entry_points={ 'distutils.commands': ['flake8 = flake8.main:Flake8Command'], - 'console_scripts': ['flake8 = flake8.main:main'], + 'console_scripts': ['flake8 = flake8.main.cli:main'], 'flake8.extension': [ - 'F = flake8._pyflakes:FlakesChecker', + 'F = flake8.plugins.pyflakes:FlakesChecker', ], }, classifiers=[ From ba56c34494f035a7ba48baf6f42094827a0be498 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 16 Jan 2016 07:05:12 -0600 Subject: [PATCH 078/204] Add venv testenv for testing the new version of flake8 --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 15218a1..77df69a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,11 @@ deps = commands = py.test {posargs} +[testenv:venv] +deps = + . +commands = {posargs} + [testenv:flake8] skipsdist = true skip_install = true From dfac6a2131403a037f0823c932ca85b1f6bccb38 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 16 Jan 2016 09:50:18 -0600 Subject: [PATCH 079/204] Record registered plugins and format their versions --- flake8/options/manager.py | 41 +++++++++++++++++++++++++++++++++++++++ flake8/plugins/manager.py | 5 +++++ 2 files changed, 46 insertions(+) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index d328a1f..c69dba2 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -155,6 +155,11 @@ class OptionManager(object): self.options = [] self.program_name = prog self.version = version + self.registered_plugins = set() + + @staticmethod + def format_plugin(plugin_tuple): + return dict(zip(["entry", "name", "version"], plugin_tuple)) def add_option(self, *args, **kwargs): """Create and register a new option. @@ -176,11 +181,47 @@ class OptionManager(object): self.config_options_dict[option.config_name] = option LOG.debug('Registered option "%s".', option) + def generate_versions(self, format_str='%(name)s: %(version)s'): + """Generate a comma-separated list of versions of plugins.""" + return ', '.join( + format_str % self.format_plugin(plugin) + for plugin in self.registered_plugins + ) + + def update_version_string(self): + """Update the flake8 version string.""" + self.parser.version = (self.version + ' (' + + self.generate_versions() + ')') + + def generate_epilog(self): + """Create an epilog with the version and name of each of plugin.""" + plugin_version_format = '%(name)s(%(entry)s): %(version)s' + self.parser.epilog = 'Installed plugins: ' + self.generate_versions( + plugin_version_format + ) + def parse_args(self, args=None, values=None): """Simple proxy to calling the OptionParser's parse_args method.""" + self.generate_epilog() + self.update_version_string() options, xargs = self.parser.parse_args(args, values) for option in self.options: old_value = getattr(options, option.dest) setattr(options, option.dest, option.normalize(old_value)) return options, xargs + + def register_plugin(self, entry_point_name, name, version): + """Register a plugin relying on the OptionManager. + + :param str entry_point_name: + The name of the entry-point loaded with pkg_resources. For + example, if the entry-point looks like: ``C90 = mccabe.Checker`` + then the ``entry_point_name`` would be ``C90``. + :param str name: + The name of the checker itself. This will be the ``name`` + attribute of the class or function loaded from the entry-point. + :param str version: + The version of the checker that we're using. + """ + self.registered_plugins.add((entry_point_name, name, version)) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 81abdc4..1afdaa9 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -98,6 +98,11 @@ class Plugin(object): self.name, optmanager ) add_options(optmanager) + optmanager.register_plugin( + entry_point_name=self.name, + name=self.plugin.name, + version=self.plugin.version + ) class PluginManager(object): From d6dbf797bcef595d7aa62f1b19e777709b92d549 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 17 Jan 2016 13:08:52 -0600 Subject: [PATCH 080/204] Add forgotten config file fixtures --- tests/fixtures/config_files/local-config.ini | 3 +++ tests/fixtures/config_files/user-config.ini | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 tests/fixtures/config_files/local-config.ini create mode 100644 tests/fixtures/config_files/user-config.ini diff --git a/tests/fixtures/config_files/local-config.ini b/tests/fixtures/config_files/local-config.ini new file mode 100644 index 0000000..348751a --- /dev/null +++ b/tests/fixtures/config_files/local-config.ini @@ -0,0 +1,3 @@ +[flake8] +exclude = docs/ +select = E,W,F diff --git a/tests/fixtures/config_files/user-config.ini b/tests/fixtures/config_files/user-config.ini new file mode 100644 index 0000000..b06c24f --- /dev/null +++ b/tests/fixtures/config_files/user-config.ini @@ -0,0 +1,5 @@ +[flake8] +exclude = + tests/fixtures/, + docs/ +ignore = D203 From dcf696d6786f936ed9dcfbdbf8e2d883266971fa Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 17 Jan 2016 21:45:32 -0600 Subject: [PATCH 081/204] Add tests around new OptionManager methods --- flake8/options/manager.py | 2 +- tests/unit/test_option_manager.py | 73 ++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index c69dba2..46e9f86 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -139,7 +139,7 @@ class OptionManager(object): """Manage Options and OptionParser while adding post-processing.""" def __init__(self, prog=None, version=None, - usage='%prog [options] input'): + usage='%prog [options] file file ...'): """Initialize an instance of an OptionManager. :param str prog: diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index b0e4ee7..660b1a6 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -6,11 +6,13 @@ import pytest from flake8.options import manager +TEST_VERSION = '3.0.0b1' + @pytest.fixture def optmanager(): """Generate a simple OptionManager with default test arguments.""" - return manager.OptionManager(prog='flake8', version='3.0.0b1') + return manager.OptionManager(prog='flake8', version=TEST_VERSION) def test_option_manager_creates_option_parser(optmanager): @@ -113,3 +115,72 @@ def test_parse_args_normalize_paths(optmanager): 'tox.ini', os.path.abspath('flake8/some-other.cfg'), ] + + +def test_format_plugin(): + """Verify that format_plugin turns a tuple into a dictionary.""" + plugin = manager.OptionManager.format_plugin(('T101', 'Testing', '0.0.0')) + assert plugin['entry'] == 'T101' + assert plugin['name'] == 'Testing' + assert plugin['version'] == '0.0.0' + + +def test_generate_versions(optmanager): + """Verify a comma-separated string is generated of registered plugins.""" + optmanager.registered_plugins = [ + ('T100', 'Testing 100', '0.0.0'), + ('T101', 'Testing 101', '0.0.0'), + ('T300', 'Testing 300', '0.0.0'), + ] + assert (optmanager.generate_versions() == + 'Testing 100: 0.0.0, Testing 101: 0.0.0, Testing 300: 0.0.0') + + +def test_generate_versions_with_format_string(optmanager): + """Verify a comma-separated string is generated of registered plugins.""" + optmanager.registered_plugins = [ + ('T100', 'Testing', '0.0.0'), + ('T101', 'Testing', '0.0.0'), + ('T300', 'Testing', '0.0.0'), + ] + assert ( + optmanager.generate_versions('%(name)s(%(entry)s): %(version)s') == + 'Testing(T100): 0.0.0, Testing(T101): 0.0.0, Testing(T300): 0.0.0' + ) + + +def test_update_version_string(optmanager): + """Verify we update the version string idempotently.""" + assert optmanager.version == TEST_VERSION + assert optmanager.parser.version == TEST_VERSION + + optmanager.registered_plugins = [ + ('T100', 'Testing 100', '0.0.0'), + ('T101', 'Testing 101', '0.0.0'), + ('T300', 'Testing 300', '0.0.0'), + ] + + optmanager.update_version_string() + + assert optmanager.version == TEST_VERSION + assert (optmanager.parser.version == TEST_VERSION + ' (' + 'Testing 100: 0.0.0, Testing 101: 0.0.0, Testing 300: 0.0.0)') + + +def test_generate_epilog(optmanager): + """Verify how we generate the epilog for help text.""" + assert optmanager.parser.epilog is None + + optmanager.registered_plugins = [ + ('T100', 'Testing 100', '0.0.0'), + ('T101', 'Testing 101', '0.0.0'), + ('T300', 'Testing 300', '0.0.0'), + ] + + expected_value = ( + 'Installed plugins: Testing 100(T100): 0.0.0, Testing 101(T101): ' + '0.0.0, Testing 300(T300): 0.0.0' + ) + + optmanager.generate_epilog() + assert optmanager.parser.epilog == expected_value From 4e294af81c14a627fa6a4771617a00a8a1abd293 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 06:43:47 -0600 Subject: [PATCH 082/204] Update fixture README with descriptions --- tests/fixtures/config_files/README.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/fixtures/config_files/README.rst b/tests/fixtures/config_files/README.rst index b69e080..b00adad 100644 --- a/tests/fixtures/config_files/README.rst +++ b/tests/fixtures/config_files/README.rst @@ -20,3 +20,19 @@ Purposes of existing fixtures This should only be used when providing config file(s) specified by the user on the command-line. + +``tests/fixtures/config_files/local-config.ini`` + + This should be used when providing config files that would have been found + by looking for config files in the current working project directory. + + +``tests/fixtures/config_files/no-flake8-section.ini`` + + This should be used when parsing an ini file without a ``[flake8]`` + section. + +``tests/fixtures/config_files/user-config.ini`` + + This is an example configuration file that would be found in the user's + home directory (or XDG Configuration Directory). From e66fc2efa81f9a05e52c6886b9606c4039259a5c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 07:19:16 -0600 Subject: [PATCH 083/204] Start creating documentation --- .gitignore | 1 + docs/build/.keep | 0 docs/source/conf.py | 295 ++++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 22 ++++ tox.ini | 22 ++++ 5 files changed, 340 insertions(+) create mode 100644 docs/build/.keep create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/.gitignore b/.gitignore index 612baf8..8e4e9dc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.egg-info *.log .eggs/ +docs/build/html/* diff --git a/docs/build/.keep b/docs/build/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..3c4be7f --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- +# +# flake8 documentation build configuration file, created by +# sphinx-quickstart on Tue Jan 19 07:14:10 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'flake8' +copyright = u'2016, Ian Cordasco' +author = u'Ian Cordasco' + +import flake8 +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = flake8.__version__ +# The full version, including alpha/beta/rc tags. +release = flake8.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'flake8doc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'flake8.tex', u'flake8 Documentation', + u'Ian Cordasco', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'flake8', u'flake8 Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'flake8', u'flake8 Documentation', + author, 'flake8', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..904ca87 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,22 @@ +.. flake8 documentation master file, created by + sphinx-quickstart on Tue Jan 19 07:14:10 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to flake8's documentation! +================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/tox.ini b/tox.ini index 77df69a..b2c0a91 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,28 @@ deps = commands = flake8 +[testenv:docs] +deps = + sphinx>=1.3.0 +commands = + sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html + +[testenv:serve-docs] +basepython = python3.4 +skipsdist = true +skip_install = true +use_develop = false +changedir = docs/_build/html +deps = +commands = + python -m http.server {posargs} + +[testenv:readme] +deps = + readme_renderer +commands = + python setup.py check -r -s + [flake8] # Ignore some flake8-docstrings errors ignore = D203 From 11be73dbbb7c40ac973c71599b4647d2c076d237 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 07:27:39 -0600 Subject: [PATCH 084/204] Note that we need at least Sphinx 1.3 Exclude sphinx conf.py from flake8 --- docs/source/conf.py | 2 +- tox.ini | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3c4be7f..835a673 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,7 @@ import os # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +needs_sphinx = '1.3' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom diff --git a/tox.ini b/tox.ini index b2c0a91..67c9478 100644 --- a/tox.ini +++ b/tox.ini @@ -49,3 +49,7 @@ commands = [flake8] # Ignore some flake8-docstrings errors ignore = D203 +# NOTE(sigmavirus24): Once we release 3.0.0 this exclude option can be specified +# across multiple lines. Presently it cannot be specified across multiple lines. +# :-( +exclude = .git,__pycache__,docs/source/conf.py From 83dd81a44526eab408ffc466e355263e76c8b2e0 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 07:45:25 -0600 Subject: [PATCH 085/204] Add missing docstrings to plugin submodule --- flake8/options/manager.py | 1 + flake8/plugins/__init__.py | 1 + flake8/plugins/manager.py | 16 ++++++++++++++++ flake8/plugins/pyflakes.py | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 46e9f86..1dd6c8d 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -159,6 +159,7 @@ class OptionManager(object): @staticmethod def format_plugin(plugin_tuple): + """Convert a plugin tuple into a dictionary mapping name to value.""" return dict(zip(["entry", "name", "version"], plugin_tuple)) def add_option(self, *args, **kwargs): diff --git a/flake8/plugins/__init__.py b/flake8/plugins/__init__.py index e69de29..fda6a44 100644 --- a/flake8/plugins/__init__.py +++ b/flake8/plugins/__init__.py @@ -0,0 +1 @@ +"""Submodule of built-in plugins and plugin managers.""" diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 1afdaa9..e4ad20e 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -6,6 +6,14 @@ import pkg_resources LOG = logging.getLogger(__name__) +__all__ = ( + 'Checkers', + 'Listeners', + 'Plugin', + 'PluginManager', + 'ReportFormatters', +) + class Plugin(object): """Wrap an EntryPoint from setuptools and other logic.""" @@ -25,17 +33,23 @@ class Plugin(object): self._plugin = None def __repr__(self): + """Provide an easy to read description of the current plugin.""" return 'Plugin(name="{0}", entry_point="{1}")'.format( self.name, self.entry_point ) @property def plugin(self): + """The loaded (and cached) plugin associated with the entry-point. + + This property implicitly loads the plugin and then caches it. + """ self.load_plugin() return self._plugin @property def version(self): + """Return the version attribute on the plugin.""" return self.plugin.version def execute(self, *args, **kwargs): @@ -124,10 +138,12 @@ class PluginManager(object): self._load_all_plugins() def __contains__(self, name): + """Check if the entry-point name is in this plugin manager.""" LOG.debug('Checking for "%s" in plugin manager.', name) return name in self.plugins def __getitem__(self, name): + """Retrieve a plugin by its entry-point name.""" LOG.debug('Retrieving plugin for "%s".', name) return self.plugins[name] diff --git a/flake8/plugins/pyflakes.py b/flake8/plugins/pyflakes.py index c801e54..85465fc 100644 --- a/flake8/plugins/pyflakes.py +++ b/flake8/plugins/pyflakes.py @@ -1,3 +1,4 @@ +"""Plugin built-in to Flake8 to treat pyflakes as a plugin.""" # -*- coding: utf-8 -*- from __future__ import absolute_import try: @@ -40,10 +41,12 @@ patch_pyflakes() class FlakesChecker(pyflakes.checker.Checker): """Subclass the Pyflakes checker to conform with the flake8 API.""" + name = 'pyflakes' version = pyflakes.__version__ def __init__(self, tree, filename): + """Initialize the PyFlakes plugin with an AST tree and filename.""" filename = utils.normalize_paths(filename)[0] withDoctest = self.withDoctest included_by = [include for include in self.include_in_doctest @@ -65,6 +68,7 @@ class FlakesChecker(pyflakes.checker.Checker): @classmethod def add_options(cls, parser): + """Register options for PyFlakes on the Flake8 OptionManager.""" parser.add_option( '--builtins', parse_from_config=True, comma_separated_list=True, help="define more built-ins, comma separated", @@ -91,6 +95,7 @@ class FlakesChecker(pyflakes.checker.Checker): @classmethod def parse_options(cls, options): + """Parse option values from Flake8's OptionManager.""" if options.builtins: cls.builtIns = cls.builtIns.union(options.builtins.split(',')) cls.withDoctest = options.doctests @@ -125,6 +130,7 @@ class FlakesChecker(pyflakes.checker.Checker): 'both for doctesting.' % inc_exc) def run(self): + """Run the plugin.""" for m in self.messages: col = getattr(m, 'col', 0) yield m.lineno, col, (m.flake8_msg % m.message_args), m.__class__ From 15d7f7679c5240d09d26ecfc98b4c63bfda53dbe Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 07:45:43 -0600 Subject: [PATCH 086/204] Builtins will not be parsed by flake8 for us --- flake8/plugins/pyflakes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8/plugins/pyflakes.py b/flake8/plugins/pyflakes.py index 85465fc..d787344 100644 --- a/flake8/plugins/pyflakes.py +++ b/flake8/plugins/pyflakes.py @@ -97,7 +97,7 @@ class FlakesChecker(pyflakes.checker.Checker): def parse_options(cls, options): """Parse option values from Flake8's OptionManager.""" if options.builtins: - cls.builtIns = cls.builtIns.union(options.builtins.split(',')) + cls.builtIns = cls.builtIns.union(options.builtins) cls.withDoctest = options.doctests included_files = [] From 5a1e6943d8209b72cc697295f06f1219ba59d2c0 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 07:46:41 -0600 Subject: [PATCH 087/204] Stop calling split on a list --exclude-from-doctests is also parsed by flake8 for us --- flake8/plugins/pyflakes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8/plugins/pyflakes.py b/flake8/plugins/pyflakes.py index d787344..7f6107a 100644 --- a/flake8/plugins/pyflakes.py +++ b/flake8/plugins/pyflakes.py @@ -111,7 +111,7 @@ class FlakesChecker(pyflakes.checker.Checker): cls.include_in_doctest = utils.normalize_paths(included_files) excluded_files = [] - for excluded_file in options.exclude_from_doctest.split(','): + for excluded_file in options.exclude_from_doctest: if excluded_file == '': continue if not excluded_file.startswith((os.sep, './', '~/')): From 56a75cc39187f5fd1aca3a2c931e27a211977e49 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 22:57:04 -0600 Subject: [PATCH 088/204] Add tests for the Plugin class --- tests/unit/test_plugin.py | 144 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/unit/test_plugin.py diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py new file mode 100644 index 0000000..28b95c9 --- /dev/null +++ b/tests/unit/test_plugin.py @@ -0,0 +1,144 @@ +"""Tests for flake8.plugins.manager.Plugin.""" +import mock + +from flake8.plugins import manager + + +def test_load_plugin_fallsback_on_old_setuptools(): + """Verify we fallback gracefully to on old versions of setuptools.""" + entry_point = mock.Mock(spec=['load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin() + entry_point.load.assert_called_once_with(require=False) + + +def test_load_plugin_avoids_deprecated_entry_point_methods(): + """Verify we use the preferred methods on new versions of setuptools.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin(verify_requirements=True) + assert entry_point.load.called is False + entry_point.require.assert_called_once_with() + entry_point.resolve.assert_called_once_with() + + +def test_load_plugin_is_idempotent(): + """Verify we use the preferred methods on new versions of setuptools.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin(verify_requirements=True) + plugin.load_plugin(verify_requirements=True) + plugin.load_plugin() + assert entry_point.load.called is False + entry_point.require.assert_called_once_with() + entry_point.resolve.assert_called_once_with() + + +def test_load_plugin_only_calls_require_when_verifying_requirements(): + """Verify we do not call require when verify_requirements is False.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + plugin.load_plugin() + assert entry_point.load.called is False + assert entry_point.require.called is False + entry_point.resolve.assert_called_once_with() + + +def test_plugin_property_loads_plugin_on_first_use(): + """Verify that we load our plugin when we first try to use it.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin = manager.Plugin('T000', entry_point) + + assert plugin.plugin is not None + entry_point.resolve.assert_called_once_with() + + +def test_execute_calls_plugin_with_passed_arguments(): + """Verify that we pass arguments directly to the plugin.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock() + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + plugin.execute('arg1', 'arg2', kwarg1='value1', kwarg2='value2') + plugin_obj.assert_called_once_with( + 'arg1', 'arg2', kwarg1='value1', kwarg2='value2' + ) + + # Extra assertions + assert entry_point.load.called is False + assert entry_point.require.called is False + assert entry_point.resolve.called is False + + +def test_version_proxies_to_the_plugin(): + """Verify that we pass arguments directly to the plugin.""" + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['version']) + plugin_obj.version = 'a.b.c' + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + assert plugin.version == 'a.b.c' + + +def test_register_options(): + """Verify we call add_options on the plugin only if it exists.""" + # Set up our mocks and Plugin object + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['name', 'version', 'add_options', + 'parse_options']) + option_manager = mock.Mock(spec=['register_plugin']) + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + # Call the method we're testing. + plugin.register_options(option_manager) + + # Assert that we call add_options + plugin_obj.add_options.assert_called_once_with(option_manager) + # Assert that we register the plugin + option_manager.register_plugin.assert_called_once_with( + entry_point_name='T000', + version=plugin_obj.version, + name=plugin_obj.name, + ) + + +def test_register_options_checks_plugin_for_method(): + """Verify we call add_options on the plugin only if it exists.""" + # Set up our mocks and Plugin object + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['name', 'version', 'parse_options']) + option_manager = mock.Mock(spec=['register_plugin']) + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + # Call the method we're testing. + plugin.register_options(option_manager) + + # Assert that we register the plugin + assert option_manager.register_plugin.called is False + + +def test_provide_options(): + """Verify we call add_options on the plugin only if it exists.""" + # Set up our mocks and Plugin object + entry_point = mock.Mock(spec=['require', 'resolve', 'load']) + plugin_obj = mock.Mock(spec_set=['name', 'version', 'add_options', + 'parse_options']) + option_manager = mock.Mock() + plugin = manager.Plugin('T000', entry_point) + plugin._plugin = plugin_obj + + # Call the method we're testing. + plugin.provide_options(option_manager, 'options', None) + + # Assert that we call add_options + plugin_obj.parse_options.assert_called_once_with( + option_manager, 'options', None + ) From 6a72e36681aa237ae5e0fc04381ee1d6ab4d1e08 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 19 Jan 2016 23:01:07 -0600 Subject: [PATCH 089/204] Start tests for PluginManager --- tests/unit/test_plugin_manager.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/unit/test_plugin_manager.py diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py new file mode 100644 index 0000000..8bafd2b --- /dev/null +++ b/tests/unit/test_plugin_manager.py @@ -0,0 +1,13 @@ +"""Tests for flake8.plugins.manager.PluginManager.""" +import mock + +from flake8.plugins import manager + + +@mock.patch('pkg_resources.iter_entry_points') +def test_calls_pkg_resources_on_instantiation(iter_entry_points): + """Verify that we call iter_entry_points when we create a manager.""" + iter_entry_points.return_value = [] + manager.PluginManager(namespace='testing.pkg_resources') + + iter_entry_points.assert_called_once_with('testing.pkg_resources') From 50e8fef125341dc529f770dc7a6f48783c66a89e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 21 Jan 2016 08:26:05 -0600 Subject: [PATCH 090/204] Add more PluginManager tests --- tests/unit/test_plugin_manager.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py index 8bafd2b..022848f 100644 --- a/tests/unit/test_plugin_manager.py +++ b/tests/unit/test_plugin_manager.py @@ -4,6 +4,13 @@ import mock from flake8.plugins import manager +def create_entry_point_mock(name): + """Create a mocked EntryPoint.""" + ep = mock.Mock(spec=['name']) + ep.name = name + return ep + + @mock.patch('pkg_resources.iter_entry_points') def test_calls_pkg_resources_on_instantiation(iter_entry_points): """Verify that we call iter_entry_points when we create a manager.""" @@ -11,3 +18,45 @@ def test_calls_pkg_resources_on_instantiation(iter_entry_points): manager.PluginManager(namespace='testing.pkg_resources') iter_entry_points.assert_called_once_with('testing.pkg_resources') + + +@mock.patch('pkg_resources.iter_entry_points') +def test_calls_pkg_resources_creates_plugins_automaticaly(iter_entry_points): + """Verify that we create Plugins on instantiation.""" + iter_entry_points.return_value = [ + create_entry_point_mock('T100'), + create_entry_point_mock('T200'), + ] + plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') + + iter_entry_points.assert_called_once_with('testing.pkg_resources') + assert 'T100' in plugin_mgr.plugins + assert 'T200' in plugin_mgr.plugins + assert isinstance(plugin_mgr.plugins['T100'], manager.Plugin) + assert isinstance(plugin_mgr.plugins['T200'], manager.Plugin) + + +@mock.patch('pkg_resources.iter_entry_points') +def test_proxies_contains_to_plugins_dictionary(iter_entry_points): + """Verify that we can use the PluginManager like a dictionary.""" + iter_entry_points.return_value = [ + create_entry_point_mock('T100'), + create_entry_point_mock('T200'), + ] + plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') + + assert 'T100' in plugin_mgr + assert 'T200' in plugin_mgr + + +@mock.patch('pkg_resources.iter_entry_points') +def test_proxies_getitm_to_plugins_dictionary(iter_entry_points): + """Verify that we can use the PluginManager like a dictionary.""" + iter_entry_points.return_value = [ + create_entry_point_mock('T100'), + create_entry_point_mock('T200'), + ] + plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') + + assert isinstance(plugin_mgr['T100'], manager.Plugin) + assert isinstance(plugin_mgr['T200'], manager.Plugin) From 916bec859cfa0a0df879b2674527bce9c0926a46 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 21 Jan 2016 08:32:35 -0600 Subject: [PATCH 091/204] Finish testing PluginManager --- tests/unit/test_plugin_manager.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py index 022848f..05073cf 100644 --- a/tests/unit/test_plugin_manager.py +++ b/tests/unit/test_plugin_manager.py @@ -50,7 +50,7 @@ def test_proxies_contains_to_plugins_dictionary(iter_entry_points): @mock.patch('pkg_resources.iter_entry_points') -def test_proxies_getitm_to_plugins_dictionary(iter_entry_points): +def test_proxies_getitem_to_plugins_dictionary(iter_entry_points): """Verify that we can use the PluginManager like a dictionary.""" iter_entry_points.return_value = [ create_entry_point_mock('T100'), @@ -60,3 +60,17 @@ def test_proxies_getitm_to_plugins_dictionary(iter_entry_points): assert isinstance(plugin_mgr['T100'], manager.Plugin) assert isinstance(plugin_mgr['T200'], manager.Plugin) + + +@mock.patch('pkg_resources.iter_entry_points') +def test_handles_mapping_functions_across_plugins(iter_entry_points): + """Verify we can use the PluginManager call functions on all plugins.""" + entry_point_mocks = [ + create_entry_point_mock('T100'), + create_entry_point_mock('T200'), + ] + iter_entry_points.return_value = entry_point_mocks + plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') + plugins = [plugin_mgr[name] for name in plugin_mgr.names] + + assert list(plugin_mgr.map(lambda x: x)) == plugins From c67735792b2d6481e9a5d0de468062c776834759 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 21 Jan 2016 08:32:46 -0600 Subject: [PATCH 092/204] Add documentation to PluginManager.map --- flake8/plugins/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index e4ad20e..8fd3a24 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -169,7 +169,8 @@ class PluginManager(object): pass Any extra positional or keyword arguments specified with map will - be passed along to this function after the plugin. + be passed along to this function after the plugin. The plugin + passed is a :class:`~flake8.plugins.manager.Plugin`. :param args: Positional arguments to pass to ``func`` after each plugin. :param kwargs: From 64edc550617123b661f6e7ee9899ae4b0fc19af4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 21 Jan 2016 22:15:17 -0600 Subject: [PATCH 093/204] Add method to build Tries from Notifiers --- flake8/plugins/manager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 8fd3a24..ce74e08 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -4,6 +4,8 @@ import logging import pkg_resources +from flake8 import _trie + LOG = logging.getLogger(__name__) __all__ = ( @@ -249,6 +251,13 @@ class Listeners(PluginTypeManager): namespace = 'flake8.listen' + def build_trie(self): + """Build a Trie for our Listeners.""" + trie = _trie.Trie() + for name in self.names: + trie.add(name, self.manager[name]) + return trie + class ReportFormatters(PluginTypeManager): """All of the report formatters registered through entry-points.""" From 6546cf41d4b41e3734e382eea6765aaf51b563e5 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 21 Jan 2016 22:15:38 -0600 Subject: [PATCH 094/204] Add tests for the PluginTypeManager --- tests/unit/test_plugin_type_manager.py | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/unit/test_plugin_type_manager.py diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py new file mode 100644 index 0000000..c0aa724 --- /dev/null +++ b/tests/unit/test_plugin_type_manager.py @@ -0,0 +1,55 @@ +"""Tests for flake8.plugins.manager.PluginTypeManager.""" +import collections +import mock + +from flake8.plugins import manager + +TEST_NAMESPACE = "testing.plugin-type-manager" + + +class TestType(manager.PluginTypeManager): + """Fake PluginTypeManager.""" + + namespace = TEST_NAMESPACE + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_instantiates_a_manager(PluginManager): + """Verify we create a PluginManager on instantiation.""" + TestType() + + PluginManager.assert_called_once_with(TEST_NAMESPACE) + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxies_names_to_manager(PluginManager): + """Verify we proxy the names attribute.""" + PluginManager.return_value = mock.Mock(names=[ + 'T100', 'T200', 'T300' + ]) + type_mgr = TestType() + + assert type_mgr.names == ['T100', 'T200', 'T300'] + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxies_plugins_to_manager(PluginManager): + """Verify we proxy the plugins attribute.""" + PluginManager.return_value = mock.Mock(plugins=[ + 'T100', 'T200', 'T300' + ]) + type_mgr = TestType() + + assert type_mgr.plugins == ['T100', 'T200', 'T300'] + + +def test_generate_call_function(): + """Verify the function we generate.""" + optmanager = object() + plugin = mock.Mock(method_name=lambda x: x) + func = manager.PluginTypeManager._generate_call_function( + 'method_name', optmanager, + ) + + assert isinstance(func, collections.Callable) + assert func(plugin) is optmanager From 3b64ff2a1f1b1b90f74f6f3e9fad7a6723c6f68a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 22 Jan 2016 09:04:52 -0600 Subject: [PATCH 095/204] Make Plugin.load_plugin raise a Flake8 exception Let's catch exceptions, log them, and re-raise as a FailedToLoad exception. This also refactors the actually loading of plugins into a private method on the Plugin class. --- flake8/exceptions.py | 25 +++++++++++++++++++++++++ flake8/plugins/manager.py | 37 +++++++++++++++++++++++++------------ tests/unit/test_plugin.py | 12 ++++++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 flake8/exceptions.py diff --git a/flake8/exceptions.py b/flake8/exceptions.py new file mode 100644 index 0000000..ac78769 --- /dev/null +++ b/flake8/exceptions.py @@ -0,0 +1,25 @@ +"""Exception classes for all of Flake8.""" + + +class Flake8Exception(Exception): + """Plain Flake8 exception.""" + + pass + + +class FailedToLoadPlugin(Flake8Exception): + """Exception raised when a plugin fails to load.""" + + FORMAT = 'Flake8 failed to load plugin "%(name)s" due to %(exc)s.' + + def __init__(self, *args, **kwargs): + """Initialize our FailedToLoadPlugin exception.""" + self.plugin = kwargs.pop('plugin') + self.ep_name = self.plugin.name + self.original_exception = kwargs.pop('exception') + super(FailedToLoadPlugin, self).__init__(*args, **kwargs) + + def __str__(self): + """Return a nice string for our exception.""" + return self.FORMAT % {'name': self.ep_name, + 'exc': self.original_exception} diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index ce74e08..a80020e 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -5,6 +5,7 @@ import logging import pkg_resources from flake8 import _trie +from flake8 import exceptions LOG = logging.getLogger(__name__) @@ -58,6 +59,21 @@ class Plugin(object): r"""Call the plugin with \*args and \*\*kwargs.""" return self.plugin(*args, **kwargs) + def _load(self, verify_requirements): + # Avoid relying on hasattr() here. + resolve = getattr(self.entry_point, 'resolve', None) + require = getattr(self.entry_point, 'require', None) + if resolve and require: + if verify_requirements: + LOG.debug('Verifying plugin "%s"\'s requirements.', + self.name) + require() + self._plugin = resolve() + else: + self._plugin = self.entry_point.load( + require=verify_requirements + ) + def load_plugin(self, verify_requirements=False): """Retrieve the plugin for this entry-point. @@ -73,19 +89,16 @@ class Plugin(object): """ if self._plugin is None: LOG.debug('Loading plugin "%s" from entry-point.', self.name) - # Avoid relying on hasattr() here. - resolve = getattr(self.entry_point, 'resolve', None) - require = getattr(self.entry_point, 'require', None) - if resolve and require: - if verify_requirements: - LOG.debug('Verifying plugin "%s"\'s requirements.', - self.name) - require() - self._plugin = resolve() - else: - self._plugin = self.entry_point.load( - require=verify_requirements + try: + self._load(verify_requirements) + except Exception as load_exception: + LOG.exception(load_exception, exc_info=True) + failed_to_load = exceptions.FailedToLoadPlugin( + plugin=self, + exception=load_exception, ) + LOG.critical(str(failed_to_load)) + raise failed_to_load def provide_options(self, optmanager, options, extra_args): """Pass the parsed options and extra arguments to the plugin.""" diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 28b95c9..01454a1 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -1,6 +1,8 @@ """Tests for flake8.plugins.manager.Plugin.""" import mock +import pytest +from flake8 import exceptions from flake8.plugins import manager @@ -48,6 +50,16 @@ def test_load_plugin_only_calls_require_when_verifying_requirements(): entry_point.resolve.assert_called_once_with() +def test_load_plugin_catches_and_reraises_exceptions(): + """Verify we raise our own FailedToLoadPlugin.""" + entry_point = mock.Mock(spec=['require', 'resolve']) + entry_point.resolve.side_effect = ValueError('Test failure') + plugin = manager.Plugin('T000', entry_point) + + with pytest.raises(exceptions.FailedToLoadPlugin): + plugin.load_plugin() + + def test_plugin_property_loads_plugin_on_first_use(): """Verify that we load our plugin when we first try to use it.""" entry_point = mock.Mock(spec=['require', 'resolve', 'load']) From 6e8e2b966988d2f4e30d644d95abd8c03fe36e55 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 22 Jan 2016 09:06:24 -0600 Subject: [PATCH 096/204] Write more PluginTypeManager tests --- tests/unit/test_plugin_type_manager.py | 80 +++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py index c0aa724..1f8c646 100644 --- a/tests/unit/test_plugin_type_manager.py +++ b/tests/unit/test_plugin_type_manager.py @@ -1,12 +1,26 @@ """Tests for flake8.plugins.manager.PluginTypeManager.""" import collections -import mock +import mock +import pytest + +from flake8 import exceptions from flake8.plugins import manager TEST_NAMESPACE = "testing.plugin-type-manager" +def create_plugin_mock(raise_exception=False): + """Create an auto-spec'd mock of a flake8 Plugin.""" + plugin = mock.create_autospec(manager.Plugin, instance=True) + if raise_exception: + plugin.load_plugin.side_effect = exceptions.FailedToLoadPlugin( + plugin=mock.Mock(name='T101'), + exception=ValueError('Test failure'), + ) + return plugin + + class TestType(manager.PluginTypeManager): """Fake PluginTypeManager.""" @@ -53,3 +67,67 @@ def test_generate_call_function(): assert isinstance(func, collections.Callable) assert func(plugin) is optmanager + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_load_plugins(PluginManager): + """Verify load plugins loads *every* plugin.""" + # Create a bunch of fake plugins + plugins = [create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + + # Have a function that will actually call the method underneath + def fake_map(func): + for plugin in plugins: + yield func(plugin) + + # Mock out the PluginManager instance + manager_mock = mock.Mock(spec=['map']) + # Replace the map method + manager_mock.map = fake_map + # Return our PluginManager mock + PluginManager.return_value = manager_mock + + type_mgr = TestType() + # Load the tests (do what we're actually testing) + assert len(type_mgr.load_plugins()) == 8 + # Assert that our closure does what we think it does + for plugin in plugins: + plugin.load_plugin.assert_called_once_with() + assert type_mgr.plugins_loaded is True + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_load_plugins_fails(PluginManager): + """Verify load plugins bubbles up exceptions.""" + plugins = [create_plugin_mock(), create_plugin_mock(True), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + + # Have a function that will actually call the method underneath + def fake_map(func): + for plugin in plugins: + yield func(plugin) + + # Mock out the PluginManager instance + manager_mock = mock.Mock(spec=['map']) + # Replace the map method + manager_mock.map = fake_map + # Return our PluginManager mock + PluginManager.return_value = manager_mock + + type_mgr = TestType() + with pytest.raises(exceptions.FailedToLoadPlugin): + type_mgr.load_plugins() + + # Assert we didn't finish loading plugins + assert type_mgr.plugins_loaded is False + # Assert the first two plugins had their load_plugin method called + plugins[0].load_plugin.assert_called_once_with() + plugins[1].load_plugin.assert_called_once_with() + # Assert the rest of the plugins were not loaded + for plugin in plugins[2:]: + assert plugin.load_plugin.called is False From 04556f1a1b9c26f73ee0aabacac8e0ba542d6a65 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 23 Jan 2016 23:01:30 -0600 Subject: [PATCH 097/204] Build a Notifier instead of a Trie --- flake8/plugins/manager.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index a80020e..8d5d12c 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -4,7 +4,7 @@ import logging import pkg_resources -from flake8 import _trie +from flake8 import notifier from flake8 import exceptions LOG = logging.getLogger(__name__) @@ -264,12 +264,19 @@ class Listeners(PluginTypeManager): namespace = 'flake8.listen' - def build_trie(self): - """Build a Trie for our Listeners.""" - trie = _trie.Trie() + def build_notifier(self): + """Build a Notifier for our Listeners. + + :returns: + Object to notify our listeners of certain error codes and + warnings. + :rtype: + :class:`~flake8.notifier.Notifier` + """ + notifier_trie = notifier.Notifier() for name in self.names: - trie.add(name, self.manager[name]) - return trie + notifier_trie.register_listener(name, self.manager[name]) + return notifier_trie class ReportFormatters(PluginTypeManager): From b0a258fe79c7f8713708b03f64cdbc832a449bf4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 23 Jan 2016 23:02:06 -0600 Subject: [PATCH 098/204] Refactor our PluginTypeManager tests Add tests for PluginTypeManager#{register,provide}_optoins --- tests/unit/test_plugin_type_manager.py | 78 ++++++++++++++++++-------- 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py index 1f8c646..cb121c6 100644 --- a/tests/unit/test_plugin_type_manager.py +++ b/tests/unit/test_plugin_type_manager.py @@ -21,6 +21,20 @@ def create_plugin_mock(raise_exception=False): return plugin +def create_mapping_manager_mock(plugins): + """Create a mock for the PluginManager.""" + # Have a function that will actually call the method underneath + def fake_map(func): + for plugin in plugins: + yield func(plugin) + + # Mock out the PluginManager instance + manager_mock = mock.Mock(spec=['map']) + # Replace the map method + manager_mock.map = fake_map + return manager_mock + + class TestType(manager.PluginTypeManager): """Fake PluginTypeManager.""" @@ -77,18 +91,8 @@ def test_load_plugins(PluginManager): create_plugin_mock(), create_plugin_mock(), create_plugin_mock(), create_plugin_mock(), create_plugin_mock(), create_plugin_mock()] - - # Have a function that will actually call the method underneath - def fake_map(func): - for plugin in plugins: - yield func(plugin) - - # Mock out the PluginManager instance - manager_mock = mock.Mock(spec=['map']) - # Replace the map method - manager_mock.map = fake_map # Return our PluginManager mock - PluginManager.return_value = manager_mock + PluginManager.return_value = create_mapping_manager_mock(plugins) type_mgr = TestType() # Load the tests (do what we're actually testing) @@ -106,18 +110,8 @@ def test_load_plugins_fails(PluginManager): create_plugin_mock(), create_plugin_mock(), create_plugin_mock(), create_plugin_mock(), create_plugin_mock(), create_plugin_mock()] - - # Have a function that will actually call the method underneath - def fake_map(func): - for plugin in plugins: - yield func(plugin) - - # Mock out the PluginManager instance - manager_mock = mock.Mock(spec=['map']) - # Replace the map method - manager_mock.map = fake_map # Return our PluginManager mock - PluginManager.return_value = manager_mock + PluginManager.return_value = create_mapping_manager_mock(plugins) type_mgr = TestType() with pytest.raises(exceptions.FailedToLoadPlugin): @@ -131,3 +125,43 @@ def test_load_plugins_fails(PluginManager): # Assert the rest of the plugins were not loaded for plugin in plugins[2:]: assert plugin.load_plugin.called is False + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_register_options(PluginManager): + """Test that we map over every plugin to register options.""" + plugins = [create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + # Return our PluginManager mock + PluginManager.return_value = create_mapping_manager_mock(plugins) + optmanager = object() + + type_mgr = TestType() + type_mgr.register_options(optmanager) + + for plugin in plugins: + plugin.register_options.assert_called_with(optmanager) + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_provide_options(PluginManager): + """Test that we map over every plugin to provide parsed options.""" + plugins = [create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock(), + create_plugin_mock(), create_plugin_mock()] + # Return our PluginManager mock + PluginManager.return_value = create_mapping_manager_mock(plugins) + optmanager = object() + options = object() + extra_args = [] + + type_mgr = TestType() + type_mgr.provide_options(optmanager, options, extra_args) + + for plugin in plugins: + plugin.provide_options.assert_called_with(optmanager, + options, + extra_args) From ebdc935ffc44bbfaae863b3b07bc165c601dd88c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 15:13:58 -0600 Subject: [PATCH 099/204] Refactor NotifierBuilder into its own mixin This allows for easier unit testing --- flake8/plugins/manager.py | 24 +++++++++++++--------- tests/unit/test_plugin_type_manager.py | 28 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 8d5d12c..f73d006 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -253,16 +253,8 @@ class PluginTypeManager(object): list(self.manager.map(call_provide_options)) -class Checkers(PluginTypeManager): - """All of the checkers registered through entry-ponits.""" - - namespace = 'flake8.extension' - - -class Listeners(PluginTypeManager): - """All of the listeners registered through entry-points.""" - - namespace = 'flake8.listen' +class NotifierBuilder(object): + """Mixin class that builds a Notifier from a PluginManager.""" def build_notifier(self): """Build a Notifier for our Listeners. @@ -279,6 +271,18 @@ class Listeners(PluginTypeManager): return notifier_trie +class Checkers(PluginTypeManager): + """All of the checkers registered through entry-ponits.""" + + namespace = 'flake8.extension' + + +class Listeners(PluginTypeManager, NotifierBuilder): + """All of the listeners registered through entry-points.""" + + namespace = 'flake8.listen' + + class ReportFormatters(PluginTypeManager): """All of the report formatters registered through entry-points.""" diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py index cb121c6..2735dec 100644 --- a/tests/unit/test_plugin_type_manager.py +++ b/tests/unit/test_plugin_type_manager.py @@ -165,3 +165,31 @@ def test_provide_options(PluginManager): plugin.provide_options.assert_called_with(optmanager, options, extra_args) + + +class FakePluginTypeManager(manager.NotifierBuilder): + """Provide an easy way to test the NotifierBuilder.""" + + def __init__(self, manager): + """Initialize with our fake manager.""" + self.names = sorted(manager.keys()) + self.manager = manager + + +@pytest.fixture +def notifier_builder(): + """Create a fake plugin type manager.""" + return FakePluginTypeManager(manager={ + 'T100': object(), + 'T101': object(), + 'T110': object(), + }) + + +def test_build_notifier(notifier_builder): + """Verify we properly build a Notifier object.""" + notifier = notifier_builder.build_notifier() + for name in ('T100', 'T101', 'T110'): + assert list(notifier.listeners_for(name)) == [ + notifier_builder.manager[name] + ] From 5037fab13242a2feb48811f2e71be7e1a69c5214 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 15:16:18 -0600 Subject: [PATCH 100/204] Update design document --- DESIGN.rst | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/DESIGN.rst b/DESIGN.rst index e472671..cac20f9 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -81,7 +81,7 @@ What we *might* want is for a autofix plugin to register something like :: - 'flake8.autofix_extension': [ + 'flake8.listen': [ 'E1 = my_fixer.E1Listener', 'E2 = my_fixer.E2Listener', ] @@ -100,20 +100,15 @@ compose formatters would allow for certain formatters to highlight more important information over less important information as the user deems necessary. +:: + + 'flake8.format': [ + 'json = my_formatter.JsonFormatter', + 'xml = my_formatter.XMLFormatter', + ] + See https://gitlab.com/pycqa/flake8/issues/66 -.. _report-generation: - -Support for Report Generation -+++++++++++++++++++++++++++++ - -Flake8 should support pluggable report formats. See also pluggable report -formats for https://github.com/openstack/bandit - -Report generation plugins may also choose to implement a way to store previous -runs of flake8. As such these plugins should be designed to be composable as -well. - .. _options-passing: Support for Plugins Require Parsed Options From 551bce11a3e7ac328a3859faa0ab854c719863c1 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 15:19:44 -0600 Subject: [PATCH 101/204] Add extend_default_ignore list to OptionManager --- flake8/options/manager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 1dd6c8d..da3f73e 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -156,6 +156,7 @@ class OptionManager(object): self.program_name = prog self.version = version self.registered_plugins = set() + self.extended_default_ignore = set() @staticmethod def format_plugin(plugin_tuple): @@ -182,6 +183,15 @@ class OptionManager(object): self.config_options_dict[option.config_name] = option LOG.debug('Registered option "%s".', option) + def extend_default_ignore(self, error_codes): + """Extend the default ignore list with the error codes provided. + + :param list error_codes: + List of strings that are the error/warning codes with which to + extend the default ignore list. + """ + self.extended_default_ignore.update(error_codes) + def generate_versions(self, format_str='%(name)s: %(version)s'): """Generate a comma-separated list of versions of plugins.""" return ', '.join( From 3a490971b2984b52bcc56cc7eb74c275c3fbe5b7 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 22:13:06 -0600 Subject: [PATCH 102/204] Add --config option flag --- flake8/main/cli.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 05af72e..7f58fb2 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -127,11 +127,6 @@ def register_default_options(option_manager): # Config file options - add_option( - '--isolated', default=False, action='store_true', - help='Ignore all found configuration files.', - ) - add_option( '--append-config', action='append', help='Provide extra config files to parse in addition to the files ' @@ -140,6 +135,18 @@ def register_default_options(option_manager): 'provide the same option.', ) + add_option( + '--config', default=None, + help='Path to the config file that will be the authoritative config ' + 'source. This will cause Flake8 to ignore all other ' + 'configuration files.' + ) + + add_option( + '--isolated', default=False, action='store_true', + help='Ignore all found configuration files.', + ) + def main(argv=None): """Main entry-point for the flake8 command-line tool.""" From 51bfc6a5b1129133df4944d612f12342cbeb495a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 22:13:21 -0600 Subject: [PATCH 103/204] Pass arglist to first parse_args call for testing --- flake8/options/aggregator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8/options/aggregator.py b/flake8/options/aggregator.py index 863fa05..380b2d7 100644 --- a/flake8/options/aggregator.py +++ b/flake8/options/aggregator.py @@ -17,7 +17,7 @@ def aggregate_options(manager, arglist=None, values=None): default_values, _ = manager.parse_args([], values=values) # Get original CLI values so we can find additional config file paths and # see if --config was specified. - original_values, original_args = manager.parse_args() + original_values, original_args = manager.parse_args(arglist) extra_config_files = utils.normalize_paths(original_values.append_config) # Make our new configuration file mergerator From a66f43bc393e4d68b879a6b12c025d59799a5484 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 22:13:57 -0600 Subject: [PATCH 104/204] Extend the default ignore list with plugins' defaults --- flake8/options/aggregator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flake8/options/aggregator.py b/flake8/options/aggregator.py index 380b2d7..e12952a 100644 --- a/flake8/options/aggregator.py +++ b/flake8/options/aggregator.py @@ -31,6 +31,12 @@ def aggregate_options(manager, arglist=None, values=None): parsed_config = config_parser.parse(original_values.config, original_values.isolated) + # Extend the default ignore value with the extended default ignore list, + # registered by plugins. + extended_default_ignore = manager.extended_default_ignore.copy() + extended_default_ignore.update(default_values.ignore) + default_values.ignore = list(extended_default_ignore) + # Merge values parsed from config onto the default values returned for config_name, value in parsed_config.items(): dest_name = config_name From 2cba89c56891d3270c8c36e4236d57a468ccbab2 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 22:14:21 -0600 Subject: [PATCH 105/204] Log extended default values and add a test --- flake8/options/manager.py | 1 + tests/unit/test_option_manager.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/flake8/options/manager.py b/flake8/options/manager.py index da3f73e..9faaf7c 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -190,6 +190,7 @@ class OptionManager(object): List of strings that are the error/warning codes with which to extend the default ignore list. """ + LOG.debug('Extending default ignore list with %r', error_codes) self.extended_default_ignore.update(error_codes) def generate_versions(self, format_str='%(name)s: %(version)s'): diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index 660b1a6..99c5d18 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -184,3 +184,13 @@ def test_generate_epilog(optmanager): optmanager.generate_epilog() assert optmanager.parser.epilog == expected_value + + +def test_extend_default_ignore(optmanager): + """Verify that we update the extended default ignore list.""" + assert optmanager.extended_default_ignore == set() + + optmanager.extend_default_ignore(['T100', 'T101', 'T102']) + assert optmanager.extended_default_ignore == set(['T100', + 'T101', + 'T102']) From 17a7826dc7e878a0da3fb8b4853ea80c9967ba24 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 24 Jan 2016 22:14:36 -0600 Subject: [PATCH 106/204] Add integration tests for aggregate_options --- tests/integration/test_aggregator.py | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/integration/test_aggregator.py diff --git a/tests/integration/test_aggregator.py b/tests/integration/test_aggregator.py new file mode 100644 index 0000000..3f64fa3 --- /dev/null +++ b/tests/integration/test_aggregator.py @@ -0,0 +1,48 @@ +"""Test aggregation of config files and command-line options.""" +import os + +import pytest + +from flake8.main import cli +from flake8.options import aggregator +from flake8.options import manager + +CLI_SPECIFIED_CONFIG = 'tests/fixtures/config_files/cli-specified.ini' + + +@pytest.fixture +def optmanager(): + """Create a new OptionManager.""" + option_manager = manager.OptionManager( + prog='flake8', + version='3.0.0', + ) + cli.register_default_options(option_manager) + return option_manager + + +def test_aggregate_options_with_config(optmanager): + """Verify we aggregate options and config values appropriately.""" + arguments = ['flake8', '--config', CLI_SPECIFIED_CONFIG, '--select', + 'E11,E34,E402,W,F', '--exclude', 'tests/*'] + options, args = aggregator.aggregate_options(optmanager, arguments) + + assert options.config == CLI_SPECIFIED_CONFIG + assert options.select == ['E11', 'E34', 'E402', 'W', 'F'] + assert options.ignore == ['E123', 'W234', 'E111'] + assert options.exclude == [os.path.abspath('tests/*')] + + +def test_aggregate_options_when_isolated(optmanager): + """Verify we aggregate options and config values appropriately.""" + arguments = ['flake8', '--isolated', '--select', 'E11,E34,E402,W,F', + '--exclude', 'tests/*'] + optmanager.extend_default_ignore(['E8']) + options, args = aggregator.aggregate_options(optmanager, arguments) + + assert options.isolated is True + assert options.select == ['E11', 'E34', 'E402', 'W', 'F'] + assert sorted(options.ignore) == [ + 'E121', 'E123', 'E126', 'E226', 'E24', 'E704', 'E8', + ] + assert options.exclude == [os.path.abspath('tests/*')] From f2ca91372aec1a1ca8f13f00df7bd3013b613830 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 25 Jan 2016 20:40:33 -0600 Subject: [PATCH 107/204] Start structuring documentation --- dev/.keep | 0 docs/source/index.rst | 23 ++++++++++++++++++----- internal/.keep | 0 user/.keep | 0 4 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 dev/.keep create mode 100644 internal/.keep create mode 100644 user/.keep diff --git a/dev/.keep b/dev/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/index.rst b/docs/source/index.rst index 904ca87..3557d34 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,14 +3,28 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to flake8's documentation! -================================== +Flake8: Your Tool For Style Guide Enforcement +============================================= -Contents: +User Guide +---------- .. toctree:: - :maxdepth: 2 + :maxdepth: 2 +Plugin Developer Guide +---------------------- + +.. toctree:: + :maxdepth: 2 + + internal/option_handling.rst + +Developer Guide +--------------- + +.. toctree:: + :maxdepth: 2 Indices and tables @@ -19,4 +33,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/internal/.keep b/internal/.keep new file mode 100644 index 0000000..e69de29 diff --git a/user/.keep b/user/.keep new file mode 100644 index 0000000..e69de29 From 6590fab3f2dc168e8f3e35cec0afa2e8d108359a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 25 Jan 2016 21:15:04 -0600 Subject: [PATCH 108/204] Move docs directories into docs/source --- {dev => docs/source/dev}/.keep | 0 {internal => docs/source/internal}/.keep | 0 {user => docs/source/user}/.keep | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {dev => docs/source/dev}/.keep (100%) rename {internal => docs/source/internal}/.keep (100%) rename {user => docs/source/user}/.keep (100%) diff --git a/dev/.keep b/docs/source/dev/.keep similarity index 100% rename from dev/.keep rename to docs/source/dev/.keep diff --git a/internal/.keep b/docs/source/internal/.keep similarity index 100% rename from internal/.keep rename to docs/source/internal/.keep diff --git a/user/.keep b/docs/source/user/.keep similarity index 100% rename from user/.keep rename to docs/source/user/.keep From 7898b5c12efcd9accfd71da5b1a0cab0df8584eb Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 25 Jan 2016 22:05:46 -0600 Subject: [PATCH 109/204] Add documentation around Option Management --- docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- docs/source/internal/option_handling.rst | 133 +++++++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 docs/source/internal/option_handling.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index 835a673..991579f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -292,4 +292,4 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'python': ('https://docs.python.org/3.4', None)} diff --git a/docs/source/index.rst b/docs/source/index.rst index 3557d34..0263664 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,7 +18,7 @@ Plugin Developer Guide .. toctree:: :maxdepth: 2 - internal/option_handling.rst + internal/option_handling Developer Guide --------------- diff --git a/docs/source/internal/option_handling.rst b/docs/source/internal/option_handling.rst new file mode 100644 index 0000000..f81a4be --- /dev/null +++ b/docs/source/internal/option_handling.rst @@ -0,0 +1,133 @@ +Option and Configuration Handling +================================= + +Option Management +----------------- + +Command-line options are often also set in configuration files for Flake8. +While not all options are meant to be parsed from configuration files, many +default options are also parsed from configuration files as are most plugin +options. + +In Flake8 2, plugins received a :class:`optparse.OptionParser` instance and +called :meth:`optparse.OptionParser.add_option` to register options. If the +plugin author also wanted to have that option parsed from config files they +also had to do something like: + +.. code-block:: python + + parser.config_options.append('my_config_option') + parser.config_options.extend(['config_opt1', 'config_opt2']) + +This was previously undocumented and led to a lot of confusion as to why +registered options were not automatically parsed from configuration files. + +Since Flake8 3 was rewritten from scratch, we decided to take a different +approach to configuration file parsing. Instead of needing to know about an +undocumented attribute that pep8 looks for, Flake8 3 now accepts a parameter +to ``add_option``, specifically ``parse_from_config`` which is a boolean +value. + +Flake8 does this by creating its own abstractions on top of :mod:`optparse`. +The first abstraction is the :class:`flake8.options.manager.Option` class. The +second is the :class:`flake8.options.manager.OptionManager`. In fact, we add +three new parameters: + +- ``parse_from_config`` + +- ``comma_separated_list`` + +- ``normalize_paths`` + +The last two are not specifically for configuration file handling, but they +do improve that dramatically. We found that there were options that when +specified in a configuration file, lended themselves to being split across +multiple lines and those options were almost always comma-separated. For +example, let's consider a user's list of ignored error codes for a project: + +.. code-block:: ini + + [flake8] + ignore = + E111, # Reasoning + E711, # Reasoning + E712, # Reasoning + E121, # Reasoning + E122, # Reasoning + E123, # Reasoning + E131, # Reasoning + E251 # Reasoning + +It makes sense here to allow users to specify the value this way, but, the +standard libary's :class:`configparser.RawConfigParser` class does returns a +string that looks like + +.. code-block:: python + + "\nE111, \nE711, \nE712, \nE121, \nE122, \nE123, \nE131, \nE251 " + +This means that a typical call to :meth:`str.split` with ``','`` will not be +sufficient here. Telling Flake8 that something is a comma-separated list +(e.g., ``comma_separated_list=True``) will handle this for you. Flake8 will +return: + +.. code-block:: python + + ["E111", "E711", "E712", "E121", "E122", "E123", "E131", "E251"] + +Next let's look at how users might like to specify their ``exclude`` list. +Presently OpenStack's Nova project has this line in their `tox.ini`_: + +.. code-block:: ini + + exclude = .venv,.git,.tox,dist,doc,*openstack/common/*,*lib/python*,*egg,build,tools/xenserver*,releasenotes + +I think we can all agree that this would be easier to read like this: + +.. code-block:: ini + + exclude = + .venv, + .git, + .tox, + dist, + doc, + *openstack/common/*, + *lib/python*, + *egg, + build, + tools/xenserver*, + releasenotes + +In this case, since these are actually intended to be paths, we would specify +both ``comma_separated_list=True`` and ``normalize_paths=True`` because we +want the paths to be provided to us with some consistency (either all absolute +paths or not). + +Now let's look at how this would actually be utilized. Most plugin developers +will receive an instance of :class:`~flake8.options.manager.OptionManager` so +to ease the transition we kept the same API as the +:class:`optparse.OptionParser` object. The only difference is that +:meth:`~flake8.options.manager.OptionManager.add_option` accepts the three +extra arguments we highlighted above. + +.. _tox.ini: + https://github.com/openstack/nova/blob/3eb190c4cfc0eefddac6c2cc1b94a699fb1687f8/tox.ini#L155 + +Configuration File Management +----------------------------- + +.. todo:: Add notes about Config File Management + +API Documentation +----------------- + +.. autoclass:: flake8.options.manager.Option + :members: __init__, normalize, to_optparse + +.. autoclass:: flake8.options.manager.OptionManager + :members: + :special-members: + +.. autoclass:: flake8.options.config.MergedConfigParser + :members: From 938d2c450aca196e3b20996f2a17c5913d5a9a89 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 26 Jan 2016 08:40:45 -0600 Subject: [PATCH 110/204] Switch to the RTD theme --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 991579f..0d10af2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -115,7 +115,7 @@ todo_include_todos = True # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From f59152cc01d7da57a279ccdb1e14550b3573f607 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 26 Jan 2016 08:40:56 -0600 Subject: [PATCH 111/204] Start documenting configuration parsing --- docs/source/internal/option_handling.rst | 64 +++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/source/internal/option_handling.rst b/docs/source/internal/option_handling.rst index f81a4be..01b3862 100644 --- a/docs/source/internal/option_handling.rst +++ b/docs/source/internal/option_handling.rst @@ -117,7 +117,64 @@ extra arguments we highlighted above. Configuration File Management ----------------------------- -.. todo:: Add notes about Config File Management +In Flake8 2, configuration file discovery and management was handled by pep8. +In pep8's 1.6 release series, it drastically broke how discovery and merging +worked (as a result of trying to improve it). To avoid a dependency breaking +Flake8 again in the future, we have created our own discovery and management. +As part of managing this ourselves, we decided to change management/discovery +for 3.0.0. We have done the following: + +- User files (files stored in a user's home directory or in the XDG directory + inside their home directory) are the first files read. For example, if the + user has a ``~/.flake8`` file, we will read that first. + +- Project files (files stored in the current directory) are read next and + merged on top of the user file. In other words, configuration in project + files takes precedence over configuration in user files. + +- **New in 3.0.0** The user can specify ``--append-config `` + repeatedly to include extra configuration files that should be read and + take precedence over user and project files. + +- **New in 3.0.0** The user can specify ``--config `` to so this + file is the only configuration file used. This is a change from Flake8 2 + where pep8 would simply merge this configuration file into the configuration + generated by user and project files (where this takes precedence). + +- **New in 3.0.0** The user can specify ``--isolated`` to disable + configuration via discovered configuration files. + +To facilitate the configuration file management, we've taken a different +approach to discovery and management of files than pep8. In pep8 1.5, 1.6, and +1.7 configuration discovery and management was centralized in `66 lines of +very terse python`_ which was confusing and not very explicit. The terseness +of this function (Flake8's authors believe) caused the confusion and problems +with pep8's 1.6 series. As such, Flake8 has separated out discovery, +management, and merging into a module to make reasoning about each of these +pieces easier and more explicit (as well as easier to test). + +Configuration file discovery is managed by the +:class:`~flake8.options.config.ConfigFileFinder` object. This object needs to +know information about the program's name, any extra arguments passed to it, +and any configuration files that should be appended to the list of discovered +files. It provides methods for finding the files and similiar methods for +parsing those fles. For example, it provides +:meth:`~flake8.options.config.ConfigFileFinder.local_config_files` to find +known local config files (and append the extra configuration files) and it +also provides :meth:`~flake8.options.config.ConfigFileFinder.local_configs` +to parse those configuration files. + +.. note:: ``local_config_files`` also filters out non-existent files. + +Configuration file merging and managemnt is controlled by the +:class:`~flake8.options.config.MergedConfigParser`. This requires the instance +of :class:`~flake8.options.manager.OptionManager` that the program is using, +the list of appended config files, and the list of extra arguments. + +.. todo:: Describe how the MergedConfigParser parses and merges config options + +.. _66 lines of very terse python: + https://github.com/PyCQA/pep8/blob/b8088a2b6bc5b76bece174efad877f764529bc74/pep8.py#L1981..L2047 API Documentation ----------------- @@ -129,5 +186,10 @@ API Documentation :members: :special-members: +.. autoclass:: flake8.options.config.ConfigFileFinder + :members: + :special-members: + .. autoclass:: flake8.options.config.MergedConfigParser :members: + :special-members: From 4cf48b9b772c558363340d64bacb35a867c945c6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 27 Jan 2016 21:32:40 -0600 Subject: [PATCH 112/204] Add documentation about option aggregation Add logging around default ignore lists --- docs/source/internal/option_handling.rst | 34 ++++++++++++++++++++++-- flake8/options/aggregator.py | 20 +++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/docs/source/internal/option_handling.rst b/docs/source/internal/option_handling.rst index 01b3862..8ab1911 100644 --- a/docs/source/internal/option_handling.rst +++ b/docs/source/internal/option_handling.rst @@ -169,9 +169,37 @@ to parse those configuration files. Configuration file merging and managemnt is controlled by the :class:`~flake8.options.config.MergedConfigParser`. This requires the instance of :class:`~flake8.options.manager.OptionManager` that the program is using, -the list of appended config files, and the list of extra arguments. +the list of appended config files, and the list of extra arguments. This +object is currently the sole user of the +:class:`~flake8.options.config.ConfigFileFinder` object. It appropriately +initializes the object and uses it in each of -.. todo:: Describe how the MergedConfigParser parses and merges config options +- :meth:`~flake8.options.config.MergedConfigParser.parse_cli_config` +- :meth:`~flake8.options.config.MergedConfigParser.parse_local_config` +- :meth:`~flake8.options.config.MergedConfigParser.parse_user_config` + +Finally, +:meth:`~flake8.options.config.MergedConfigParser.merge_user_and_local_config` +takes the user and local configuration files that are parsed by +:meth:`~flake8.options.config.MergedConfigParser.parse_local_config` and +:meth:`~flake8.options.config.MergedConfigParser.parse_user_config`. The +main usage of the ``MergedConfigParser`` is in +:func:`~flake8.options.aggregator.aggregate_options`. + +Aggregating Configuration File and Command Line Arguments +--------------------------------------------------------- + +:func:`~flake8.options.aggregator.aggregate_options` accepts an instance of +:class:`~flake8.options.maanger.OptionManager` and does the work to parse the +command-line arguments passed by the user necessary for creating an instance +of :class:`~flake8.options.config.MergedConfigParser`. + +After parsing the configuration file, we determine the default ignore list. We +use the defaults from the OptionManager and update those with the parsed +configuration files. Finally we parse the user-provided options one last time +using the option defaults and configuration file values as defaults. The +parser merges on the command-line specified arguments for us so we have our +final, definitive, aggregated options. .. _66 lines of very terse python: https://github.com/PyCQA/pep8/blob/b8088a2b6bc5b76bece174efad877f764529bc74/pep8.py#L1981..L2047 @@ -179,6 +207,8 @@ the list of appended config files, and the list of extra arguments. API Documentation ----------------- +.. autofunction:: flake8.options.aggregator.aggregate_options + .. autoclass:: flake8.options.manager.Option :members: __init__, normalize, to_optparse diff --git a/flake8/options/aggregator.py b/flake8/options/aggregator.py index e12952a..5c02730 100644 --- a/flake8/options/aggregator.py +++ b/flake8/options/aggregator.py @@ -12,7 +12,22 @@ LOG = logging.getLogger(__name__) def aggregate_options(manager, arglist=None, values=None): - """Function that aggregates the CLI and Config options.""" + """Aggregate and merge CLI and config file options. + + :param flake8.option.manager.OptionManager manager: + The instance of the OptionManager that we're presently using. + :param list arglist: + The list of arguments to pass to ``manager.parse_args``. In most cases + this will be None so ``parse_args`` uses ``sys.argv``. This is mostly + available to make testing easier. + :param optparse.Values values: + Previously parsed set of parsed options. + :returns: + Tuple of the parsed options and extra arguments returned by + ``manager.parse_args``. + :rtype: + tuple(optparse.Values, list) + """ # Get defaults from the option parser default_values, _ = manager.parse_args([], values=values) # Get original CLI values so we can find additional config file paths and @@ -34,8 +49,11 @@ def aggregate_options(manager, arglist=None, values=None): # Extend the default ignore value with the extended default ignore list, # registered by plugins. extended_default_ignore = manager.extended_default_ignore.copy() + LOG.debug('Extended default ignore list: %s', + list(extended_default_ignore)) extended_default_ignore.update(default_values.ignore) default_values.ignore = list(extended_default_ignore) + LOG.debug('Merged default ignore list: %s', default_values.ignore) # Merge values parsed from config onto the default values returned for config_name, value in parsed_config.items(): From f4f68b6442e26a29a86cb7bcd3a3b2cfe01438d6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 31 Jan 2016 23:59:53 -0600 Subject: [PATCH 113/204] Document our plugin handling --- docs/source/index.rst | 1 + docs/source/internal/plugin_handling.rst | 73 ++++++++++++++++++++++++ flake8/plugins/manager.py | 2 +- 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/source/internal/plugin_handling.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 0263664..4a50a5d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Plugin Developer Guide :maxdepth: 2 internal/option_handling + internal/plugin_handling Developer Guide --------------- diff --git a/docs/source/internal/plugin_handling.rst b/docs/source/internal/plugin_handling.rst new file mode 100644 index 0000000..547b5ca --- /dev/null +++ b/docs/source/internal/plugin_handling.rst @@ -0,0 +1,73 @@ +Plugin Handling +=============== + +Plugin Management +----------------- + +Flake8 3.0 added support for two other plugins besides those which define new +checks. It now supports: + +- extra checks + +- alternative report formatters + +- listeners to auto-correct violations of checks + +To facilitate this, Flake8 needed a more mature way of managing plugins. As +such, we developed the |PluginManager| which accepts a namespace and will load +the plugins for that namespace. A |PluginManager| creates and manages many +|Plugin| instances. + +A |Plugin| lazily loads the underlying entry-point provided by setuptools. +The entry-point will be loaded either by calling +:meth:`~flake8.plugins.manager.Plugin.load_plugin` or accessing the ``plugin`` +attribute. We also use this abstraction to retrieve options that the plugin +wishes to register and parse. + +The only public method that the |PluginManager| provides is +:meth:`~flake8.plugins.manager.PluginManager.map`. This will accept a function +(or other callable) and call it with each plugin as the first parameter. + +We build atop the |PluginManager| with the |PTM|. It is expected that users of +the |PTM| will subclass it and specify the ``namespace``, e.g., + +.. code-block:: python + + class ExamplePluginType(flake8.plugin.manager.PluginTypeManager): + namespace = 'example-plugins' + +This provides a few extra methods via the |PluginManager|'s ``map`` method. + +Finally, we create three classes of plugins: + +- :class:`~flake8.plugins.manager.Checkers` + +- :class:`~flake8.plugins.manager.Listeners` + +- :class:`~flake8.plugins.manager.ReportFormatters` + +These are used to interact with each of the types of plugins individually. + +API Documentation +----------------- + +.. autoclass:: flake8.plugins.manager.PluginManager + :members: + :special-members: __init__, __contains__, __getitem__ + +.. autoclass:: flake8.plugins.manager.Plugin + :members: + :special-members: __init__ + +.. autoclass:: flake8.plugins.manager.PluginTypeManager + :members: + +.. autoclass:: flake8.plugins.manager.Checkers + +.. autoclass:: flake8.plugins.manager.Listeners + +.. autoclass:: flake8.plugins.manager.ReportFormatters + +.. |PluginManager| replace:: :class:`~flake8.plugins.manager.PluginManager` +.. |Plugin| replace:: :class:`~flake8.plugins.manager.Plugin` +.. |PTM| replace:: :class:`~flake8.plugins.manager.PluginTypeManager` diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index f73d006..570c1d0 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -171,7 +171,7 @@ class PluginManager(object): LOG.info('Loaded %r for plugin "%s".', self.plugins[name], name) def map(self, func, *args, **kwargs): - """Call ``func`` with the plugin and *args and **kwargs after. + r"""Call ``func`` with the plugin and \*args and \**kwargs after. This yields the return value from ``func`` for each plugin. From 51d11d1a308e72f5e460d77bf37f98c818dd650f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 1 Feb 2016 00:00:29 -0600 Subject: [PATCH 114/204] Note inspiration --- docs/source/internal/plugin_handling.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/internal/plugin_handling.rst b/docs/source/internal/plugin_handling.rst index 547b5ca..d69957d 100644 --- a/docs/source/internal/plugin_handling.rst +++ b/docs/source/internal/plugin_handling.rst @@ -48,6 +48,11 @@ Finally, we create three classes of plugins: These are used to interact with each of the types of plugins individually. +.. note:: + + Our inspiration for our plugin handling comes from the author's extensive + experience with ``stevedore``. + API Documentation ----------------- From 5dc7440a2ba228bc6d7b693ff4737d3a2453e965 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 1 Feb 2016 20:09:15 -0600 Subject: [PATCH 115/204] Start documenting our Trie implementation --- docs/source/internal/plugin_handling.rst | 3 +++ flake8/{ => plugins}/_trie.py | 0 flake8/plugins/manager.py | 2 +- flake8/{ => plugins}/notifier.py | 2 +- tests/unit/test_notifier.py | 2 +- tests/unit/test_trie.py | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) rename flake8/{ => plugins}/_trie.py (100%) rename flake8/{ => plugins}/notifier.py (97%) diff --git a/docs/source/internal/plugin_handling.rst b/docs/source/internal/plugin_handling.rst index d69957d..e217497 100644 --- a/docs/source/internal/plugin_handling.rst +++ b/docs/source/internal/plugin_handling.rst @@ -53,6 +53,9 @@ These are used to interact with each of the types of plugins individually. Our inspiration for our plugin handling comes from the author's extensive experience with ``stevedore``. +Notifying Listener Plugins +-------------------------- + API Documentation ----------------- diff --git a/flake8/_trie.py b/flake8/plugins/_trie.py similarity index 100% rename from flake8/_trie.py rename to flake8/plugins/_trie.py diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 570c1d0..4a06c75 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -4,8 +4,8 @@ import logging import pkg_resources -from flake8 import notifier from flake8 import exceptions +from flake8.plugins import notifier LOG = logging.getLogger(__name__) diff --git a/flake8/notifier.py b/flake8/plugins/notifier.py similarity index 97% rename from flake8/notifier.py rename to flake8/plugins/notifier.py index e7bf3ba..323ea1d 100644 --- a/flake8/notifier.py +++ b/flake8/plugins/notifier.py @@ -1,5 +1,5 @@ """Implementation of the class that registers and notifies listeners.""" -from flake8 import _trie +from flake8.plugins import _trie class Notifier(object): diff --git a/tests/unit/test_notifier.py b/tests/unit/test_notifier.py index 84b8a45..effcc88 100644 --- a/tests/unit/test_notifier.py +++ b/tests/unit/test_notifier.py @@ -1,7 +1,7 @@ """Unit tests for the Notifier object.""" import pytest -from flake8 import notifier +from flake8.plugins import notifier class _Listener(object): diff --git a/tests/unit/test_trie.py b/tests/unit/test_trie.py index 4cbd8c5..152b5b6 100644 --- a/tests/unit/test_trie.py +++ b/tests/unit/test_trie.py @@ -1,5 +1,5 @@ """Unit test for the _trie module.""" -from flake8 import _trie as trie +from flake8.plugins import _trie as trie class TestTrie(object): From 4dc1cc386d91caa381ef9ca98eba309d23e070f9 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 1 Feb 2016 20:33:18 -0600 Subject: [PATCH 116/204] Add documentation around plugin notifications and pyflakes --- docs/source/internal/plugin_handling.rst | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/source/internal/plugin_handling.rst b/docs/source/internal/plugin_handling.rst index e217497..e430bfb 100644 --- a/docs/source/internal/plugin_handling.rst +++ b/docs/source/internal/plugin_handling.rst @@ -56,6 +56,33 @@ These are used to interact with each of the types of plugins individually. Notifying Listener Plugins -------------------------- +One of the interesting challenges with allowing plugins to be notified each +time an error or warning is emitted by a checker is finding listeners quickly +and efficiently. It makes sense to allow a listener to listen for a certain +class of warnings or just a specific warning. As such, we need to allow all +plugins that listen to a specific warning or class to be notified. For +example, someone might register a listener for ``E1`` and another for ``E111`` +if ``E111`` is triggered by the code, both listeners should be notified. +If ``E112`` is returned, then only ``E1`` (and any other listeners) would be +notified. + +To implement this goal, we needed an object to store listeners in that would +allow for efficient look up - a Trie (or Prefix Tree). Given that none of the +existing packages on PyPI allowed for storing data on each node of the trie, +it was left up to write our own as :class:`~flake8.plugins._trie.Trie`. On +top of that we layer our :class:`~flake8.plugins.notifier.Notifier` class. + +Now when Flake8 receives an error or warning, we can easily call the +:meth:`~flake8.plugins.notifier.Notifier.notify` method and let plugins act on +that knowledge. + +Default Plugins +--------------- + +Finally, Flake8 has always provided its own plugin shim for Pyflakes. As part +of that we carry our own shim in-tree and now store that in +:mod:`flake8.plugins.pyflakes`. + API Documentation ----------------- @@ -76,6 +103,10 @@ API Documentation .. autoclass:: flake8.plugins.manager.ReportFormatters +.. autoclass:: flake8.plugins.notifier.Notifier + +.. autoclass:: flake8.plugins._trie.Trie + .. |PluginManager| replace:: :class:`~flake8.plugins.manager.PluginManager` .. |Plugin| replace:: :class:`~flake8.plugins.manager.Plugin` .. |PTM| replace:: :class:`~flake8.plugins.manager.PluginTypeManager` From 91327c75e0da54fb7f4479499009fb6946f59820 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 1 Feb 2016 20:55:57 -0600 Subject: [PATCH 117/204] Start work on our StyleGuide implementation --- flake8/style_guide.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index a3f63eb..ec000b7 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -1 +1,25 @@ """Implementation of the StyleGuide used by Flake8.""" + + +class StyleGuide(object): + """Manage a Flake8 user's style guide.""" + + def __init__(self, options, arguments, checker_plugins, listening_plugins, + formatting_plugins): + """Initialize our StyleGuide. + + .. todo:: Add parameter documentation. + """ + pass + + +# Should separate style guide logic from code that runs checks +# StyleGuide should manage select/ignore logic as well as include/exclude +# logic. See also https://github.com/PyCQA/pep8/pull/433 + +# StyleGuide shoud dispatch check execution in a way that can use +# multiprocessing but also retry in serial. See also: +# https://gitlab.com/pycqa/flake8/issues/74 + +# StyleGuide should interface with Reporter and aggregate errors/notify +# listeners From 0645ec3ef771c8e2e6ef39bdd5eb360067d147b9 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 2 Feb 2016 16:53:24 -0600 Subject: [PATCH 118/204] Move __contains__ and __getitem__ to proper class --- flake8/plugins/manager.py | 20 ++++++++-------- tests/unit/test_plugin_manager.py | 28 +--------------------- tests/unit/test_plugin_type_manager.py | 33 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 4a06c75..02a136c 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -152,16 +152,6 @@ class PluginManager(object): self.names = [] self._load_all_plugins() - def __contains__(self, name): - """Check if the entry-point name is in this plugin manager.""" - LOG.debug('Checking for "%s" in plugin manager.', name) - return name in self.plugins - - def __getitem__(self, name): - """Retrieve a plugin by its entry-point name.""" - LOG.debug('Retrieving plugin for "%s".', name) - return self.plugins[name] - def _load_all_plugins(self): LOG.debug('Loading entry-points for "%s".', self.namespace) for entry_point in pkg_resources.iter_entry_points(self.namespace): @@ -203,6 +193,16 @@ class PluginTypeManager(object): self.manager = PluginManager(self.namespace) self.plugins_loaded = False + def __contains__(self, name): + """Check if the entry-point name is in this plugin type manager.""" + LOG.debug('Checking for "%s" in plugin type manager.', name) + return name in self.plugins + + def __getitem__(self, name): + """Retrieve a plugin by its name.""" + LOG.debug('Retrieving plugin for "%s".', name) + return self.plugins[name] + @property def names(self): """Proxy attribute to underlying manager.""" diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py index 05073cf..8991b96 100644 --- a/tests/unit/test_plugin_manager.py +++ b/tests/unit/test_plugin_manager.py @@ -36,32 +36,6 @@ def test_calls_pkg_resources_creates_plugins_automaticaly(iter_entry_points): assert isinstance(plugin_mgr.plugins['T200'], manager.Plugin) -@mock.patch('pkg_resources.iter_entry_points') -def test_proxies_contains_to_plugins_dictionary(iter_entry_points): - """Verify that we can use the PluginManager like a dictionary.""" - iter_entry_points.return_value = [ - create_entry_point_mock('T100'), - create_entry_point_mock('T200'), - ] - plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') - - assert 'T100' in plugin_mgr - assert 'T200' in plugin_mgr - - -@mock.patch('pkg_resources.iter_entry_points') -def test_proxies_getitem_to_plugins_dictionary(iter_entry_points): - """Verify that we can use the PluginManager like a dictionary.""" - iter_entry_points.return_value = [ - create_entry_point_mock('T100'), - create_entry_point_mock('T200'), - ] - plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') - - assert isinstance(plugin_mgr['T100'], manager.Plugin) - assert isinstance(plugin_mgr['T200'], manager.Plugin) - - @mock.patch('pkg_resources.iter_entry_points') def test_handles_mapping_functions_across_plugins(iter_entry_points): """Verify we can use the PluginManager call functions on all plugins.""" @@ -71,6 +45,6 @@ def test_handles_mapping_functions_across_plugins(iter_entry_points): ] iter_entry_points.return_value = entry_point_mocks plugin_mgr = manager.PluginManager(namespace='testing.pkg_resources') - plugins = [plugin_mgr[name] for name in plugin_mgr.names] + plugins = [plugin_mgr.plugins[name] for name in plugin_mgr.names] assert list(plugin_mgr.map(lambda x: x)) == plugins diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py index 2735dec..36fb92a 100644 --- a/tests/unit/test_plugin_type_manager.py +++ b/tests/unit/test_plugin_type_manager.py @@ -35,6 +35,13 @@ def create_mapping_manager_mock(plugins): return manager_mock +def create_manager_with_plugins(plugins): + """Create a fake PluginManager with a plugins dictionary.""" + manager_mock = mock.create_autospec(manager.PluginManager) + manager_mock.plugins = plugins + return manager_mock + + class TestType(manager.PluginTypeManager): """Fake PluginTypeManager.""" @@ -167,6 +174,32 @@ def test_provide_options(PluginManager): extra_args) +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxy_contains_to_managers_plugins_dict(PluginManager): + """Verify that we proxy __contains__ to the manager's dictionary.""" + plugins = {'T10%i' % i: create_plugin_mock() for i in range(8)} + # Return our PluginManager mock + PluginManager.return_value = create_manager_with_plugins(plugins) + + type_mgr = TestType() + for i in range(8): + key = 'T10%i' % i + assert key in type_mgr + + +@mock.patch('flake8.plugins.manager.PluginManager') +def test_proxies_getitem_to_managers_plugins_dictionary(PluginManager): + """Verify that we can use the PluginTypeManager like a dictionary.""" + plugins = {'T10%i' % i: create_plugin_mock() for i in range(8)} + # Return our PluginManager mock + PluginManager.return_value = create_manager_with_plugins(plugins) + + type_mgr = TestType() + for i in range(8): + key = 'T10%i' % i + assert type_mgr[key] is plugins[key] + + class FakePluginTypeManager(manager.NotifierBuilder): """Provide an easy way to test the NotifierBuilder.""" From eafc91ae6ad5c05b3e036733d73a0edac9070192 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 2 Feb 2016 20:48:26 -0600 Subject: [PATCH 119/204] Add handling and decision making for select and ignore --- flake8/style_guide.py | 106 ++++++++++++++++++++++++++- setup.cfg | 4 ++ setup.py | 17 +++-- tests/unit/test_style_guide.py | 128 +++++++++++++++++++++++++++++++++ 4 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_style_guide.py diff --git a/flake8/style_guide.py b/flake8/style_guide.py index ec000b7..ef1cbae 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -1,4 +1,30 @@ """Implementation of the StyleGuide used by Flake8.""" +import enum + +__all__ = ( + 'StyleGuide', +) + + +class Selected(enum.Enum): + """Enum representing an explicitly or implicitly selected code.""" + + Explicitly = 'explicitly selected' + Implicitly = 'implicitly selected' + + +class Ignored(enum.Enum): + """Enum representing an explicitly or implicitly ignored code.""" + + Explicitly = 'explicitly ignored' + Implicitly = 'implicitly ignored' + + +class Decision(enum.Enum): + """Enum representing whether a code should be ignored or selected.""" + + Ignored = 'ignored error' + Selected = 'selected error' class StyleGuide(object): @@ -10,7 +36,85 @@ class StyleGuide(object): .. todo:: Add parameter documentation. """ - pass + self.options = options + self.arguments = arguments + self.checkers = checker_plugins + self.listeners = listening_plugins + self.formatters = formatting_plugins + self._selected = tuple(options.select) + self._ignored = tuple(options.ignore) + self._decision_cache = {} + + def is_user_selected(self, code): + """Determine if the code has been selected by the user. + + :param str code: + The code for the check that has been run. + :returns: + Selected.Implicitly if the selected list is empty, + Selected.Explicitly if the selected list is not empty and a match + was found, + Ignored.Implicitly if the selected list is not empty but no match + was found. + """ + if not self._selected: + return Selected.Implicitly + + if code.startswith(self._selected): + return Selected.Explicitly + + return Ignored.Implicitly + + def is_user_ignored(self, code): + """Determine if the code has been ignored by the user. + + :param str code: + The code for the check that has been run. + :returns: + Selected.Implicitly if the ignored list is empty, + Ignored.Explicitly if the ignored list is not empty and a match was + found, + Selected.Implicitly if the ignored list is not empty but no match + was found. + """ + if self._ignored and code.startswith(self._ignored): + return Ignored.Explicitly + + return Selected.Implicitly + + def _decision_for(self, code): + startswith = code.startswith + selected = sorted([s for s in self._selected if startswith(s)])[0] + ignored = sorted([i for i in self._ignored if startswith(i)])[0] + + if selected.startswith(ignored): + return Decision.Selected + return Decision.Ignored + + def should_report_error(self, code): + """Determine if the error code should be reported or ignored. + + :param str code: + The code for the check that has been run. + """ + decision = self._decision_cache.get(code) + if decision is None: + selected = self.is_user_selected(code) + ignored = self.is_user_ignored(code) + + if ((selected is Selected.Explicitly or + selected is Selected.Implicitly) and + ignored is Selected.Implicitly): + decision = Decision.Selected + elif (selected is Selected.Explicitly and + ignored is Ignored.Explicitly): + decision = self._decision_for(code) + elif (selected is Ignored.Implicitly or + ignored is Ignored.Explicitly): + decision = Decision.Ignored + + self._decision_cache[code] = decision + return decision # Should separate style guide logic from code that runs checks diff --git a/setup.cfg b/setup.cfg index f05f933..e96761b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,7 @@ test=pytest [bdist_wheel] universal=1 + +[metadata] +requires-dist = + enum34; python_version<"3.4" diff --git a/setup.py b/setup.py index 93e4fdb..0b90cf7 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from __future__ import with_statement import setuptools +import sys import flake8 @@ -25,6 +26,16 @@ if mock is None: tests_require += ['mock'] +requires = [ + "pyflakes >= 0.8.1, < 1.1", + "pep8 >= 1.5.7, != 1.6.0, != 1.6.1, != 1.6.2", + # "mccabe >= 0.2.1, < 0.4", +] + +if sys.version_info < (3, 4): + requires.append("enum34") + + def get_long_description(): """Generate a long description from the README and CHANGES files.""" descr = [] @@ -51,11 +62,7 @@ setuptools.setup( "flake8.options", "flake8.plugins", ], - install_requires=[ - "pyflakes >= 0.8.1, < 1.1", - "pep8 >= 1.5.7, != 1.6.0, != 1.6.1, != 1.6.2", - # "mccabe >= 0.2.1, < 0.4", - ], + install_requires=requires, entry_points={ 'distutils.commands': ['flake8 = flake8.main:Flake8Command'], 'console_scripts': ['flake8 = flake8.main.cli:main'], diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py new file mode 100644 index 0000000..51e9f61 --- /dev/null +++ b/tests/unit/test_style_guide.py @@ -0,0 +1,128 @@ +"""Tests for the flake8.style_guide.StyleGuide class.""" +import optparse + +from flake8 import style_guide + +import pytest + + +def create_options(**kwargs): + """Create and return an instance of optparse.Values.""" + kwargs.setdefault('select', []) + kwargs.setdefault('ignore', []) + return optparse.Values(kwargs) + + +@pytest.mark.parametrize('ignore_list,error_code', [ + (['E111', 'E121'], 'E111'), + (['E111', 'E121'], 'E121'), + (['E11', 'E12'], 'E121'), + (['E2', 'E12'], 'E121'), + (['E2', 'E12'], 'E211'), +]) +def test_is_user_ignored_ignores_errors(ignore_list, error_code): + """Verify we detect users explicitly ignoring an error.""" + guide = style_guide.StyleGuide(create_options(ignore=ignore_list), + arguments=[], + checker_plugins=None, + listening_plugins=None, + formatting_plugins=None) + + assert guide.is_user_ignored(error_code) is style_guide.Ignored.Explicitly + + +@pytest.mark.parametrize('ignore_list,error_code', [ + (['E111', 'E121'], 'E112'), + (['E111', 'E121'], 'E122'), + (['E11', 'E12'], 'W121'), + (['E2', 'E12'], 'E112'), + (['E2', 'E12'], 'E111'), +]) +def test_is_user_ignored_implicitly_selects_errors(ignore_list, error_code): + """Verify we detect users does not explicitly ignore an error.""" + guide = style_guide.StyleGuide(create_options(ignore=ignore_list), + arguments=[], + checker_plugins=None, + listening_plugins=None, + formatting_plugins=None) + + assert guide.is_user_ignored(error_code) is style_guide.Selected.Implicitly + + +@pytest.mark.parametrize('select_list,error_code', [ + (['E111', 'E121'], 'E111'), + (['E111', 'E121'], 'E121'), + (['E11', 'E12'], 'E121'), + (['E2', 'E12'], 'E121'), + (['E2', 'E12'], 'E211'), +]) +def test_is_user_selected_selects_errors(select_list, error_code): + """Verify we detect users explicitly selecting an error.""" + guide = style_guide.StyleGuide(create_options(select=select_list), + arguments=[], + checker_plugins=None, + listening_plugins=None, + formatting_plugins=None) + + assert (guide.is_user_selected(error_code) is + style_guide.Selected.Explicitly) + + +def test_is_user_selected_implicitly_selects_errors(): + """Verify we detect users implicitly selecting an error.""" + select_list = [] + error_code = 'E121' + guide = style_guide.StyleGuide(create_options(select=select_list), + arguments=[], + checker_plugins=None, + listening_plugins=None, + formatting_plugins=None) + + assert (guide.is_user_selected(error_code) is + style_guide.Selected.Implicitly) + + +@pytest.mark.parametrize('select_list,error_code', [ + (['E111', 'E121'], 'E112'), + (['E111', 'E121'], 'E122'), + (['E11', 'E12'], 'E132'), + (['E2', 'E12'], 'E321'), + (['E2', 'E12'], 'E410'), +]) +def test_is_user_selected_excludes_errors(select_list, error_code): + """Verify we detect users implicitly excludes an error.""" + guide = style_guide.StyleGuide(create_options(select=select_list), + arguments=[], + checker_plugins=None, + listening_plugins=None, + formatting_plugins=None) + + assert guide.is_user_selected(error_code) is style_guide.Ignored.Implicitly + + +@pytest.mark.parametrize('select_list,ignore_list,error_code,expected', [ + (['E111', 'E121'], [], 'E111', style_guide.Decision.Selected), + (['E111', 'E121'], [], 'E112', style_guide.Decision.Ignored), + (['E111', 'E121'], [], 'E121', style_guide.Decision.Selected), + (['E111', 'E121'], [], 'E122', style_guide.Decision.Ignored), + (['E11', 'E12'], [], 'E132', style_guide.Decision.Ignored), + (['E2', 'E12'], [], 'E321', style_guide.Decision.Ignored), + (['E2', 'E12'], [], 'E410', style_guide.Decision.Ignored), + (['E11', 'E121'], ['E1'], 'E112', style_guide.Decision.Selected), + (['E111', 'E121'], ['E2'], 'E122', style_guide.Decision.Ignored), + (['E11', 'E12'], ['E13'], 'E132', style_guide.Decision.Ignored), + (['E1', 'E3'], ['E32'], 'E321', style_guide.Decision.Ignored), + ([], ['E2', 'E12'], 'E410', style_guide.Decision.Selected), + (['E4'], ['E2', 'E12', 'E41'], 'E410', style_guide.Decision.Ignored), + (['E41'], ['E2', 'E12', 'E4'], 'E410', style_guide.Decision.Selected), +]) +def test_should_report_error(select_list, ignore_list, error_code, expected): + """Verify we decide when to report an error.""" + guide = style_guide.StyleGuide(create_options(select=select_list, + ignore=ignore_list), + arguments=[], + checker_plugins=None, + listening_plugins=None, + formatting_plugins=None) + + assert guide.should_report_error(error_code) is expected From 691bdc97eab5676f80d1874657e844c2931c7a7e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 3 Feb 2016 21:11:27 -0600 Subject: [PATCH 120/204] Add option to disable # noqa Sometimes it is nice to be able to ignore what someone has used #noqa on and see what the results would be. This can also be specified in a config file to prevent people from using # noqa in a code base. --- flake8/main/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 7f58fb2..51071b5 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -86,6 +86,13 @@ def register_default_options(option_manager): ' For example, ``--select=E4,E51,W234``. (Default: %default)', ) + add_option( + '--disable-noqa', default=False, parse_from_config=True, + action='store_true', + help='Disable the effect of "# noqa". This will report errors on ' + 'lines with "# noqa" at the end.' + ) + # TODO(sigmavirus24): Decide what to do about --show-pep8 add_option( From 1543ff8bab8b3832d59b4d54360bcf39fd18ce46 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 3 Feb 2016 22:04:08 -0600 Subject: [PATCH 121/204] Add some logging to the style guide --- flake8/style_guide.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index ef1cbae..2177b78 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -1,11 +1,16 @@ """Implementation of the StyleGuide used by Flake8.""" +import logging + import enum __all__ = ( 'StyleGuide', ) +LOG = logging.getLogger(__name__) + +# TODO(sigmavirus24): Determine if we need to use enum/enum34 class Selected(enum.Enum): """Enum representing an explicitly or implicitly selected code.""" @@ -99,8 +104,11 @@ class StyleGuide(object): """ decision = self._decision_cache.get(code) if decision is None: + LOG.debug('Deciding if "%s" should be reported', code) selected = self.is_user_selected(code) ignored = self.is_user_ignored(code) + LOG.debug('The user configured "%s" to be "%s", "%s"', + code, selected, ignored) if ((selected is Selected.Explicitly or selected is Selected.Implicitly) and @@ -114,6 +122,7 @@ class StyleGuide(object): decision = Decision.Ignored self._decision_cache[code] = decision + LOG.debug('"%s" will be "%s"', code, decision) return decision From 253211f5ad09dbaef681f018860fe778026150b5 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 3 Feb 2016 22:30:03 -0600 Subject: [PATCH 122/204] Add caching stdin_get_value function --- flake8/utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/flake8/utils.py b/flake8/utils.py index 0dc16e2..dd83748 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -1,5 +1,7 @@ """Utility methods for flake8.""" +import io import os +import sys def parse_comma_separated_list(value): @@ -43,3 +45,16 @@ def normalize_path(path, parent=os.curdir): if '/' in path: path = os.path.abspath(os.path.join(parent, path)) return path.rstrip('/') + + +def stdin_get_value(): + """Get and cache it so plugins can use it.""" + cached_value = getattr(stdin_get_value, 'cached_stdin', None) + if cached_value is None: + stdin_value = sys.stdin.read() + if sys.version_info < (3, 0): + cached_value = io.BytesIO(stdin_value) + else: + cached_value = io.StringIO(stdin_value) + stdin_get_value.cached_stdin = cached_value + return cached_value.getvalue() From a4e051614f851cf4e642bf00991d5952af104723 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 3 Feb 2016 23:11:45 -0600 Subject: [PATCH 123/204] Document flake8.utils --- docs/source/internal/utils.rst | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/source/internal/utils.rst diff --git a/docs/source/internal/utils.rst b/docs/source/internal/utils.rst new file mode 100644 index 0000000..69dee45 --- /dev/null +++ b/docs/source/internal/utils.rst @@ -0,0 +1,47 @@ +=================== + Utility Functions +=================== + +Flake8 has a few utility functions that it uses and provides to plugins. + +.. autofunction:: flake8.utils.parse_comma_separated_list + +:func:`~flake8.utils.parse_comma_separated_list` takes either a string like + +.. code-block:: python + + "E121,W123,F904" + "E121,\nW123,\nF804" + "E121,\n\tW123,\n\tF804" + +Or it will take a list of strings (potentially with whitespace) such as + +.. code-block:: python + + [" E121\n", "\t\nW123 ", "\n\tF904\n "] + +And converts it to a list that looks as follows + +.. code-block:: python + + ["E121", "W123", "F904"] + +This function helps normalize any kind of comma-separated input you or Flake8 +might receive. This is most helpful when taking advantage of Flake8's +additional parameters to :class:`~flake8.options.manager.Option`. + +.. autofunction:: flake8.utils.normalize_path + +This utility takes a string that represents a path and returns the absolute +path if the string has a ``/`` in it. It also removes trailing ``/``\ s. + +.. autofunction:: flake8.utils.normalize_paths + +This function utilizes :func:`~flake8.utils.parse_comma_separated_list` and +:func:`~flake8.utils.normalize_path` to normalize it's input to a list of +strings that should be paths. + +.. autofunction:: flake8.utils.stdin_get_value + +This function retrieves and caches the value provided on ``sys.stdin``. This +allows plugins to use this to retrieve ``stdin`` if necessary. From 276f823de66164240cf25c237762d8af6d6c947a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 09:32:38 -0600 Subject: [PATCH 124/204] Add type annotations to flake8.utils for fun --- flake8/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake8/utils.py b/flake8/utils.py index dd83748..98e5cea 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -5,6 +5,7 @@ import sys def parse_comma_separated_list(value): + # type: (Union[Sequence[str], str]) -> List[str] """Parse a comma-separated list. :param value: @@ -24,6 +25,7 @@ def parse_comma_separated_list(value): def normalize_paths(paths, parent=os.curdir): + # type: (Union[Sequence[str], str], str) -> List[str] """Parse a comma-separated list of paths. :returns: @@ -35,6 +37,7 @@ def normalize_paths(paths, parent=os.curdir): def normalize_path(path, parent=os.curdir): + # type: (str, str) -> str """Normalize a single-path. :returns: @@ -48,6 +51,7 @@ def normalize_path(path, parent=os.curdir): def stdin_get_value(): + # type: () -> str """Get and cache it so plugins can use it.""" cached_value = getattr(stdin_get_value, 'cached_stdin', None) if cached_value is None: From 5870d136ed6f0891c78993b726c9d12e83bcd737 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 13:51:35 -0600 Subject: [PATCH 125/204] Represent an error with a namedtuple --- flake8/style_guide.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 2177b78..e8451f1 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -1,4 +1,5 @@ """Implementation of the StyleGuide used by Flake8.""" +import collections import logging import enum @@ -32,6 +33,13 @@ class Decision(enum.Enum): Selected = 'selected error' +Error = collections.namedtuple('Error', ['code', + 'filename', + 'line_number', + 'column_number', + 'text']) + + class StyleGuide(object): """Manage a Flake8 user's style guide.""" From 35bb32d94e56e13d4c98c0874b2a6d45b5bc2c91 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 13:52:32 -0600 Subject: [PATCH 126/204] Start working on default formatters for Flake8 --- flake8/formatting/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 flake8/formatting/__init__.py diff --git a/flake8/formatting/__init__.py b/flake8/formatting/__init__.py new file mode 100644 index 0000000..bf44801 --- /dev/null +++ b/flake8/formatting/__init__.py @@ -0,0 +1 @@ +"""Submodule containing the default formatters for Flake8.""" From 63b50bc1f4dbfb848d24cafd1c4a9e5e379508dc Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 14:36:26 -0600 Subject: [PATCH 127/204] Add more functionality to the BaseFormatter Ensure setuptools knows about flake8.formatting --- flake8/formatting/base.py | 105 ++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 106 insertions(+) create mode 100644 flake8/formatting/base.py diff --git a/flake8/formatting/base.py b/flake8/formatting/base.py new file mode 100644 index 0000000..85a931c --- /dev/null +++ b/flake8/formatting/base.py @@ -0,0 +1,105 @@ +"""The base class and interface for all formatting plugins.""" +from __future__ import print_function + + +class BaseFormatter(object): + """Class defining the formatter interface. + + .. attribute:: options + + The options parsed from both configuration files and the command-line. + + .. attribute:: filename + + If specified by the user, the path to store the results of the run. + + .. attribute:: output_fd + + Initialized when the :meth:`start` is called. This will be a file + object opened for writing. + + .. attribute:: newline + + The string to add to the end of a line. This is only used when the + output filename has been specified. + """ + + def __init__(self, options): + """Initialize with the options parsed from config and cli. + + This also calls a hook, :meth:`after_init`, so subclasses do not need + to call super to call this method. + + :param optparse.Values options: + User specified configuration parsed from both configuration files + and the command-line interface. + """ + self.options = options + self.filename = options.output_file + self.output_fd = None + self.newline = '\n' + self.after_init() + + def after_init(self): + """Initialize the formatter further.""" + pass + + def start(self): + """Prepare the formatter to receive input. + + This defaults to initializing :attr:`output_fd` if :attr:`filename` + """ + if self.filename: + self.output_fd = open(self.filename, 'w') + + def handle(self, error): + """Handle an error reported by Flake8. + + This defaults to calling :meth:`format` and then :meth:`write`. To + extend how errors are handled, override this method. + + :param error: + This will be an instance of :class:`~flake8.style_guide.Error`. + :type error: + flake8.style_guide.Error + """ + line = self.format(error) + self.write(line) + + def format(self, error): + """Format an error reported by Flake8. + + This method **must** be implemented by subclasses. + + :param error: + This will be an instance of :class:`~flake8.style_guide.Error`. + :type error: + flake8.style_guide.Error + :returns: + The formatted error string. + :rtype: + str + """ + raise NotImplementedError('Subclass of BaseFormatter did not implement' + ' format.') + + def write(self, line): + """Write the line either to the output file or stdout. + + This handles deciding whether to write to a file or print to standard + out for subclasses. Override this if you want behaviour that differs + from the default. + + :param str line: + The formatted string to print or write. + """ + if self.output_fd is not None: + self.output_fd.write(line + self.newline) + else: + print(self.output_fd) + + def stop(self): + """Clean up after reporting is finished.""" + if self.output_fd is not None: + self.output_fd.close() + self.output_fd = None diff --git a/setup.py b/setup.py index 0b90cf7..00b5a68 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ setuptools.setup( url="https://gitlab.com/pycqa/flake8", packages=[ "flake8", + "flake8.formatting", "flake8.main", "flake8.options", "flake8.plugins", From fe1b628c0f702c2af0303b667aa0504b9b3348bb Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 14:36:44 -0600 Subject: [PATCH 128/204] Add documentation around plugins and formatters --- docs/source/dev/formatters.rst | 16 ++++ docs/source/dev/registering_plugins.rst | 113 ++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 docs/source/dev/formatters.rst create mode 100644 docs/source/dev/registering_plugins.rst diff --git a/docs/source/dev/formatters.rst b/docs/source/dev/formatters.rst new file mode 100644 index 0000000..b96c40b --- /dev/null +++ b/docs/source/dev/formatters.rst @@ -0,0 +1,16 @@ +=========================================== + Developing a Formatting Plugin for Flake8 +=========================================== + +Flake8 added the ability to develop custom formatting plugins in version +3.0.0. Let's write a plugin together: + +.. code-block:: python + + from flake8.formatting import base + + + class Example(base.BaseFormatter): + """Flake8's example formatter.""" + + pass diff --git a/docs/source/dev/registering_plugins.rst b/docs/source/dev/registering_plugins.rst new file mode 100644 index 0000000..1444af0 --- /dev/null +++ b/docs/source/dev/registering_plugins.rst @@ -0,0 +1,113 @@ +================================== + Registering a Plugin with Flake8 +================================== + +To register any kind of plugin with Flake8, you need a few things: + +#. You need a way to install the plugin (whether it is packaged on its own or + as part of something else). In this section, we will use a ``setup.py`` + written for an example plugin. + +#. A name for your plugin that will (ideally) be unique. + +#. A somewhat recent version of setuptools (newer than 0.7.0 but preferably as + recent as you can attain). + +Flake8 presently relies on a functionality provided by setuptools called +`Entry Points`_. These allow any package to register a plugin with Flake8 via +that package's ``setup.py`` file. + +Let's presume that we already have our plugin written and it's in a module +called ``flake8_example``. We might have a ``setup.py`` that looks something +like: + +.. code-block:: python + + from __future__ import with_statement + import setuptools + + requires = [ + "flake8 > 3.0.0", + ] + + flake8_entry_point = # ... + + setuptools.setup( + name="flake8_example", + license="MIT", + version="0.1.0", + description="our extension to flake8", + author="Me", + author_email="example@example.com", + url="https://gitlab.com/me/flake8_example", + packages=[ + "flake8_example", + ], + install_requires=requires, + entry_points={ + flake8_entry_point: [ + 'X = flake8_example.ExamplePlugin', + ], + }, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", + ], + ) + +Note specifically these lines: + +.. code-block:: python + + flake8_entry_point = # ... + + setuptools.setup( + # snip ... + entry_points={ + flake8_entry_point: [ + 'X = flake8_example.ExamplePlugin', + ], + }, + # snip ... + ) + +We tell setuptools to register our entry point "X" inside the specific +grouping of entry-points that flake8 should look in. + +Flake8 presently looks at three groups: + +- ``flake8.extension`` + +- ``flake8.format`` + +- ``flake8.report`` + +If your plugin is one that adds checks to Flake8, you will use +``flake8.extension``. If your plugin formats the output and provides that to +the user, you will use ``flake8.format``. Finally, if your plugin performs +extra report handling (filtering, etc.) it will use ``flake8.report``. + +If our ``ExamplePlugin`` is something that adds checks, our code would look +like: + +.. code-block:: python + + setuptools.setup( + # snip ... + entry_points={ + 'flake8.extension': [ + 'X = flake8_example.ExamplePlugin', + ], + }, + # snip ... + ) + + +.. _Entry Points: + https://pythonhosted.org/setuptools/pkg_resources.html#entry-points From 7828b63002fdebad678698e16eb744c494713f93 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 15:18:41 -0600 Subject: [PATCH 129/204] Finish developer documentation around formatter plugins --- docs/source/dev/formatters.rst | 34 +++++++++++++++++++++++++ docs/source/dev/registering_plugins.rst | 2 ++ 2 files changed, 36 insertions(+) diff --git a/docs/source/dev/formatters.rst b/docs/source/dev/formatters.rst index b96c40b..8f90892 100644 --- a/docs/source/dev/formatters.rst +++ b/docs/source/dev/formatters.rst @@ -14,3 +14,37 @@ Flake8 added the ability to develop custom formatting plugins in version """Flake8's example formatter.""" pass + +We notice, as soon as we start, that we inherit from Flake8's +:class:`~flake8.formatting.base.BaseFormatter` class. If we follow the +:ref:`instructions to register a plugin ` and try to use +our example formatter, e.g., ``flake8 --format=example`` then Flake8 will fail +because we did not implement the ``format`` method. Let's do that next. + +.. code-block:: python + + class Example(base.BaseFormatter): + """Flake8's example formatter.""" + + def format(self, error): + return 'Example formatter: {0!r}'.format(error) + +With that we're done. Obviously this isn't a very useful formatter, but it +should highlight the simplicitly of creating a formatter with Flake8. If we +wanted to instead create a formatter that aggregated the results and returned +XML, JSON, or subunit we could also do that. Flake8 interacts with the +formatter in two ways: + +#. It creates the formatter and provides it the options parsed from the + configuration files and command-line + +#. It uses the instance of the formatter and calls ``handle`` with the error. + +By default :meth:`flake8.formatting.base.BaseFormatter.handle` simply calls +the ``format`` method and then ``write``. Any extra handling you wish to do +for formatting purposes should override the ``handle`` method. + +API Documentation +================= + +.. autoclass:: flake8.formatting.base.BaseFormatter diff --git a/docs/source/dev/registering_plugins.rst b/docs/source/dev/registering_plugins.rst index 1444af0..c0edf63 100644 --- a/docs/source/dev/registering_plugins.rst +++ b/docs/source/dev/registering_plugins.rst @@ -1,3 +1,5 @@ +.. _register-a-plugin: + ================================== Registering a Plugin with Flake8 ================================== From 81495fd859192462ee738564c24fb14b51ebbfa1 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 6 Feb 2016 15:22:36 -0600 Subject: [PATCH 130/204] Add default formatting class --- flake8/formatting/default.py | 32 ++++++++++++++++++++++++++++++++ flake8/main/cli.py | 2 +- setup.py | 3 +++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 flake8/formatting/default.py diff --git a/flake8/formatting/default.py b/flake8/formatting/default.py new file mode 100644 index 0000000..0b69d7a --- /dev/null +++ b/flake8/formatting/default.py @@ -0,0 +1,32 @@ +"""Default formatting class for Flake8.""" +from flake8.formatting import base + + +class Default(base.BaseFormatter): + """Default formatter for Flake8. + + This also handles backwards compatibility for people specifying a custom + format string. + """ + + error_format = '%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + + def after_init(self): + """Check for a custom format string.""" + self.output_fd = None + if self.options.format.lower() != 'default': + self.error_format = self.options.format + + def format(self, error): + """Format and write error out. + + If an output filename is specified, write formatted errors to that + file. Otherwise, print the formatted error to standard out. + """ + return self.error_format % { + "code": error.code, + "text": error.text, + "path": error.filename, + "row": error.line_number, + "col": error.column_number, + } diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 51071b5..e77be84 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -54,7 +54,7 @@ def register_default_options(option_manager): # TODO(sigmavirus24): Figure out --first/--repeat add_option( - '--format', metavar='format', default='default', choices=['default'], + '--format', metavar='format', default='default', parse_from_config=True, help='Format errors according to the chosen formatter.', ) diff --git a/setup.py b/setup.py index 00b5a68..281ed00 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,9 @@ setuptools.setup( 'flake8.extension': [ 'F = flake8.plugins.pyflakes:FlakesChecker', ], + 'flake8.format': [ + 'default = flake8.formatting.default.Default', + ], }, classifiers=[ "Environment :: Console", From 56976671d0bb9ba5ea725e8aeda3387c402ce938 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 09:39:19 -0600 Subject: [PATCH 131/204] Fix entry-point names and example usage --- docs/source/dev/registering_plugins.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/dev/registering_plugins.rst b/docs/source/dev/registering_plugins.rst index c0edf63..48e39fc 100644 --- a/docs/source/dev/registering_plugins.rst +++ b/docs/source/dev/registering_plugins.rst @@ -48,7 +48,7 @@ like: install_requires=requires, entry_points={ flake8_entry_point: [ - 'X = flake8_example.ExamplePlugin', + 'X = flake8_example:ExamplePlugin', ], }, classifiers=[ @@ -73,7 +73,7 @@ Note specifically these lines: # snip ... entry_points={ flake8_entry_point: [ - 'X = flake8_example.ExamplePlugin', + 'X = flake8_example:ExamplePlugin', ], }, # snip ... @@ -86,14 +86,14 @@ Flake8 presently looks at three groups: - ``flake8.extension`` -- ``flake8.format`` +- ``flake8.listen`` - ``flake8.report`` If your plugin is one that adds checks to Flake8, you will use -``flake8.extension``. If your plugin formats the output and provides that to -the user, you will use ``flake8.format``. Finally, if your plugin performs -extra report handling (filtering, etc.) it will use ``flake8.report``. +``flake8.extension``. If your plugin automatically fixes errors in code, you +will use ``flake8.listen``. Finally, if your plugin performs extra report +handling (formatting, filtering, etc.) it will use ``flake8.report``. If our ``ExamplePlugin`` is something that adds checks, our code would look like: @@ -104,7 +104,7 @@ like: # snip ... entry_points={ 'flake8.extension': [ - 'X = flake8_example.ExamplePlugin', + 'X = flake8_example:ExamplePlugin', ], }, # snip ... From 57174a0743b55c543556984105440685c4a69287 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 09:39:58 -0600 Subject: [PATCH 132/204] Print the line not the output file object --- flake8/formatting/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8/formatting/base.py b/flake8/formatting/base.py index 85a931c..e419470 100644 --- a/flake8/formatting/base.py +++ b/flake8/formatting/base.py @@ -96,7 +96,7 @@ class BaseFormatter(object): if self.output_fd is not None: self.output_fd.write(line + self.newline) else: - print(self.output_fd) + print(line) def stop(self): """Clean up after reporting is finished.""" From 853985e1dca99e3f265c02f0f733f9c4b33106ca Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 09:40:18 -0600 Subject: [PATCH 133/204] Allow dictionary-like get behaviour with plugins --- flake8/plugins/manager.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 02a136c..3a20b91 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -203,6 +203,22 @@ class PluginTypeManager(object): LOG.debug('Retrieving plugin for "%s".', name) return self.plugins[name] + def get(self, name, default=None): + """Retrieve the plugin referred to by ``name`` or return the default. + + :param str name: + Name of the plugin to retrieve. + :param default: + Default value to return. + :returns: + Plugin object referred to by name, if it exists. + :rtype: + :class:`Plugin` + """ + if name in self: + return self[name] + return default + @property def names(self): """Proxy attribute to underlying manager.""" From 0b8f11acc2ab95f5c1a3cbe7ff7e70139104f884 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 09:40:32 -0600 Subject: [PATCH 134/204] Fix our entry-points for report formatters --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 281ed00..2ab68dd 100644 --- a/setup.py +++ b/setup.py @@ -70,8 +70,8 @@ setuptools.setup( 'flake8.extension': [ 'F = flake8.plugins.pyflakes:FlakesChecker', ], - 'flake8.format': [ - 'default = flake8.formatting.default.Default', + 'flake8.report': [ + 'default = flake8.formatting.default:Default', ], }, classifiers=[ From 2fc853b7726c5fa917adba6e64bd0d62d461d3d3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 10:44:46 -0600 Subject: [PATCH 135/204] Stop assigning output_fd in Default formatter --- flake8/formatting/default.py | 1 - flake8/main/cli.py | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/flake8/formatting/default.py b/flake8/formatting/default.py index 0b69d7a..95cc129 100644 --- a/flake8/formatting/default.py +++ b/flake8/formatting/default.py @@ -13,7 +13,6 @@ class Default(base.BaseFormatter): def after_init(self): """Check for a custom format string.""" - self.output_fd = None if self.options.format.lower() != 'default': self.error_format = self.options.format diff --git a/flake8/main/cli.py b/flake8/main/cli.py index e77be84..8cc4791 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -174,3 +174,7 @@ def main(argv=None): # Parse out our options from our found config files and user-provided CLI # options options, args = aggregator.aggregate_options(option_manager) + + # formatter = formatting_plugins.get( + # options.format, formatting_plugins['default'] + # ).execute(options) From b8a38c2573eb00f30a563ee233050ae0b5db1842 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 10:45:19 -0600 Subject: [PATCH 136/204] Add pylint formatter --- flake8/formatting/default.py | 14 ++++++++++++++ setup.py | 1 + 2 files changed, 15 insertions(+) diff --git a/flake8/formatting/default.py b/flake8/formatting/default.py index 95cc129..036d13f 100644 --- a/flake8/formatting/default.py +++ b/flake8/formatting/default.py @@ -29,3 +29,17 @@ class Default(base.BaseFormatter): "row": error.line_number, "col": error.column_number, } + + +class Pylint(Default): + """Pylint formatter for Flake8.""" + + error_format = '%(path)s:%(row)d: [%(code)s] %(text)s' + + def after_init(self): + """Do not check the value of --format. + + In the default formatter, this makes sense for backwards + compatibility, but it does not make sense here. + """ + pass diff --git a/setup.py b/setup.py index 2ab68dd..f8c3ff2 100644 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ setuptools.setup( ], 'flake8.report': [ 'default = flake8.formatting.default:Default', + 'pylint = flake8.formatting.default:Pylint', ], }, classifiers=[ From f824cbae9321527214b9fb9c3e7688c1d05f4b47 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 7 Feb 2016 19:54:12 -0600 Subject: [PATCH 137/204] Update docs index and tox env --- docs/source/dev/formatters.rst | 1 + docs/source/index.rst | 7 +++++-- tox.ini | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/source/dev/formatters.rst b/docs/source/dev/formatters.rst index 8f90892..0b37abd 100644 --- a/docs/source/dev/formatters.rst +++ b/docs/source/dev/formatters.rst @@ -48,3 +48,4 @@ API Documentation ================= .. autoclass:: flake8.formatting.base.BaseFormatter + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst index 4a50a5d..a2bd76a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,8 +18,8 @@ Plugin Developer Guide .. toctree:: :maxdepth: 2 - internal/option_handling - internal/plugin_handling + dev/formatters + dev/registering_plugins Developer Guide --------------- @@ -27,6 +27,9 @@ Developer Guide .. toctree:: :maxdepth: 2 + internal/option_handling + internal/plugin_handling + internal/utils Indices and tables ================== diff --git a/tox.ini b/tox.ini index 67c9478..8b952f3 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ basepython = python3.4 skipsdist = true skip_install = true use_develop = false -changedir = docs/_build/html +changedir = docs/build/html deps = commands = python -m http.server {posargs} From c20793b49c8421059b702943360d38fd2c76304f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 10 Feb 2016 22:44:23 -0600 Subject: [PATCH 138/204] Add internal documentation around default formatters --- docs/source/index.rst | 1 + docs/source/internal/formatters.rst | 47 +++++++++++++++++++++++++++ flake8/formatting/default.py | 49 +++++++++++++++++------------ 3 files changed, 77 insertions(+), 20 deletions(-) create mode 100644 docs/source/internal/formatters.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index a2bd76a..c8a4f35 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -27,6 +27,7 @@ Developer Guide .. toctree:: :maxdepth: 2 + internal/formatters internal/option_handling internal/plugin_handling internal/utils diff --git a/docs/source/internal/formatters.rst b/docs/source/internal/formatters.rst new file mode 100644 index 0000000..caa5718 --- /dev/null +++ b/docs/source/internal/formatters.rst @@ -0,0 +1,47 @@ +===================== + Built-in Formatters +===================== + +By default Flake8 has two formatters built-in, ``default`` and ``pylint``. +These correspond to two classes |DefaultFormatter| and |PylintFormatter|. + +In Flake8 2.0, pep8 handled formatting of errors and also allowed users to +specify an arbitrary format string as a parameter to ``--format``. In order +to allow for this backwards compatibility, Flake8 3.0 made two choices: + +#. To not limit a user's choices for ``--format`` to the format class names + +#. To make the default formatter attempt to use the string provided by the + user if it cannot find a formatter with that name. + +Default Formatter +================= + +The |DefaultFormatter| continues to use the same default format string as +pep8: ``'%(path)s:%(row)d:%(col)d: %(code)s %(text)s'``. + +In order to provide the default functionality it overrides two methods: + +#. ``after_init`` + +#. ``format`` + +The former allows us to inspect the value provided to ``--format`` by the +user and alter our own format based on that value. The second simply uses +that format string to format the error. + +.. autoclass:: flake8.formatters.default.Default + :members: + +Pylint Formatter +================ + +The |PylintFormatter| simply defines the default Pylint format string from +pep8: ``'%(path)s:%(row)d: [%(code)s] %(text)s'``. + +.. autoclass:: flake8.formatters.default.Pylint + :members: + + +.. |DefaultFormatter| replace:: :class:`~flake8.formatters.default.Default` +.. |PylintFormatter| replace:: :class:`~flake8.formatters.default.Pylint` diff --git a/flake8/formatting/default.py b/flake8/formatting/default.py index 036d13f..e0686a0 100644 --- a/flake8/formatting/default.py +++ b/flake8/formatting/default.py @@ -2,20 +2,22 @@ from flake8.formatting import base -class Default(base.BaseFormatter): - """Default formatter for Flake8. +class SimpleFormatter(base.BaseFormatter): + """Simple abstraction for Default and Pylint formatter commonality. + + Sub-classes of this need to define an ``error_format`` attribute in order + to succeed. The ``format`` method relies on that attribute and expects the + ``error_format`` string to use the old-style formatting strings with named + parameters: + + * code + * text + * path + * row + * col - This also handles backwards compatibility for people specifying a custom - format string. """ - error_format = '%(path)s:%(row)d:%(col)d: %(code)s %(text)s' - - def after_init(self): - """Check for a custom format string.""" - if self.options.format.lower() != 'default': - self.error_format = self.options.format - def format(self, error): """Format and write error out. @@ -31,15 +33,22 @@ class Default(base.BaseFormatter): } -class Pylint(Default): +class Default(SimpleFormatter): + """Default formatter for Flake8. + + This also handles backwards compatibility for people specifying a custom + format string. + """ + + error_format = '%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + + def after_init(self): + """Check for a custom format string.""" + if self.options.format.lower() != 'default': + self.error_format = self.options.format + + +class Pylint(SimpleFormatter): """Pylint formatter for Flake8.""" error_format = '%(path)s:%(row)d: [%(code)s] %(text)s' - - def after_init(self): - """Do not check the value of --format. - - In the default formatter, this makes sense for backwards - compatibility, but it does not make sense here. - """ - pass From 1d241a22c926052bce40d2ed601920f2e58e66c6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 10 Feb 2016 22:45:20 -0600 Subject: [PATCH 139/204] Sketch out how the StyleGuide would be used from main --- flake8/main/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 8cc4791..183b951 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -4,6 +4,7 @@ from flake8 import defaults from flake8.options import aggregator from flake8.options import manager from flake8.plugins import manager as plugin_manager +from flake8 import style_guide def register_default_options(option_manager): @@ -178,3 +179,5 @@ def main(argv=None): # formatter = formatting_plugins.get( # options.format, formatting_plugins['default'] # ).execute(options) + # listener_trie = listening_plugins.build_notifier() + # guide = style_guide.StyleGuide(options, args, listener_trie, formatter) From 99b96e95ceba764d6d227886bc486f243462f466 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 10 Feb 2016 22:45:50 -0600 Subject: [PATCH 140/204] Simplify StyleGuide and add handle_error method --- flake8/style_guide.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index e8451f1..7a39a4a 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -43,22 +43,21 @@ Error = collections.namedtuple('Error', ['code', class StyleGuide(object): """Manage a Flake8 user's style guide.""" - def __init__(self, options, arguments, checker_plugins, listening_plugins, - formatting_plugins): + def __init__(self, options, arguments, listener_trie, formatter): """Initialize our StyleGuide. .. todo:: Add parameter documentation. """ self.options = options self.arguments = arguments - self.checkers = checker_plugins - self.listeners = listening_plugins - self.formatters = formatting_plugins + self.listener = listener_trie + self.formatter = formatter self._selected = tuple(options.select) self._ignored = tuple(options.ignore) self._decision_cache = {} def is_user_selected(self, code): + # type: (Error) -> Union[Selected, Ignored] """Determine if the code has been selected by the user. :param str code: @@ -79,6 +78,7 @@ class StyleGuide(object): return Ignored.Implicitly def is_user_ignored(self, code): + # type: (Error) -> Union[Selected, Ignored] """Determine if the code has been ignored by the user. :param str code: @@ -96,6 +96,7 @@ class StyleGuide(object): return Selected.Implicitly def _decision_for(self, code): + # type: (Error) -> Decision startswith = code.startswith selected = sorted([s for s in self._selected if startswith(s)])[0] ignored = sorted([i for i in self._ignored if startswith(i)])[0] @@ -105,6 +106,7 @@ class StyleGuide(object): return Decision.Ignored def should_report_error(self, code): + # type: (Error) -> Decision """Determine if the error code should be reported or ignored. :param str code: @@ -133,6 +135,13 @@ class StyleGuide(object): LOG.debug('"%s" will be "%s"', code, decision) return decision + def handle_error(self, code, filename, line_number, column_number, text): + # type: (str, str, int, int, str) -> NoneType + """Handle an error reported by a check.""" + error = Error(code, filename, line_number, column_number, text) + if self.should_report_error(error): + self.formatter.handle(error) + self.notifier.notify(error.code, error) # Should separate style guide logic from code that runs checks # StyleGuide should manage select/ignore logic as well as include/exclude From a9a939cbbc93f860aee607d1d1f40068f10067a6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 15 Feb 2016 22:31:56 -0600 Subject: [PATCH 141/204] Update StyleGuide tests and add new tests for handle_error --- flake8/style_guide.py | 4 +- tests/unit/test_style_guide.py | 83 ++++++++++++++++++++++++++-------- 2 files changed, 67 insertions(+), 20 deletions(-) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 7a39a4a..98e21c8 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -139,9 +139,9 @@ class StyleGuide(object): # type: (str, str, int, int, str) -> NoneType """Handle an error reported by a check.""" error = Error(code, filename, line_number, column_number, text) - if self.should_report_error(error): + if self.should_report_error(error.code) is Decision.Selected: self.formatter.handle(error) - self.notifier.notify(error.code, error) + self.listener.notify(error.code, error) # Should separate style guide logic from code that runs checks # StyleGuide should manage select/ignore logic as well as include/exclude diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py index 51e9f61..f1e18b6 100644 --- a/tests/unit/test_style_guide.py +++ b/tests/unit/test_style_guide.py @@ -1,8 +1,11 @@ """Tests for the flake8.style_guide.StyleGuide class.""" import optparse +from flake8.formatting import base +from flake8.plugins import notifier from flake8 import style_guide +import mock import pytest @@ -24,9 +27,8 @@ def test_is_user_ignored_ignores_errors(ignore_list, error_code): """Verify we detect users explicitly ignoring an error.""" guide = style_guide.StyleGuide(create_options(ignore=ignore_list), arguments=[], - checker_plugins=None, - listening_plugins=None, - formatting_plugins=None) + listener_trie=None, + formatter=None) assert guide.is_user_ignored(error_code) is style_guide.Ignored.Explicitly @@ -42,9 +44,8 @@ def test_is_user_ignored_implicitly_selects_errors(ignore_list, error_code): """Verify we detect users does not explicitly ignore an error.""" guide = style_guide.StyleGuide(create_options(ignore=ignore_list), arguments=[], - checker_plugins=None, - listening_plugins=None, - formatting_plugins=None) + listener_trie=None, + formatter=None) assert guide.is_user_ignored(error_code) is style_guide.Selected.Implicitly @@ -60,9 +61,8 @@ def test_is_user_selected_selects_errors(select_list, error_code): """Verify we detect users explicitly selecting an error.""" guide = style_guide.StyleGuide(create_options(select=select_list), arguments=[], - checker_plugins=None, - listening_plugins=None, - formatting_plugins=None) + listener_trie=None, + formatter=None) assert (guide.is_user_selected(error_code) is style_guide.Selected.Explicitly) @@ -74,9 +74,8 @@ def test_is_user_selected_implicitly_selects_errors(): error_code = 'E121' guide = style_guide.StyleGuide(create_options(select=select_list), arguments=[], - checker_plugins=None, - listening_plugins=None, - formatting_plugins=None) + listener_trie=None, + formatter=None) assert (guide.is_user_selected(error_code) is style_guide.Selected.Implicitly) @@ -93,9 +92,8 @@ def test_is_user_selected_excludes_errors(select_list, error_code): """Verify we detect users implicitly excludes an error.""" guide = style_guide.StyleGuide(create_options(select=select_list), arguments=[], - checker_plugins=None, - listening_plugins=None, - formatting_plugins=None) + listener_trie=None, + formatter=None) assert guide.is_user_selected(error_code) is style_guide.Ignored.Implicitly @@ -121,8 +119,57 @@ def test_should_report_error(select_list, ignore_list, error_code, expected): guide = style_guide.StyleGuide(create_options(select=select_list, ignore=ignore_list), arguments=[], - checker_plugins=None, - listening_plugins=None, - formatting_plugins=None) + listener_trie=None, + formatter=None) assert guide.should_report_error(error_code) is expected + + +@pytest.mark.parametrize('select_list,ignore_list,error_code', [ + (['E111', 'E121'], [], 'E111'), + (['E111', 'E121'], [], 'E121'), + (['E11', 'E121'], ['E1'], 'E112'), + ([], ['E2', 'E12'], 'E410'), + (['E41'], ['E2', 'E12', 'E4'], 'E410'), +]) +def test_handle_error_notifies_listeners(select_list, ignore_list, error_code): + """Verify that error codes notify the listener trie appropriately.""" + listener_trie = mock.create_autospec(notifier.Notifier, instance=True) + formatter = mock.create_autospec(base.BaseFormatter, instance=True) + guide = style_guide.StyleGuide(create_options(select=select_list, + ignore=ignore_list), + arguments=[], + listener_trie=listener_trie, + formatter=formatter) + + guide.handle_error(error_code, 'stdin', 1, 1, 'error found') + error = style_guide.Error(error_code, 'stdin', 1, 1, 'error found') + listener_trie.notify.assert_called_once_with(error_code, error) + formatter.handle.assert_called_once_with(error) + + +@pytest.mark.parametrize('select_list,ignore_list,error_code', [ + (['E111', 'E121'], [], 'E122'), + (['E11', 'E12'], [], 'E132'), + (['E2', 'E12'], [], 'E321'), + (['E2', 'E12'], [], 'E410'), + (['E111', 'E121'], ['E2'], 'E122'), + (['E11', 'E12'], ['E13'], 'E132'), + (['E1', 'E3'], ['E32'], 'E321'), + (['E4'], ['E2', 'E12', 'E41'], 'E410'), + (['E111', 'E121'], [], 'E112'), +]) +def test_handle_error_does_not_notify_listeners(select_list, ignore_list, + error_code): + """Verify that error codes notify the listener trie appropriately.""" + listener_trie = mock.create_autospec(notifier.Notifier, instance=True) + formatter = mock.create_autospec(base.BaseFormatter, instance=True) + guide = style_guide.StyleGuide(create_options(select=select_list, + ignore=ignore_list), + arguments=[], + listener_trie=listener_trie, + formatter=formatter) + + guide.handle_error(error_code, 'stdin', 1, 1, 'error found') + assert listener_trie.notify.called is False + assert formatter.handle.called is False From f4b18229a0294249be22076ad6a52e244acb5a5a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 09:08:24 -0600 Subject: [PATCH 142/204] Add checking for inline #noqa comments This also supports ignoring only specified codes --- flake8/style_guide.py | 50 +++++++++++++++++++++++++++++++++- tests/unit/test_style_guide.py | 6 ++-- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 98e21c8..af68dce 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -1,9 +1,13 @@ """Implementation of the StyleGuide used by Flake8.""" import collections +import linecache import logging +import re import enum +from flake8 import utils + __all__ = ( 'StyleGuide', ) @@ -43,6 +47,20 @@ Error = collections.namedtuple('Error', ['code', class StyleGuide(object): """Manage a Flake8 user's style guide.""" + NOQA_INLINE_REGEXP = re.compile( + # We're looking for items that look like this: + # ``# noqa`` + # ``# noqa: E123`` + # ``# noqa: E123,W451,F921`` + # ``# NoQA: E123,W451,F921`` + # ``# NOQA: E123,W451,F921`` + # We do not care about the ``: `` that follows ``noqa`` + # We do not care about the casing of ``noqa`` + # We want a comma-separated list of errors + '# noqa(?:: )?(?P[A-Z0-9,]+)?$', + re.IGNORECASE + ) + def __init__(self, options, arguments, listener_trie, formatter): """Initialize our StyleGuide. @@ -109,6 +127,12 @@ class StyleGuide(object): # type: (Error) -> Decision """Determine if the error code should be reported or ignored. + This method only cares about the select and ignore rules as specified + by the user in their configuration files and command-line flags. + + This method does not look at whether the specific line is being + ignored in the file itself. + :param str code: The code for the check that has been run. """ @@ -135,11 +159,35 @@ class StyleGuide(object): LOG.debug('"%s" will be "%s"', code, decision) return decision + def is_inline_ignored(self, error): + """Determine if an comment has been added to ignore this line.""" + physical_line = linecache.getline(error.filename, error.line_number) + noqa_match = self.NOQA_INLINE_REGEXP.search(physical_line) + if noqa_match is None: + LOG.debug('%r is not inline ignored', error) + return False + + codes_str = noqa_match.groupdict()['codes'] + if codes_str is None: + LOG.debug('%r is ignored by a blanket ``# noqa``', error) + return True + + codes = set(utils.parse_comma_separated_list(codes_str)) + if error.code in codes: + LOG.debug('%r is ignored specifically inline with ``# noqa: %s``', + error, codes_str) + return True + + LOG.debug('%r is not ignored inline with ``# noqa: %s``', + error, codes_str) + return False + def handle_error(self, code, filename, line_number, column_number, text): # type: (str, str, int, int, str) -> NoneType """Handle an error reported by a check.""" error = Error(code, filename, line_number, column_number, text) - if self.should_report_error(error.code) is Decision.Selected: + if (self.should_report_error(error.code) is Decision.Selected and + self.is_inline_ignored(error) is False): self.formatter.handle(error) self.listener.notify(error.code, error) diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py index f1e18b6..e02fdc7 100644 --- a/tests/unit/test_style_guide.py +++ b/tests/unit/test_style_guide.py @@ -142,7 +142,8 @@ def test_handle_error_notifies_listeners(select_list, ignore_list, error_code): listener_trie=listener_trie, formatter=formatter) - guide.handle_error(error_code, 'stdin', 1, 1, 'error found') + with mock.patch('linecache.getline', return_value=''): + guide.handle_error(error_code, 'stdin', 1, 1, 'error found') error = style_guide.Error(error_code, 'stdin', 1, 1, 'error found') listener_trie.notify.assert_called_once_with(error_code, error) formatter.handle.assert_called_once_with(error) @@ -170,6 +171,7 @@ def test_handle_error_does_not_notify_listeners(select_list, ignore_list, listener_trie=listener_trie, formatter=formatter) - guide.handle_error(error_code, 'stdin', 1, 1, 'error found') + with mock.patch('linecache.getline', return_value=''): + guide.handle_error(error_code, 'stdin', 1, 1, 'error found') assert listener_trie.notify.called is False assert formatter.handle.called is False From fbd5944f15f90114d4d9c2e299ec26a30c646d82 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 11:02:22 -0600 Subject: [PATCH 143/204] Add specific tests for is_inline_ignored Update the logic so someone can use a class in their ``# noqa`` ignore list --- flake8/style_guide.py | 2 +- tests/unit/test_style_guide.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index af68dce..6a05249 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -173,7 +173,7 @@ class StyleGuide(object): return True codes = set(utils.parse_comma_separated_list(codes_str)) - if error.code in codes: + if error.code in codes or error.code.startswith(tuple(codes)): LOG.debug('%r is ignored specifically inline with ``# noqa: %s``', error, codes_str) return True diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py index e02fdc7..4ae4f5f 100644 --- a/tests/unit/test_style_guide.py +++ b/tests/unit/test_style_guide.py @@ -125,6 +125,26 @@ def test_should_report_error(select_list, ignore_list, error_code, expected): assert guide.should_report_error(error_code) is expected +@pytest.mark.parametrize('error_code,physical_line,expected_result', [ + ('E111', 'a = 1', False), + ('E121', 'a = 1 # noqa: E111', False), + ('E121', 'a = 1 # noqa: E111,W123,F821', False), + ('E111', 'a = 1 # noqa: E111,W123,F821', True), + ('W123', 'a = 1 # noqa: E111,W123,F821', True), + ('E111', 'a = 1 # noqa: E11,W123,F821', True), +]) +def test_is_inline_ignored(error_code, physical_line, expected_result): + """Verify that we detect inline usage of ``# noqa``.""" + guide = style_guide.StyleGuide(create_options(select=['E', 'W', 'F']), + arguments=[], + listener_trie=None, + formatter=None) + error = style_guide.Error(error_code, 'filename.py', 1, 1, 'error text') + + with mock.patch('linecache.getline', return_value=physical_line): + assert guide.is_inline_ignored(error) is expected_result + + @pytest.mark.parametrize('select_list,ignore_list,error_code', [ (['E111', 'E121'], [], 'E111'), (['E111', 'E121'], [], 'E121'), From 85c199ea34d39bdb811249d26fc535a2228139c6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 15:10:17 -0600 Subject: [PATCH 144/204] Add pylint config and testenv --- .pylintrc | 378 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 11 ++ 2 files changed, 389 insertions(+) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..8b82de9 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,378 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS,.git,flake8.egg-info + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=intern-builtin,nonzero-method,parameter-unpacking,backtick,raw_input-builtin,dict-view-method,filter-builtin-not-iterating,long-builtin,unichr-builtin,input-builtin,unicode-builtin,file-builtin,map-builtin-not-iterating,delslice-method,apply-builtin,cmp-method,setslice-method,coerce-method,long-suffix,raising-string,import-star-module-level,buffer-builtin,reload-builtin,unpacking-in-except,print-statement,hex-method,old-octal-literal,metaclass-assignment,dict-iter-method,range-builtin-not-iterating,using-cmp-argument,indexing-exception,no-absolute-import,coerce-builtin,getslice-method,suppressed-message,execfile-builtin,round-builtin,useless-suppression,reduce-builtin,old-raise-syntax,zip-builtin-not-iterating,cmp-builtin,xrange-builtin,standarderror-builtin,old-division,oct-method,next-method-called,old-ne-operator,basestring-builtin + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=100.0 - ((float(5 * error + warning + refactor + convention) / statement) * 100) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=yes + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes= + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=20 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=10 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/tox.ini b/tox.ini index 8b952f3..9b56c4c 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,17 @@ deps = commands = flake8 +[testenv:pylint] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + . + pylint +commands = + pylint flake8 + [testenv:docs] deps = sphinx>=1.3.0 From 69b8be71dca9f0d8b9a22189801a4a4c8328f07f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 15:10:46 -0600 Subject: [PATCH 145/204] Make pylint happier --- flake8/__init__.py | 12 ++++++------ flake8/formatting/default.py | 2 ++ flake8/main/cli.py | 13 +++++++------ flake8/options/aggregator.py | 3 +-- flake8/options/config.py | 9 +++++---- flake8/options/manager.py | 16 +++++++++++----- flake8/plugins/manager.py | 12 ++++++++---- flake8/plugins/pyflakes.py | 23 +++++++++++++---------- flake8/style_guide.py | 8 ++++---- flake8/utils.py | 9 +++++---- tests/unit/test_config_file_finder.py | 4 ++-- 11 files changed, 64 insertions(+), 47 deletions(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index a6520e4..996f036 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -10,8 +10,6 @@ This module """ import logging -import sys - try: from logging import NullHandler except ImportError: @@ -21,6 +19,7 @@ except ImportError: def emit(self, record): """Do nothing.""" pass +import sys LOG = logging.getLogger(__name__) LOG.addHandler(NullHandler()) @@ -51,7 +50,6 @@ def configure_logging(verbosity, filename=None, If the name is "stdout" or "stderr" this will log to the appropriate stream. """ - global LOG if verbosity <= 0: return if verbosity > 2: @@ -60,12 +58,14 @@ def configure_logging(verbosity, filename=None, log_level = _VERBOSITY_TO_LOG_LEVEL[verbosity] if not filename or filename in ('stderr', 'stdout'): - handler = logging.StreamHandler(getattr(sys, filename)) + fileobj = getattr(sys, filename or 'stderr') + handler = logging.StreamHandler else: - handler = logging.FileHandler(filename) + fileobj = filename + handler = logging.FileHandler handler.setFormatter(logging.Formatter(logformat)) - LOG.addHandler(handler) + LOG.addHandler(handler(fileobj)) LOG.setLevel(log_level) LOG.debug('Added a %s logging handler to logger root at %s', filename, __name__) diff --git a/flake8/formatting/default.py b/flake8/formatting/default.py index e0686a0..bef8c88 100644 --- a/flake8/formatting/default.py +++ b/flake8/formatting/default.py @@ -18,6 +18,8 @@ class SimpleFormatter(base.BaseFormatter): """ + error_format = None + def format(self, error): """Format and write error out. diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 183b951..c9503f1 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -174,10 +174,11 @@ def main(argv=None): # Parse out our options from our found config files and user-provided CLI # options - options, args = aggregator.aggregate_options(option_manager) + options, args = aggregator.aggregate_options(option_manager, argv) - # formatter = formatting_plugins.get( - # options.format, formatting_plugins['default'] - # ).execute(options) - # listener_trie = listening_plugins.build_notifier() - # guide = style_guide.StyleGuide(options, args, listener_trie, formatter) + formatter = formatting_plugins.get( + options.format, formatting_plugins['default'] + ).execute(options) + listener_trie = listening_plugins.build_notifier() + guide = style_guide.StyleGuide(options, args, listener_trie, formatter) + guide.handle_error('E111', 'stdin', 1, 1, 'faketext') diff --git a/flake8/options/aggregator.py b/flake8/options/aggregator.py index 5c02730..b344cf0 100644 --- a/flake8/options/aggregator.py +++ b/flake8/options/aggregator.py @@ -71,5 +71,4 @@ def aggregate_options(manager, arglist=None, values=None): setattr(default_values, dest_name, value) # Finally parse the command-line options - final_values, args = manager.parse_args(arglist, default_values) - return final_values, args + return manager.parse_args(arglist, default_values) diff --git a/flake8/options/config.py b/flake8/options/config.py index 2484cdd..ecc40f7 100644 --- a/flake8/options/config.py +++ b/flake8/options/config.py @@ -66,7 +66,7 @@ class ConfigFileFinder(object): LOG.debug('Found cli configuration files: %s', found_files) return config - def generate_possible_local_config_files(self): + def generate_possible_local_files(self): """Find and generate all local config files.""" tail = self.tail parent = self.parent @@ -84,7 +84,7 @@ class ConfigFileFinder(object): """Find all local config files which actually exist. Filter results from - :meth:`~ConfigFileFinder.generate_possible_local_config_files` based + :meth:`~ConfigFileFinder.generate_possible_local_files` based on whether the filename exists or not. :returns: @@ -93,11 +93,12 @@ class ConfigFileFinder(object): :rtype: [str] """ + exists = os.path.exists return [ filename - for filename in self.generate_possible_local_config_files() + for filename in self.generate_possible_local_files() if os.path.exists(filename) - ] + list(filter(os.path.exists, self.extra_config_files)) + ] + [f for f in self.extra_config_files if exists(f)] def local_configs(self): """Parse all local config files into one config object.""" diff --git a/flake8/options/manager.py b/flake8/options/manager.py index 9faaf7c..cb4c831 100644 --- a/flake8/options/manager.py +++ b/flake8/options/manager.py @@ -1,6 +1,6 @@ """Option handling and Option management logic.""" import logging -import optparse +import optparse # pylint: disable=deprecated-module from flake8 import utils @@ -18,8 +18,7 @@ class Option(object): metavar=None, # Options below here are specific to Flake8 parse_from_config=False, comma_separated_list=False, - normalize_paths=False, - ): + normalize_paths=False): """Initialize an Option instance wrapping optparse.Option. The following are all passed directly through to optparse. @@ -69,12 +68,17 @@ class Option(object): """ self.short_option_name = short_option_name self.long_option_name = long_option_name - self.option_args = filter(None, (short_option_name, long_option_name)) + self.option_args = [ + x for x in (short_option_name, long_option_name) if x is not None + ] self.option_kwargs = { 'action': action, 'default': default, 'type': type, 'dest': self._make_dest(dest), + 'nargs': nargs, + 'const': const, + 'choices': choices, 'callback': callback, 'callback_args': callback_args, 'callback_kwargs': callback_kwargs, @@ -97,6 +101,8 @@ class Option(object): 'a long_option_name must also be specified.') self.config_name = long_option_name[2:].replace('-', '_') + self._opt = None + def __repr__(self): """Simple representation of an Option class.""" return ( @@ -129,7 +135,7 @@ class Option(object): def to_optparse(self): """Convert a Flake8 Option to an optparse Option.""" - if not hasattr(self, '_opt'): + if self._opt is None: self._opt = optparse.Option(*self.option_args, **self.option_kwargs) return self._opt diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 3a20b91..21171ef 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -57,7 +57,7 @@ class Plugin(object): def execute(self, *args, **kwargs): r"""Call the plugin with \*args and \*\*kwargs.""" - return self.plugin(*args, **kwargs) + return self.plugin(*args, **kwargs) # pylint: disable=not-callable def _load(self, verify_requirements): # Avoid relying on hasattr() here. @@ -134,7 +134,7 @@ class Plugin(object): ) -class PluginManager(object): +class PluginManager(object): # pylint: disable=too-few-public-methods """Find and manage plugins consistently.""" def __init__(self, namespace, verify_requirements=False): @@ -188,6 +188,8 @@ class PluginManager(object): class PluginTypeManager(object): """Parent class for most of the specific plugin types.""" + namespace = None + def __init__(self): """Initialize the plugin type's manager.""" self.manager = PluginManager(self.namespace) @@ -232,6 +234,7 @@ class PluginTypeManager(object): @staticmethod def _generate_call_function(method_name, optmanager, *args, **kwargs): def generated_function(plugin): + """Function that attempts to call a specific method on a plugin.""" method = getattr(plugin, method_name, None) if (method is not None and isinstance(method, collections.Callable)): @@ -244,6 +247,7 @@ class PluginTypeManager(object): return def load_plugin(plugin): + """Call each plugin's load_plugin method.""" return plugin.load_plugin() plugins = list(self.manager.map(load_plugin)) @@ -269,7 +273,7 @@ class PluginTypeManager(object): list(self.manager.map(call_provide_options)) -class NotifierBuilder(object): +class NotifierBuilderMixin(object): # pylint: disable=too-few-public-methods """Mixin class that builds a Notifier from a PluginManager.""" def build_notifier(self): @@ -293,7 +297,7 @@ class Checkers(PluginTypeManager): namespace = 'flake8.extension' -class Listeners(PluginTypeManager, NotifierBuilder): +class Listeners(PluginTypeManager, NotifierBuilderMixin): """All of the listeners registered through entry-points.""" namespace = 'flake8.listen' diff --git a/flake8/plugins/pyflakes.py b/flake8/plugins/pyflakes.py index 7f6107a..f512511 100644 --- a/flake8/plugins/pyflakes.py +++ b/flake8/plugins/pyflakes.py @@ -23,7 +23,7 @@ def patch_pyflakes(): 'F402 ImportShadowedByLoopVar', 'F403 ImportStarUsed', 'F404 LateFutureImport', - 'F810 Redefined', # XXX Obsolete? + 'F810 Redefined', 'F811 RedefinedWhileUnused', 'F812 RedefinedInListComp', 'F821 UndefinedName', @@ -48,23 +48,23 @@ class FlakesChecker(pyflakes.checker.Checker): def __init__(self, tree, filename): """Initialize the PyFlakes plugin with an AST tree and filename.""" filename = utils.normalize_paths(filename)[0] - withDoctest = self.withDoctest + with_doctest = self.with_doctest included_by = [include for include in self.include_in_doctest if include != '' and filename.startswith(include)] if included_by: - withDoctest = True + with_doctest = True for exclude in self.exclude_from_doctest: if exclude != '' and filename.startswith(exclude): - withDoctest = False + with_doctest = False overlaped_by = [include for include in included_by if include.startswith(exclude)] if overlaped_by: - withDoctest = True + with_doctest = True super(FlakesChecker, self).__init__(tree, filename, - withDoctest=withDoctest) + withDoctest=with_doctest) @classmethod def add_options(cls, parser): @@ -98,7 +98,7 @@ class FlakesChecker(pyflakes.checker.Checker): """Parse option values from Flake8's OptionManager.""" if options.builtins: cls.builtIns = cls.builtIns.union(options.builtins) - cls.withDoctest = options.doctests + cls.with_doctest = options.doctests included_files = [] for included_file in options.include_in_doctest: @@ -131,6 +131,9 @@ class FlakesChecker(pyflakes.checker.Checker): def run(self): """Run the plugin.""" - for m in self.messages: - col = getattr(m, 'col', 0) - yield m.lineno, col, (m.flake8_msg % m.message_args), m.__class__ + for message in self.messages: + col = getattr(message, 'col', 0) + yield (message.lineno, + col, + (message.flake8_msg % message.message_args), + message.__class__) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 6a05249..00a46fe 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -145,15 +145,15 @@ class StyleGuide(object): code, selected, ignored) if ((selected is Selected.Explicitly or - selected is Selected.Implicitly) and + selected is Selected.Implicitly) and ignored is Selected.Implicitly): decision = Decision.Selected elif (selected is Selected.Explicitly and - ignored is Ignored.Explicitly): + ignored is Ignored.Explicitly): decision = self._decision_for(code) elif (selected is Ignored.Implicitly or - ignored is Ignored.Explicitly): - decision = Decision.Ignored + ignored is Ignored.Explicitly): + decision = Decision.Ignored # pylint: disable=R0204 self._decision_cache[code] = decision LOG.debug('"%s" will be "%s"', code, decision) diff --git a/flake8/utils.py b/flake8/utils.py index 98e5cea..1165470 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -33,7 +33,8 @@ def normalize_paths(paths, parent=os.curdir): :rtype: [str] """ - return [normalize_path(p) for p in parse_comma_separated_list(paths)] + return [normalize_path(p, parent) + for p in parse_comma_separated_list(paths)] def normalize_path(path, parent=os.curdir): @@ -57,8 +58,8 @@ def stdin_get_value(): if cached_value is None: stdin_value = sys.stdin.read() if sys.version_info < (3, 0): - cached_value = io.BytesIO(stdin_value) + cached_type = io.BytesIO else: - cached_value = io.StringIO(stdin_value) - stdin_get_value.cached_stdin = cached_value + cached_type = io.StringIO + stdin_get_value.cached_stdin = cached_type(stdin_value) return cached_value.getvalue() diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 200f4e5..6ac1d16 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -68,11 +68,11 @@ def test_cli_config(): os.path.abspath('tox.ini'), os.path.abspath('.flake8')]), ]) -def test_generate_possible_local_config_files(args, expected): +def test_generate_possible_local_files(args, expected): """Verify generation of all possible config paths.""" finder = config.ConfigFileFinder('flake8', args, []) - assert (list(finder.generate_possible_local_config_files()) == + assert (list(finder.generate_possible_local_files()) == expected) From bda3124ffafa1f6babfb16f14dbdced9ff68e722 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 15:14:10 -0600 Subject: [PATCH 146/204] Increase the confidence level for pylint to report something --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 8b82de9..ed80ac9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -43,7 +43,7 @@ optimize-ast=no # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +confidence=INFERENCE_FAILURE # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 0827df3e2ad1c0e6787e3fc85cae0a9b266af416 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 15:30:54 -0600 Subject: [PATCH 147/204] Fix our evaluation formula --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index ed80ac9..f778dd4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -82,7 +82,7 @@ reports=no # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). -evaluation=100.0 - ((float(5 * error + warning + refactor + convention) / statement) * 100) +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details From 6c90e1045b64670f3ecb63289cd67a7be04284eb Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 16:07:28 -0600 Subject: [PATCH 148/204] Fix tests while keeping pylint happy --- flake8/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index 996f036..8e32938 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -59,13 +59,14 @@ def configure_logging(verbosity, filename=None, if not filename or filename in ('stderr', 'stdout'): fileobj = getattr(sys, filename or 'stderr') - handler = logging.StreamHandler + handler_cls = logging.StreamHandler else: fileobj = filename - handler = logging.FileHandler + handler_cls = logging.FileHandler + handler = handler_cls(fileobj) handler.setFormatter(logging.Formatter(logformat)) - LOG.addHandler(handler(fileobj)) + LOG.addHandler(handler) LOG.setLevel(log_level) LOG.debug('Added a %s logging handler to logger root at %s', filename, __name__) From 6d3d955ba976279fa3f31a805ebe5dab7e6cc010 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 16:10:00 -0600 Subject: [PATCH 149/204] Fix other tests after pylint fixes --- tests/unit/test_option.py | 3 +++ tests/unit/test_plugin_type_manager.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py index 5facf2a..67e2255 100644 --- a/tests/unit/test_option.py +++ b/tests/unit/test_option.py @@ -33,6 +33,9 @@ def test_to_optparse_creates_an_option_as_we_expect(Option): 'default': None, 'type': None, 'dest': 'test', + 'nargs': None, + 'const': None, + 'choices': None, 'callback': None, 'callback_args': None, 'callback_kwargs': None, diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py index 36fb92a..f64e472 100644 --- a/tests/unit/test_plugin_type_manager.py +++ b/tests/unit/test_plugin_type_manager.py @@ -200,8 +200,8 @@ def test_proxies_getitem_to_managers_plugins_dictionary(PluginManager): assert type_mgr[key] is plugins[key] -class FakePluginTypeManager(manager.NotifierBuilder): - """Provide an easy way to test the NotifierBuilder.""" +class FakePluginTypeManager(manager.NotifierBuilderMixin): + """Provide an easy way to test the NotifierBuilderMixin.""" def __init__(self, manager): """Initialize with our fake manager.""" From 58b2ca69e466ed8669d830dc6a95fc32363448d7 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 16:10:38 -0600 Subject: [PATCH 150/204] Remove arguments from StyleGuide - StyleGuides do not need the arguments passed in - Add a test for is_inline_ignored obeying disable_noqa --- flake8/main/cli.py | 2 +- flake8/style_guide.py | 7 +++++-- tests/unit/test_style_guide.py | 23 ++++++++++++++--------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index c9503f1..670e755 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -180,5 +180,5 @@ def main(argv=None): options.format, formatting_plugins['default'] ).execute(options) listener_trie = listening_plugins.build_notifier() - guide = style_guide.StyleGuide(options, args, listener_trie, formatter) + guide = style_guide.StyleGuide(options, listener_trie, formatter) guide.handle_error('E111', 'stdin', 1, 1, 'faketext') diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 00a46fe..5deff7a 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -61,13 +61,12 @@ class StyleGuide(object): re.IGNORECASE ) - def __init__(self, options, arguments, listener_trie, formatter): + def __init__(self, options, listener_trie, formatter): """Initialize our StyleGuide. .. todo:: Add parameter documentation. """ self.options = options - self.arguments = arguments self.listener = listener_trie self.formatter = formatter self._selected = tuple(options.select) @@ -161,6 +160,10 @@ class StyleGuide(object): def is_inline_ignored(self, error): """Determine if an comment has been added to ignore this line.""" + # TODO(sigmavirus24): Determine how to handle stdin with linecache + if self.options.disable_noqa: + return False + physical_line = linecache.getline(error.filename, error.line_number) noqa_match = self.NOQA_INLINE_REGEXP.search(physical_line) if noqa_match is None: diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py index 4ae4f5f..03bb341 100644 --- a/tests/unit/test_style_guide.py +++ b/tests/unit/test_style_guide.py @@ -13,6 +13,7 @@ def create_options(**kwargs): """Create and return an instance of optparse.Values.""" kwargs.setdefault('select', []) kwargs.setdefault('ignore', []) + kwargs.setdefault('disable_noqa', False) return optparse.Values(kwargs) @@ -26,7 +27,6 @@ def create_options(**kwargs): def test_is_user_ignored_ignores_errors(ignore_list, error_code): """Verify we detect users explicitly ignoring an error.""" guide = style_guide.StyleGuide(create_options(ignore=ignore_list), - arguments=[], listener_trie=None, formatter=None) @@ -43,7 +43,6 @@ def test_is_user_ignored_ignores_errors(ignore_list, error_code): def test_is_user_ignored_implicitly_selects_errors(ignore_list, error_code): """Verify we detect users does not explicitly ignore an error.""" guide = style_guide.StyleGuide(create_options(ignore=ignore_list), - arguments=[], listener_trie=None, formatter=None) @@ -60,7 +59,6 @@ def test_is_user_ignored_implicitly_selects_errors(ignore_list, error_code): def test_is_user_selected_selects_errors(select_list, error_code): """Verify we detect users explicitly selecting an error.""" guide = style_guide.StyleGuide(create_options(select=select_list), - arguments=[], listener_trie=None, formatter=None) @@ -73,7 +71,6 @@ def test_is_user_selected_implicitly_selects_errors(): select_list = [] error_code = 'E121' guide = style_guide.StyleGuide(create_options(select=select_list), - arguments=[], listener_trie=None, formatter=None) @@ -91,7 +88,6 @@ def test_is_user_selected_implicitly_selects_errors(): def test_is_user_selected_excludes_errors(select_list, error_code): """Verify we detect users implicitly excludes an error.""" guide = style_guide.StyleGuide(create_options(select=select_list), - arguments=[], listener_trie=None, formatter=None) @@ -118,7 +114,6 @@ def test_should_report_error(select_list, ignore_list, error_code, expected): """Verify we decide when to report an error.""" guide = style_guide.StyleGuide(create_options(select=select_list, ignore=ignore_list), - arguments=[], listener_trie=None, formatter=None) @@ -136,7 +131,6 @@ def test_should_report_error(select_list, ignore_list, error_code, expected): def test_is_inline_ignored(error_code, physical_line, expected_result): """Verify that we detect inline usage of ``# noqa``.""" guide = style_guide.StyleGuide(create_options(select=['E', 'W', 'F']), - arguments=[], listener_trie=None, formatter=None) error = style_guide.Error(error_code, 'filename.py', 1, 1, 'error text') @@ -145,6 +139,19 @@ def test_is_inline_ignored(error_code, physical_line, expected_result): assert guide.is_inline_ignored(error) is expected_result +def test_disable_is_inline_ignored(): + """Verify that is_inline_ignored exits immediately if disabling NoQA.""" + guide = style_guide.StyleGuide(create_options(disable_noqa=True), + listener_trie=None, + formatter=None) + error = style_guide.Error('E121', 'filename.py', 1, 1, 'error text') + + with mock.patch('linecache.getline') as getline: + assert guide.is_inline_ignored(error) is False + + assert getline.called is False + + @pytest.mark.parametrize('select_list,ignore_list,error_code', [ (['E111', 'E121'], [], 'E111'), (['E111', 'E121'], [], 'E121'), @@ -158,7 +165,6 @@ def test_handle_error_notifies_listeners(select_list, ignore_list, error_code): formatter = mock.create_autospec(base.BaseFormatter, instance=True) guide = style_guide.StyleGuide(create_options(select=select_list, ignore=ignore_list), - arguments=[], listener_trie=listener_trie, formatter=formatter) @@ -187,7 +193,6 @@ def test_handle_error_does_not_notify_listeners(select_list, ignore_list, formatter = mock.create_autospec(base.BaseFormatter, instance=True) guide = style_guide.StyleGuide(create_options(select=select_list, ignore=ignore_list), - arguments=[], listener_trie=listener_trie, formatter=formatter) From 98357e71db4591ecf9e74321608def35125aa40d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 18:55:01 -0600 Subject: [PATCH 151/204] Fix Notifier.listeners_for If no Trie.find returns None, then node.data will return an AttributeError. --- flake8/plugins/notifier.py | 3 ++- tests/unit/test_notifier.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/flake8/plugins/notifier.py b/flake8/plugins/notifier.py index 323ea1d..dc255c4 100644 --- a/flake8/plugins/notifier.py +++ b/flake8/plugins/notifier.py @@ -31,7 +31,8 @@ class Notifier(object): path = error_code while path: node = self.listeners.find(path) - for listener in node.data: + listeners = getattr(node, 'data', []) + for listener in listeners: yield listener path = path[:-1] diff --git a/tests/unit/test_notifier.py b/tests/unit/test_notifier.py index effcc88..6a162cf 100644 --- a/tests/unit/test_notifier.py +++ b/tests/unit/test_notifier.py @@ -38,3 +38,17 @@ class TestNotifier(object): self.notifier.notify('E111', 'extra', 'args') assert self.listener_map['E111'].was_notified is True assert self.listener_map['E1'].was_notified is True + + @pytest.mark.parametrize('code', ['W123', 'W12', 'W1', 'W']) + def test_no_listeners_for(self, code): + """Show that we return an empty list of listeners.""" + assert list(self.notifier.listeners_for(code)) == [] + + @pytest.mark.parametrize('code,expected', [ + ('E101', ['E101', 'E1']), + ('E211', ['E211', 'E2']), + ]) + def test_listeners_for(self, code, expected): + """Verify that we retrieve the correct listeners.""" + assert ([l.error_code for l in self.notifier.listeners_for(code)] == + expected) From 8e2d8ec6f59fdd055a88a40d743782f4feaa8376 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 18:58:42 -0600 Subject: [PATCH 152/204] Update the default log format --- flake8/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index 8e32938..416c951 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -37,9 +37,10 @@ _VERBOSITY_TO_LOG_LEVEL = { 2: logging.DEBUG, } +LOG_FORMAT = '[flake8] %(asctime)s %(levelname)s %(message)s' -def configure_logging(verbosity, filename=None, - logformat='%(asctime)s %(levelname)s %(message)s'): + +def configure_logging(verbosity, filename=None, logformat=LOG_FORMAT): """Configure logging for flake8. :param int verbosity: From 8914ccd582d7e664340b90ed789bc2c46e983a75 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 18:59:38 -0600 Subject: [PATCH 153/204] Configure logging based on --verbose --- flake8/main/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 670e755..444c60d 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -172,6 +172,8 @@ def main(argv=None): listening_plugins.register_options(option_manager) formatting_plugins.register_options(option_manager) + preliminary_opts, _ = option_manager.parse_args(argv) + flake8.configure_logging(preliminary_opts.verbose) # Parse out our options from our found config files and user-provided CLI # options options, args = aggregator.aggregate_options(option_manager, argv) From aa6766861edac47562cd19c33fe12f5e298f0f0c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 18:59:48 -0600 Subject: [PATCH 154/204] Be consistent in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8c3ff2..1f4a6a0 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ except ImportError: tests_require = ['pytest'] if mock is None: - tests_require += ['mock'] + tests_require.append('mock') requires = [ From 0c968f7a158f60e891d0d4d9ec8b5165a8639eb8 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 19 Feb 2016 19:38:27 -0600 Subject: [PATCH 155/204] Add parity with ``python -m flake8`` --- flake8/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 flake8/__main__.py diff --git a/flake8/__main__.py b/flake8/__main__.py new file mode 100644 index 0000000..42bc428 --- /dev/null +++ b/flake8/__main__.py @@ -0,0 +1,4 @@ +"""Module allowing for ``python -m flake8 ...``.""" +from flake8.main import cli + +cli.main() From 1b91c8d66ab6f15c2b5946c5ceba9fe907a3976e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 21 Feb 2016 11:12:56 -0600 Subject: [PATCH 156/204] Start working on the code that runs checks --- flake8/checker.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++ flake8/utils.py | 26 ++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 flake8/checker.py diff --git a/flake8/checker.py b/flake8/checker.py new file mode 100644 index 0000000..2883555 --- /dev/null +++ b/flake8/checker.py @@ -0,0 +1,100 @@ +"""Checker Manager and Checker classes.""" +import logging + +LOG = logging.getLogger(__name__) + +try: + import multiprocessing +except ImportError: + multiprocessing = None + +from flake8 import utils + + +class Manager(object): + """Manage the parallelism and checker instances for each plugin and file. + + This class will be responsible for the following: + + - Determining the parallelism of Flake8, e.g.: + + * Do we use :mod:`multiprocessing` or is it unavailable? + + * Do we automatically decide on the number of jobs to use or did the + user provide that? + + - Falling back to a serial way of processing files if we run into an + OSError related to :mod:`multiprocessing` + + - Organizing the results of each checker so we can group the output + together and make our output deterministic. + """ + + def __init__(self, options, arguments, checker_plugins): + """Initialize our Manager instance. + + :param options: + The options parsed from config files and CLI. + :type options: + optparse.Values + :param list arguments: + The extra arguments parsed from the CLI (if any) + :param checker_plugins: + The plugins representing checks parsed from entry-points. + :type checker_plugins: + flake8.plugins.manager.Checkers + """ + self.arguments = arguments + self.options = options + self.checks = checker_plugins + self.jobs = self._job_count() + + def _job_count(self): + # First we walk through all of our error cases: + # - multiprocessing library is not present + # - we're running on windows in which case we know we have significant + # implemenation issues + # - the user provided stdin and that's not something we can handle + # well + # - we're processing a diff, which again does not work well with + # multiprocessing and which really shouldn't require multiprocessing + # - the user provided some awful input + if not multiprocessing: + LOG.warning('The multiprocessing module is not available. ' + 'Ignoring --jobs arguments.') + return None + + if utils.is_windows(): + LOG.warning('The --jobs option is not available on Windows. ' + 'Ignoring --jobs arguments.') + return None + + if utils.is_using_stdin(self.arguments): + LOG.warning('The --jobs option is not compatible with supplying ' + 'input using - . Ignoring --jobs arguments.') + return None + + if self.options.diff: + LOG.warning('The --diff option was specified with --jobs but ' + 'they are not compatible. Ignoring --jobs arguments.') + return None + + jobs = self.options.jobs + if jobs != 'auto' and not jobs.isdigit(): + LOG.warning('"%s" is not a valid parameter to --jobs. Must be one ' + 'of "auto" or a numerical value, e.g., 4.', jobs) + return None + + # If the value is "auto", we want to let the multiprocessing library + # decide the number based on the number of CPUs. However, if that + # function is not implemented for this particular value of Python we + # default to 1 + if jobs == 'auto': + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 1 + + # Otherwise, we know jobs should be an integer and we can just convert + # it to an integer + return int(jobs) diff --git a/flake8/utils.py b/flake8/utils.py index 1165470..e642d26 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -63,3 +63,29 @@ def stdin_get_value(): cached_type = io.StringIO stdin_get_value.cached_stdin = cached_type(stdin_value) return cached_value.getvalue() + + +def is_windows(): + # type: () -> bool + """Determine if we're running on Windows. + + :returns: + True if running on Windows, otherwise False + :rtype: + bool + """ + return os.name == 'nt' + + +def is_using_stdin(paths): + # type: (List[str]) -> bool + """Determine if we're going to read from stdin. + + :param list paths: + The paths that we're going to check. + :returns: + True if stdin (-) is in the path, otherwise False + :rtype: + bool + """ + return '-' in paths From 1312e4e0ef4a6bb372178a67a92dcc113356ea49 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 21 Feb 2016 13:09:44 -0600 Subject: [PATCH 157/204] Add mypy env There are still kinks to be worked out but this works reasonably well. This will fail until we can figure out how to import things from the typing module for the type: comments only. We do not want to add a dependency on that backport only for this mypy env. --- flake8/style_guide.py | 7 ++++--- tox.ini | 10 ++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 5deff7a..c3b3d5f 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -74,7 +74,7 @@ class StyleGuide(object): self._decision_cache = {} def is_user_selected(self, code): - # type: (Error) -> Union[Selected, Ignored] + # type: (str) -> Union[Selected, Ignored] """Determine if the code has been selected by the user. :param str code: @@ -95,7 +95,7 @@ class StyleGuide(object): return Ignored.Implicitly def is_user_ignored(self, code): - # type: (Error) -> Union[Selected, Ignored] + # type: (str) -> Union[Selected, Ignored] """Determine if the code has been ignored by the user. :param str code: @@ -123,7 +123,7 @@ class StyleGuide(object): return Decision.Ignored def should_report_error(self, code): - # type: (Error) -> Decision + # type: (str) -> Decision """Determine if the error code should be reported or ignored. This method only cares about the select and ignore rules as specified @@ -159,6 +159,7 @@ class StyleGuide(object): return decision def is_inline_ignored(self, error): + # type: (Error) -> bool """Determine if an comment has been added to ignore this line.""" # TODO(sigmavirus24): Determine how to handle stdin with linecache if self.options.disable_noqa: diff --git a/tox.ini b/tox.ini index 9b56c4c..353cb3b 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,16 @@ deps = commands = pylint flake8 +[testenv:mypy] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + mypy-lang +commands = + mypy flake8 + [testenv:docs] deps = sphinx>=1.3.0 From a21c328870cf177492927687445a55244bfc78ab Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 21 Feb 2016 13:45:56 -0600 Subject: [PATCH 158/204] Refactor our app setup/running a bit --- flake8/main/cli.py | 126 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 26 deletions(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 444c60d..72ca829 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -156,31 +156,105 @@ def register_default_options(option_manager): ) +class Application(object): + """Abstract our application into a class.""" + + def __init__(self, program='flake8', version=flake8.__version__): + # type: (str, str) -> NoneType + """Initialize our application. + + :param str program: + The name of the program/application that we're executing. + :param str version: + The version of the program/application we're executing. + """ + self.program = program + self.version = version + self.option_manager = manager.OptionManager( + prog='flake8', version=flake8.__version__ + ) + register_default_options(self.option_manager) + + # Set the verbosity of the program + preliminary_opts, _ = self.option_manager.parse_args() + flake8.configure_logging(preliminary_opts.verbose) + + self.check_plugins = None + self.listening_plugins = None + self.formatting_plugigns = None + self.formatter = None + self.listener_trie = None + self.guide = None + + self.options = None + self.args = None + + def find_plugins(self): + # type: () -> NoneType + """Find and load the plugins for this application.""" + if self.check_plugins is None: + self.check_plugins = plugin_manager.Checkers() + + if self.listening_plugins is None: + self.listening_plugins = plugin_manager.Listeners() + + if self.formatting_plugins is None: + self.formatting_plugins = plugin_manager.ReportFormatters() + + def register_plugin_options(self): + # type: () -> NoneType + """Register options provided by plugins to our option manager.""" + self.check_plugins.register_options(self.option_manager) + self.listening_plugins.register_options(self.option_manager) + self.formatting_plugins.register_options(self.option_manager) + + def parse_configuration_and_cli(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Parse configuration files and the CLI options. + + :param list argv: + Command-line arguments passed in directly. + """ + if self.options is None and self.args is None: + self.options, self.args = aggregator.aggregate_options( + self.option_manager, argv + ) + + def make_formatter(self): + # type: () -> NoneType + """Initialize a formatter based on the parsed options.""" + if self.formatter is None: + self.formatter = self.formatting_plugins.get( + self.options.format, self.formatting_plugins['default'] + ).execute(self.options) + + def make_notifier(self): + # type: () -> NoneType + """Initialize our listener Notifier.""" + if self.listener_trie is None: + self.listener_trie = self.listening_plugins.build_notifier() + + def make_guide(self): + # type: () -> NoneType + """Initialize our StyleGuide.""" + if self.guide is None: + self.guide = style_guide.StyleGuide( + self.options, self.listener_trie, self.formatter + ) + + def run(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Run our application.""" + self.find_plugins() + self.register_plugin_options() + self.parse_configuration_and_cli(argv) + self.make_formatter() + self.make_notifier() + self.make_guide() + + def main(argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType """Main entry-point for the flake8 command-line tool.""" - option_manager = manager.OptionManager( - prog='flake8', version=flake8.__version__ - ) - # Load our plugins - check_plugins = plugin_manager.Checkers() - listening_plugins = plugin_manager.Listeners() - formatting_plugins = plugin_manager.ReportFormatters() - - # Register all command-line and config-file options - register_default_options(option_manager) - check_plugins.register_options(option_manager) - listening_plugins.register_options(option_manager) - formatting_plugins.register_options(option_manager) - - preliminary_opts, _ = option_manager.parse_args(argv) - flake8.configure_logging(preliminary_opts.verbose) - # Parse out our options from our found config files and user-provided CLI - # options - options, args = aggregator.aggregate_options(option_manager, argv) - - formatter = formatting_plugins.get( - options.format, formatting_plugins['default'] - ).execute(options) - listener_trie = listening_plugins.build_notifier() - guide = style_guide.StyleGuide(options, listener_trie, formatter) - guide.handle_error('E111', 'stdin', 1, 1, 'faketext') + app = Application() + app.run(argv) From 8792c30872e7e49c2cf2076c5765238b2e92c729 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 22 Feb 2016 21:47:43 -0600 Subject: [PATCH 159/204] Add utility functions around filename matching We add utils.fnmatch and utils.filenames_for in anticipation of our checker manager creating file checkers for each file. We also include tests for these functions and a couple previously untested utility functions. --- flake8/utils.py | 55 ++++++++++++++++++++++++++++++++++++++++ tests/unit/test_utils.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/flake8/utils.py b/flake8/utils.py index e642d26..54c1990 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -1,4 +1,5 @@ """Utility methods for flake8.""" +import fnmatch as _fnmatch import io import os import sys @@ -89,3 +90,57 @@ def is_using_stdin(paths): bool """ return '-' in paths + + +def _default_predicate(*args): + return False + + +def filenames_from(arg, predicate=None): + # type: (str) -> Generator + """Generate filenames from an argument. + + :param str arg: + Parameter from the command-line. + :param callable predicate: + Predicate to use to filter out filenames. If the predicate + returns ``True`` we will exclude the filename, otherwise we + will yield it. + :returns: + Generator of paths + """ + if predicate is None: + predicate = _default_predicate + if os.path.isdir(arg): + for root, sub_directories, files in os.walk(arg): + for filename in files: + joined = os.path.join(root, filename) + if predicate(filename) or predicate(joined): + continue + yield joined + # NOTE(sigmavirus24): os.walk() will skip a directory if you + # remove it from the list of sub-directories. + for directory in sub_directories: + if predicate(directory): + sub_directories.remove(directory) + else: + yield arg + + +def fnmatch(filename, patterns, default=True): + # type: (str, List[str], bool) -> bool + """Wrap :func:`fnmatch.fnmatch` to add some functionality. + + :param str filename: + Name of the file we're trying to match. + :param list patterns: + Patterns we're using to try to match the filename. + :param bool default: + The default value if patterns is empty + :returns: + True if a pattern matches the filename, False if it doesn't. + ``default`` if patterns is empty. + """ + if not patterns: + return default + return any(_fnmatch.fnmatch(filename, pattern) for pattern in patterns) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a6096d4..34b4909 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,5 +1,6 @@ """Tests for flake8's utils module.""" import os +import mock import pytest @@ -40,3 +41,48 @@ def test_normalize_path(value, expected): def test_normalize_paths(value, expected): """Verify we normalize comma-separated paths provided to the tool.""" assert utils.normalize_paths(value) == expected + + +def test_is_windows_checks_for_nt(): + """Verify that we correctly detect Windows.""" + with mock.patch.object(os, 'name', 'nt'): + assert utils.is_windows() is True + + with mock.patch.object(os, 'name', 'posix'): + assert utils.is_windows() is False + + +@pytest.mark.parametrize('filename,patterns,expected', [ + ('foo.py', [], True), + ('foo.py', ['*.pyc'], False), + ('foo.pyc', ['*.pyc'], True), + ('foo.pyc', ['*.swp', '*.pyc', '*.py'], True), +]) +def test_fnmatch(filename, patterns, expected): + """Verify that our fnmatch wrapper works as expected.""" + assert utils.fnmatch(filename, patterns) is expected + + +def test_filenames_from_a_directory(): + """Verify that filenames_from walks a directory.""" + filenames = list(utils.filenames_from('flake8/')) + assert len(filenames) > 2 + assert 'flake8/__init__.py' in filenames + + +def test_filenames_from_a_directory_with_a_predicate(): + """Verify that predicates filter filenames_from.""" + filenames = list(utils.filenames_from( + arg='flake8/', + predicate=lambda filename: filename == 'flake8/__init__.py', + )) + assert len(filenames) > 2 + assert 'flake8/__init__.py' not in filenames + + +def test_filenames_from_a_single_file(): + """Verify that we simply yield that filename.""" + filenames = list(utils.filenames_from('flake8/__init__.py')) + + assert len(filenames) == 1 + assert ['flake8/__init__.py'] == filenames From d5a480a46401949c42fe42fa2eb50a646e0ec3f6 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 22 Feb 2016 21:51:50 -0600 Subject: [PATCH 160/204] Add method to determine if filename is excluded Add a method to make FileCheckers from the list of arguments the user passes. Start to work on FileCheckers --- flake8/checker.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/flake8/checker.py b/flake8/checker.py index 2883555..0a3a86f 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -1,5 +1,6 @@ """Checker Manager and Checker classes.""" import logging +import os LOG = logging.getLogger(__name__) @@ -48,6 +49,14 @@ class Manager(object): self.options = options self.checks = checker_plugins self.jobs = self._job_count() + self.process_queue = None + self.using_multiprocessing = False + self.processes = [] + self.checkers = [] + + if self.jobs is not None and self.jobs > 1: + self.using_multiprocessing = True + self.process_queue = multiprocessing.Queue() def _job_count(self): # First we walk through all of our error cases: @@ -93,8 +102,76 @@ class Manager(object): try: return multiprocessing.cpu_count() except NotImplementedError: - return 1 + return 0 # Otherwise, we know jobs should be an integer and we can just convert # it to an integer return int(jobs) + + def start(self): + """Start checking files.""" + pass + # for i in range(self.jobs or 0): + # proc = multiprocessing.Process(target=self.process_files) + # proc.daemon = True + # proc.start() + # self.processes.append(proc) + + def make_checkers(self, paths=None): + # type: (List[str]) -> NoneType + """Create checkers for each file.""" + if paths is None: + paths = self.arguments + filename_patterns = self.options.filename + self.checkers = [ + FileChecker(filename, self.checks) + for argument in paths + for filename in utils.filenames_from(argument, + self.is_path_excluded) + if utils.fnmatch(filename, filename_patterns) + ] + + def is_path_excluded(self, path): + # type: (str) -> bool + """Check if a path is excluded. + + :param str path: + Path to check against the exclude patterns. + :returns: + True if there are exclude patterns and the path matches, + otherwise False. + :rtype: + bool + """ + exclude = self.options.exclude + if not exclude: + return False + basename = os.path.basename(path) + if utils.fnmatch(basename, exclude): + LOG.info('"%s" has been excluded', basename) + return True + + absolute_path = os.path.abspath(path) + match = utils.fnmatch(absolute_path, exclude) + LOG.info('"%s" has %sbeen excluded', absolute_path, + '' if match else 'not ') + return match + + +class FileChecker(object): + """Manage running checks for a file and aggregate the results.""" + + def __init__(self, filename, checks): + # type: (str, flake8.plugins.manager.Checkers) -> NoneType + """Initialize our file checker. + + :param str filename: + Name of the file to check. + :param checks: + The plugins registered to check the file. + :type checks: + flake8.plugins.manager.Checkers + """ + self.filename = filename + self.checks = checks + self.results = [] From 7addb72615a53aaf05b4e7e121bc0fbdfdd7d789 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 22 Feb 2016 21:59:59 -0600 Subject: [PATCH 161/204] Add unified linters testenv --- flake8/checker.py | 4 ++-- tox.ini | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 0a3a86f..fc2da6b 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -2,8 +2,6 @@ import logging import os -LOG = logging.getLogger(__name__) - try: import multiprocessing except ImportError: @@ -11,6 +9,8 @@ except ImportError: from flake8 import utils +LOG = logging.getLogger(__name__) + class Manager(object): """Manage the parallelism and checker instances for each plugin and file. diff --git a/tox.ini b/tox.ini index 353cb3b..00d6469 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = . commands = {posargs} +# Linters [testenv:flake8] skipsdist = true skip_install = true @@ -30,7 +31,7 @@ skipsdist = true skip_install = true use_develop = false deps = - . + pyflakes pylint commands = pylint flake8 @@ -45,6 +46,19 @@ deps = commands = mypy flake8 +[testenv:linters] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + {[testenv:flake8]deps} + {[testenv:pylint]deps} +commands = + {[testenv:flake8]commands} + {[testenv:pylint]commands} + +# Documentation [testenv:docs] deps = sphinx>=1.3.0 @@ -67,6 +81,7 @@ deps = commands = python setup.py check -r -s +# Flake8 Configuration [flake8] # Ignore some flake8-docstrings errors ignore = D203 From 28f4811cb99418c43dd90a9dc31d0ec1ba06c8b7 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 22 Feb 2016 22:17:37 -0600 Subject: [PATCH 162/204] Read lines from the file in our checker --- flake8/checker.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/flake8/checker.py b/flake8/checker.py index fc2da6b..f400f1a 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -1,6 +1,9 @@ """Checker Manager and Checker classes.""" +import io import logging import os +import sys +import tokenize try: import multiprocessing @@ -175,3 +178,36 @@ class FileChecker(object): self.filename = filename self.checks = checks self.results = [] + self.lines = [] + + def read_lines(self): + """Read the lines for this file checker.""" + if self.filename is None or self.filename == '-': + self.filename = 'stdin' + return self.read_lines_from_stdin() + return self.read_lines_from_filename() + + def read_lines_from_stdin(self): + """Read the lines from standard in.""" + return utils.stdin_get_value().splitlines(True) + + def read_lines_from_filename(self): + """Read the lines for a file.""" + if (2, 6) <= sys.version_info < (3, 0): + with open(self.filename, 'rU') as fd: + return fd.readlines() + + elif (3, 0) <= sys.version_info < (4, 0): + try: + with open(self.filename, 'rb') as fd: + (coding, lines) = tokenize.detect_encoding(fd.readline) + textfd = io.TextIOWrapper(fd, coding, line_buffering=True) + return ([l.decode(coding) for l in lines] + + textfd.readlines()) + except (LookupError, SyntaxError, UnicodeError): + with open(self.filename, encoding='latin-1') as fd: + return fd.readlines() + + def run_checks(self): + """Run checks against the file.""" + self.lines = self.read_lines() From 5ee061b810caf03025927f2e8fc7fb15fdcabd58 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 11:17:11 -0600 Subject: [PATCH 163/204] Add line splitting and file reading Add some tests around reading lines and striping UTF BOMs --- flake8/checker.py | 68 +++++++++++++++++++++++++-------- tests/unit/test_file_checker.py | 26 +++++++++++++ 2 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_file_checker.py diff --git a/flake8/checker.py b/flake8/checker.py index f400f1a..33e58d4 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -187,27 +187,65 @@ class FileChecker(object): return self.read_lines_from_stdin() return self.read_lines_from_filename() - def read_lines_from_stdin(self): - """Read the lines from standard in.""" - return utils.stdin_get_value().splitlines(True) + def _readlines_py2(self): + with open(self.filename, 'rU') as fd: + return fd.readlines() + + def _readlines_py3(self): + try: + with open(self.filename, 'rb') as fd: + (coding, lines) = tokenize.detect_encoding(fd.readline) + textfd = io.TextIOWrapper(fd, coding, line_buffering=True) + return ([l.decode(coding) for l in lines] + + textfd.readlines()) + except (LookupError, SyntaxError, UnicodeError): + # If we can't detect the codec with tokenize.detect_encoding, or + # the detected encoding is incorrect, just fallback to latin-1. + with open(self.filename, encoding='latin-1') as fd: + return fd.readlines() def read_lines_from_filename(self): """Read the lines for a file.""" if (2, 6) <= sys.version_info < (3, 0): - with open(self.filename, 'rU') as fd: - return fd.readlines() - + readlines = self._readlines_py2 elif (3, 0) <= sys.version_info < (4, 0): - try: - with open(self.filename, 'rb') as fd: - (coding, lines) = tokenize.detect_encoding(fd.readline) - textfd = io.TextIOWrapper(fd, coding, line_buffering=True) - return ([l.decode(coding) for l in lines] + - textfd.readlines()) - except (LookupError, SyntaxError, UnicodeError): - with open(self.filename, encoding='latin-1') as fd: - return fd.readlines() + readlines = self._readlines_py3 + + try: + return readlines() + except IOError: + # If we can not read the file due to an IOError (e.g., the file + # does not exist or we do not have the permissions to open it) + # then we need to format that exception for the user. + # NOTE(sigmavirus24): Historically, pep8 has always reported this + # as an E902. We probably *want* a better error code for this + # going forward. + (exc_type, exception) = sys.exc_info()[:2] + message = '{0}: {1}'.format(exc_type.__name__, exception) + self.results.append('E902', self.filename, 0, 0, message) + return [] + + def read_lines_from_stdin(self): + """Read the lines from standard in.""" + return utils.stdin_get_value().splitlines(True) def run_checks(self): """Run checks against the file.""" self.lines = self.read_lines() + self.strip_utf_bom() + + def strip_utf_bom(self): + """Strip the UTF bom from the lines of the file.""" + if not self.lines: + # If we have nothing to analyze quit early + return + + first_byte = ord(self.lines[0][0]) + if first_byte not in (0xEF, 0xFEFF): + return + + # If the first byte of the file is a UTF-8 BOM, strip it + if first_byte == 0xFEFF: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xEF\xBB\xBF': + self.lines[0] = self.lines[0][3:] diff --git a/tests/unit/test_file_checker.py b/tests/unit/test_file_checker.py new file mode 100644 index 0000000..e26bc83 --- /dev/null +++ b/tests/unit/test_file_checker.py @@ -0,0 +1,26 @@ +"""Tests for the FileChecker class.""" +from flake8 import checker + +import pytest + + +def test_read_lines_splits_lines(): + """Verify that read_lines splits the lines of the file.""" + file_checker = checker.FileChecker(__file__, []) + lines = file_checker.read_lines() + assert len(lines) > 5 + assert '"""Tests for the FileChecker class."""\n' in lines + + +@pytest.mark.parametrize('first_line', [ + '\xEF\xBB\xBF"""Module docstring."""\n', + '\uFEFF"""Module docstring."""\n', +]) +def test_strip_utf_bom(first_line): + r"""Verify that we strip '\xEF\xBB\xBF' from the first line.""" + lines = [first_line] + file_checker = checker.FileChecker('stdin', []) + file_checker.lines = lines[:] + file_checker.strip_utf_bom() + assert file_checker.lines != lines + assert file_checker.lines[0] == '"""Module docstring."""\n' From 54ad972e5601436e6a89b68b1d23376edbc3a3ff Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 14:42:09 -0600 Subject: [PATCH 164/204] Add doc8 linting --- docs/source/dev/registering_plugins.rst | 2 +- tox.ini | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/source/dev/registering_plugins.rst b/docs/source/dev/registering_plugins.rst index 48e39fc..0cc18d9 100644 --- a/docs/source/dev/registering_plugins.rst +++ b/docs/source/dev/registering_plugins.rst @@ -79,7 +79,7 @@ Note specifically these lines: # snip ... ) -We tell setuptools to register our entry point "X" inside the specific +We tell setuptools to register our entry point "X" inside the specific grouping of entry-points that flake8 should look in. Flake8 presently looks at three groups: diff --git a/tox.ini b/tox.ini index 00d6469..ba6c1af 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,17 @@ deps = commands = pylint flake8 +[testenv:doc8] +basepython = python3 +skipsdist = true +skip_install = true +use_develop = false +deps = + sphinx + doc8 +commands = + doc8 docs/source/ + [testenv:mypy] basepython = python3 skipsdist = true From 08fd403e345f383124c00e9c5352b207284567a3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 14:42:50 -0600 Subject: [PATCH 165/204] Add doc8 to linters --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index ba6c1af..e4424a3 100644 --- a/tox.ini +++ b/tox.ini @@ -65,9 +65,11 @@ use_develop = false deps = {[testenv:flake8]deps} {[testenv:pylint]deps} + {[testenv:doc8]deps} commands = {[testenv:flake8]commands} {[testenv:pylint]commands} + {[testenv:doc8]commands} # Documentation [testenv:docs] From 51d15295df1d81a0fa38068a46236e99d9fdb2c5 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 17:08:54 -0600 Subject: [PATCH 166/204] Set the maximum complexity for mccabe --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e4424a3..9ea64cf 100644 --- a/tox.ini +++ b/tox.ini @@ -102,3 +102,4 @@ ignore = D203 # across multiple lines. Presently it cannot be specified across multiple lines. # :-( exclude = .git,__pycache__,docs/source/conf.py +max-complexity = 10 From 1cd5fea73066f96753bbc69cbbfa7f96e8d50e4a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 21:09:19 -0600 Subject: [PATCH 167/204] Add type annotations --- flake8/checker.py | 15 ++++++++++++++- flake8/utils.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 33e58d4..07c90f4 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -62,6 +62,7 @@ class Manager(object): self.process_queue = multiprocessing.Queue() def _job_count(self): + # type: () -> Union[int, NoneType] # First we walk through all of our error cases: # - multiprocessing library is not present # - we're running on windows in which case we know we have significant @@ -181,6 +182,7 @@ class FileChecker(object): self.lines = [] def read_lines(self): + # type: () -> List[str] """Read the lines for this file checker.""" if self.filename is None or self.filename == '-': self.filename = 'stdin' @@ -188,10 +190,12 @@ class FileChecker(object): return self.read_lines_from_filename() def _readlines_py2(self): + # type: () -> List[str] with open(self.filename, 'rU') as fd: return fd.readlines() def _readlines_py3(self): + # type: () -> List[str] try: with open(self.filename, 'rb') as fd: (coding, lines) = tokenize.detect_encoding(fd.readline) @@ -205,6 +209,7 @@ class FileChecker(object): return fd.readlines() def read_lines_from_filename(self): + # type: () -> List[str] """Read the lines for a file.""" if (2, 6) <= sys.version_info < (3, 0): readlines = self._readlines_py2 @@ -222,19 +227,27 @@ class FileChecker(object): # going forward. (exc_type, exception) = sys.exc_info()[:2] message = '{0}: {1}'.format(exc_type.__name__, exception) - self.results.append('E902', self.filename, 0, 0, message) + self.report('E902', 0, 0, message) return [] def read_lines_from_stdin(self): + # type: () -> List[str] """Read the lines from standard in.""" return utils.stdin_get_value().splitlines(True) + def report(self, error_code, line_number, column, text): + # type: (str, int, int, str) -> NoneType + """Report an error by storing it in the results list.""" + error = (error_code, self.filename, line_number, column, text) + self.results.append(error) + def run_checks(self): """Run checks against the file.""" self.lines = self.read_lines() self.strip_utf_bom() def strip_utf_bom(self): + # type: () -> NoneType """Strip the UTF bom from the lines of the file.""" if not self.lines: # If we have nothing to analyze quit early diff --git a/flake8/utils.py b/flake8/utils.py index 54c1990..fbc90bd 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -97,7 +97,7 @@ def _default_predicate(*args): def filenames_from(arg, predicate=None): - # type: (str) -> Generator + # type: (str, callable) -> Generator """Generate filenames from an argument. :param str arg: From 24d2689a0519cb453eb67d6a0ef37f60efc8f0fd Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 23:20:34 -0600 Subject: [PATCH 168/204] Distinguish check types via plugin type manager Flake8 and pep8 has historically supported three types of checks: - Plugins that accept the physical line - Plugins that accept the logical line - Plugins that accept the AST tree The logical place to make this distinction is on the Checkers plugin type manager class. This adds the foundation for finding plugins that fall into each class. --- flake8/plugins/manager.py | 20 ++++++++++++++++++++ flake8/utils.py | 38 ++++++++++++++++++++++++++++++++++++++ tests/unit/test_utils.py | 22 ++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 21171ef..21639f9 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -6,6 +6,7 @@ import pkg_resources from flake8 import exceptions from flake8.plugins import notifier +from flake8 import utils LOG = logging.getLogger(__name__) @@ -296,6 +297,25 @@ class Checkers(PluginTypeManager): namespace = 'flake8.extension' + def checks_expecting(self, argument_name): + """Retrieve checks that expect an argument with the specified name. + + Find all checker plugins that are expecting a specific argument. + """ + for plugin in self.plugins.values(): + parameters = utils.parameters_for(plugin) + if argument_name in parameters: + yield plugin + + @property + def physical_line_plugins(self): + """List of plugins that expect the physical lines.""" + plugins = getattr(self, '_physical_line_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('physical_line')) + self._physical_line_plugins = plugins + return plugins + class Listeners(PluginTypeManager, NotifierBuilderMixin): """All of the listeners registered through entry-points.""" diff --git a/flake8/utils.py b/flake8/utils.py index fbc90bd..cd1cf72 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -1,5 +1,6 @@ """Utility methods for flake8.""" import fnmatch as _fnmatch +import inspect import io import os import sys @@ -144,3 +145,40 @@ def fnmatch(filename, patterns, default=True): if not patterns: return default return any(_fnmatch.fnmatch(filename, pattern) for pattern in patterns) + + +def parameters_for(plugin): + # type: (flake8.plugins.manager.Plugin) -> List[str] + """Return the parameters for the plugin. + + This will inspect the plugin and return either the function parameters + if the plugin is a function or the parameters for ``__init__`` after + ``self`` if the plugin is a class. + + :param plugin: + The internal plugin object. + :type plugin: + flake8.plugins.manager.Plugin + :returns: + Parameters to the plugin. + :rtype: + list(str) + """ + func = plugin.plugin + is_class = not inspect.isfunction(func) + if is_class: # The plugin is a class + func = plugin.plugin.__init__ + + if sys.version_info < (3, 3): + parameters = inspect.getargspec(func)[0] + else: + parameters = [ + parameter.name + for parameter in inspect.signature(func).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD + ] + + if is_class: + parameters.remove('self') + + return parameters diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 34b4909..615f327 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ import mock import pytest +from flake8.plugins import manager as plugin_manager from flake8 import utils @@ -86,3 +87,24 @@ def test_filenames_from_a_single_file(): assert len(filenames) == 1 assert ['flake8/__init__.py'] == filenames + + +def test_parameters_for_class_plugin(): + """Verify that we can retrieve the parameters for a class plugin.""" + class FakeCheck(object): + def __init__(self, tree): + pass + + plugin = plugin_manager.Plugin('plugin-name', object()) + plugin._plugin = FakeCheck + assert utils.parameters_for(plugin) == ['tree'] + + +def test_parameters_for_function_plugin(): + """Verify that we retrieve the parameters for a function plugin.""" + def fake_plugin(physical_line, self, tree): + pass + + plugin = plugin_manager.Plugin('plugin-name', object()) + plugin._plugin = fake_plugin + assert utils.parameters_for(plugin) == ['physical_line', 'self', 'tree'] From cd18b9f175a3a73b03f58d4db7fd789c48c671bb Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 23 Feb 2016 23:25:12 -0600 Subject: [PATCH 169/204] Constrain our search for plugin type Flake8 has previously only ever relied on the first member of the parameters list to determine what kind of check it is using. As such, we constrain ourselves to just that parameter when checking and add properties for ast and logical line checks. --- flake8/plugins/manager.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 21639f9..05e5e2b 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -304,9 +304,27 @@ class Checkers(PluginTypeManager): """ for plugin in self.plugins.values(): parameters = utils.parameters_for(plugin) - if argument_name in parameters: + if argument_name == parameters[0]: yield plugin + @property + def ast_plugins(self): + """List of plugins that expect the AST tree.""" + plugins = getattr(self, '_ast_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('tree')) + self._ast_plugins = plugins + return plugins + + @property + def logical_line_plugins(self): + """List of plugins that expect the logical lines.""" + plugins = getattr(self, '_logical_line_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('logical_line')) + self._logical_line_plugins = plugins + return plugins + @property def physical_line_plugins(self): """List of plugins that expect the physical lines.""" From a4e984dbd258faf125a115bae1561db3fbd8e934 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 25 Feb 2016 09:06:45 -0600 Subject: [PATCH 170/204] Add and fix documentation - Add more documentation around utils functions - Fix documentation about default formatting plugins - Add extra documentation of filenames_from predicate parameter - Add test for the default parameter of flake8.utils.fnmatch --- docs/source/internal/formatters.rst | 8 ++-- docs/source/internal/plugin_handling.rst | 2 + docs/source/internal/utils.rst | 53 ++++++++++++++++++++++++ flake8/utils.py | 3 +- tests/unit/test_utils.py | 6 +++ 5 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/source/internal/formatters.rst b/docs/source/internal/formatters.rst index caa5718..d54cf87 100644 --- a/docs/source/internal/formatters.rst +++ b/docs/source/internal/formatters.rst @@ -30,7 +30,7 @@ The former allows us to inspect the value provided to ``--format`` by the user and alter our own format based on that value. The second simply uses that format string to format the error. -.. autoclass:: flake8.formatters.default.Default +.. autoclass:: flake8.formatting.default.Default :members: Pylint Formatter @@ -39,9 +39,9 @@ Pylint Formatter The |PylintFormatter| simply defines the default Pylint format string from pep8: ``'%(path)s:%(row)d: [%(code)s] %(text)s'``. -.. autoclass:: flake8.formatters.default.Pylint +.. autoclass:: flake8.formatting.default.Pylint :members: -.. |DefaultFormatter| replace:: :class:`~flake8.formatters.default.Default` -.. |PylintFormatter| replace:: :class:`~flake8.formatters.default.Pylint` +.. |DefaultFormatter| replace:: :class:`~flake8.formatting.default.Default` +.. |PylintFormatter| replace:: :class:`~flake8.formatting.default.Pylint` diff --git a/docs/source/internal/plugin_handling.rst b/docs/source/internal/plugin_handling.rst index e430bfb..b3b9b0e 100644 --- a/docs/source/internal/plugin_handling.rst +++ b/docs/source/internal/plugin_handling.rst @@ -98,8 +98,10 @@ API Documentation :members: .. autoclass:: flake8.plugins.manager.Checkers + :members: .. autoclass:: flake8.plugins.manager.Listeners + :members: build_notifier .. autoclass:: flake8.plugins.manager.ReportFormatters diff --git a/docs/source/internal/utils.rst b/docs/source/internal/utils.rst index 69dee45..d8adeac 100644 --- a/docs/source/internal/utils.rst +++ b/docs/source/internal/utils.rst @@ -45,3 +45,56 @@ strings that should be paths. This function retrieves and caches the value provided on ``sys.stdin``. This allows plugins to use this to retrieve ``stdin`` if necessary. + +.. autofunction:: flake8.utils.is_windows + +This provides a convenient and explicitly named function that checks if we are +currently running on a Windows (or ``nt``) operating system. + +.. autofunction:: flake8.utils.is_using_stdin + +Another helpful function that is named only to be explicit given it is a very +trivial check, this checks if the user specified ``-`` in their arguments to +Flake8 to indicate we should read from stdin. + +.. autofunction:: flake8.utils.filenames_from + +When provided an argument to Flake8, we need to be able to traverse +directories in a convenient manner. For example, if someone runs + +.. code:: + + $ flake8 flake8/ + +Then they want us to check all of the files in the directory ``flake8/``. This +function will handle that while also handling the case where they specify a +file like: + +.. code:: + + $ flake8 flake8/__init__.py + + +.. autofunction:: flake8.utils.fnmatch + +The standard library's :func:`fnmatch.fnmatch` is excellent at deciding if a +filename matches a single pattern. In our use case, however, we typically have +a list of patterns and want to know if the filename matches any of them. This +function abstracts that logic away with a little extra logic. + +.. autofunction:: flake8.utils.parameters_for + +Flake8 analyzes the parameters to plugins to determine what input they are +expecting. Plugins may expect one of the following: + +- ``physical_line`` to receive the line as it appears in the file + +- ``logical_line`` to receive the logical line (not as it appears in the file) + +- ``tree`` to receive the abstract syntax tree (AST) for the file + +We also analyze the rest of the parameters to provide more detail to the +plugin. This function will return the parameters in a consistent way across +versions of Python and will handle both classes and functions that are used as +plugins. Further, if the plugin is a class, it will strip the ``self`` +argument so we can check the parameters of the plugin consistently. diff --git a/flake8/utils.py b/flake8/utils.py index cd1cf72..7e08e41 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -106,7 +106,8 @@ def filenames_from(arg, predicate=None): :param callable predicate: Predicate to use to filter out filenames. If the predicate returns ``True`` we will exclude the filename, otherwise we - will yield it. + will yield it. By default, we include every filename + generated. :returns: Generator of paths """ diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 615f327..28b8e02 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -64,6 +64,12 @@ def test_fnmatch(filename, patterns, expected): assert utils.fnmatch(filename, patterns) is expected +def test_fnmatch_returns_the_default_with_empty_default(): + """The default parameter should be returned when no patterns are given.""" + sentinel = object() + assert utils.fnmatch('file.py', [], default=sentinel) is sentinel + + def test_filenames_from_a_directory(): """Verify that filenames_from walks a directory.""" filenames = list(utils.filenames_from('flake8/')) From 8d36355611e1e5a7c1a81bbbf8560c01d9ead979 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 25 Feb 2016 11:14:41 -0600 Subject: [PATCH 171/204] Re-order project imports based on flake8-import-order --- flake8/main/cli.py | 2 +- flake8/options/aggregator.py | 2 +- flake8/plugins/manager.py | 6 +++--- flake8/plugins/pyflakes.py | 5 +++-- flake8/style_guide.py | 3 +-- setup.py | 6 ++++-- tox.ini | 1 + 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 72ca829..15eea8e 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -1,10 +1,10 @@ """Command-line implementation of flake8.""" import flake8 from flake8 import defaults +from flake8 import style_guide from flake8.options import aggregator from flake8.options import manager from flake8.plugins import manager as plugin_manager -from flake8 import style_guide def register_default_options(option_manager): diff --git a/flake8/options/aggregator.py b/flake8/options/aggregator.py index b344cf0..99d0cfe 100644 --- a/flake8/options/aggregator.py +++ b/flake8/options/aggregator.py @@ -5,8 +5,8 @@ applies the user-specified command-line configuration on top of it. """ import logging -from flake8.options import config from flake8 import utils +from flake8.options import config LOG = logging.getLogger(__name__) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index 05e5e2b..aba5f16 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -2,11 +2,11 @@ import collections import logging -import pkg_resources - from flake8 import exceptions -from flake8.plugins import notifier from flake8 import utils +from flake8.plugins import notifier + +import pkg_resources LOG = logging.getLogger(__name__) diff --git a/flake8/plugins/pyflakes.py b/flake8/plugins/pyflakes.py index f512511..7025baa 100644 --- a/flake8/plugins/pyflakes.py +++ b/flake8/plugins/pyflakes.py @@ -1,6 +1,7 @@ """Plugin built-in to Flake8 to treat pyflakes as a plugin.""" # -*- coding: utf-8 -*- from __future__ import absolute_import + try: # The 'demandimport' breaks pyflakes and flake8.plugins.pyflakes from mercurial import demandimport @@ -10,11 +11,11 @@ else: demandimport.disable() import os +from flake8 import utils + import pyflakes import pyflakes.checker -from flake8 import utils - def patch_pyflakes(): """Add error codes to Pyflakes messages.""" diff --git a/flake8/style_guide.py b/flake8/style_guide.py index c3b3d5f..bbfe658 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -1,11 +1,10 @@ """Implementation of the StyleGuide used by Flake8.""" import collections +import enum import linecache import logging import re -import enum - from flake8 import utils __all__ = ( diff --git a/setup.py b/setup.py index 1f4a6a0..ec40eb6 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,12 @@ """Packaging logic for Flake8.""" # -*- coding: utf-8 -*- from __future__ import with_statement -import setuptools + import sys -import flake8 +import setuptools + +import flake8 # noqa try: # Work around a traceback with Nose on Python 2.6 diff --git a/tox.ini b/tox.ini index 9ea64cf..d5e07cc 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ use_develop = false deps = flake8 flake8-docstrings + flake8-import-order commands = flake8 From de9f56addfc770ba511550546a7f607c26472e5a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 25 Feb 2016 14:41:37 -0600 Subject: [PATCH 172/204] Fix import ordering in test files --- tests/integration/test_aggregator.py | 4 ++-- tests/unit/test_config_file_finder.py | 7 ++++--- tests/unit/test_merged_config_parser.py | 7 ++++--- tests/unit/test_notifier.py | 4 ++-- tests/unit/test_option.py | 7 ++++--- tests/unit/test_option_manager.py | 4 ++-- tests/unit/test_plugin.py | 7 ++++--- tests/unit/test_plugin_manager.py | 4 ++-- tests/unit/test_plugin_type_manager.py | 7 ++++--- tests/unit/test_style_guide.py | 3 ++- tests/unit/test_utils.py | 7 ++++--- 11 files changed, 34 insertions(+), 27 deletions(-) diff --git a/tests/integration/test_aggregator.py b/tests/integration/test_aggregator.py index 3f64fa3..2186c35 100644 --- a/tests/integration/test_aggregator.py +++ b/tests/integration/test_aggregator.py @@ -1,12 +1,12 @@ """Test aggregation of config files and command-line options.""" import os -import pytest - from flake8.main import cli from flake8.options import aggregator from flake8.options import manager +import pytest + CLI_SPECIFIED_CONFIG = 'tests/fixtures/config_files/cli-specified.ini' diff --git a/tests/unit/test_config_file_finder.py b/tests/unit/test_config_file_finder.py index 6ac1d16..58009f7 100644 --- a/tests/unit/test_config_file_finder.py +++ b/tests/unit/test_config_file_finder.py @@ -7,11 +7,12 @@ except ImportError: import os import sys -import mock -import pytest - from flake8.options import config +import mock + +import pytest + CLI_SPECIFIED_FILEPATH = 'tests/fixtures/config_files/cli-specified.ini' diff --git a/tests/unit/test_merged_config_parser.py b/tests/unit/test_merged_config_parser.py index 2c4c5fd..c64cae6 100644 --- a/tests/unit/test_merged_config_parser.py +++ b/tests/unit/test_merged_config_parser.py @@ -1,12 +1,13 @@ """Unit tests for flake8.options.config.MergedConfigParser.""" import os -import mock -import pytest - from flake8.options import config from flake8.options import manager +import mock + +import pytest + @pytest.fixture def optmanager(): diff --git a/tests/unit/test_notifier.py b/tests/unit/test_notifier.py index 6a162cf..8c001da 100644 --- a/tests/unit/test_notifier.py +++ b/tests/unit/test_notifier.py @@ -1,8 +1,8 @@ """Unit tests for the Notifier object.""" -import pytest - from flake8.plugins import notifier +import pytest + class _Listener(object): def __init__(self, error_code): diff --git a/tests/unit/test_option.py b/tests/unit/test_option.py index 67e2255..45f9be7 100644 --- a/tests/unit/test_option.py +++ b/tests/unit/test_option.py @@ -1,9 +1,10 @@ """Unit tests for flake8.options.manager.Option.""" -import mock -import pytest - from flake8.options import manager +import mock + +import pytest + def test_to_optparse(): """Test conversion to an optparse.Option class.""" diff --git a/tests/unit/test_option_manager.py b/tests/unit/test_option_manager.py index 99c5d18..1ba5442 100644 --- a/tests/unit/test_option_manager.py +++ b/tests/unit/test_option_manager.py @@ -2,10 +2,10 @@ import optparse import os -import pytest - from flake8.options import manager +import pytest + TEST_VERSION = '3.0.0b1' diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 01454a1..0f6eec1 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -1,10 +1,11 @@ """Tests for flake8.plugins.manager.Plugin.""" -import mock -import pytest - from flake8 import exceptions from flake8.plugins import manager +import mock + +import pytest + def test_load_plugin_fallsback_on_old_setuptools(): """Verify we fallback gracefully to on old versions of setuptools.""" diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py index 8991b96..5a50386 100644 --- a/tests/unit/test_plugin_manager.py +++ b/tests/unit/test_plugin_manager.py @@ -1,8 +1,8 @@ """Tests for flake8.plugins.manager.PluginManager.""" -import mock - from flake8.plugins import manager +import mock + def create_entry_point_mock(name): """Create a mocked EntryPoint.""" diff --git a/tests/unit/test_plugin_type_manager.py b/tests/unit/test_plugin_type_manager.py index f64e472..271ebc0 100644 --- a/tests/unit/test_plugin_type_manager.py +++ b/tests/unit/test_plugin_type_manager.py @@ -1,12 +1,13 @@ """Tests for flake8.plugins.manager.PluginTypeManager.""" import collections -import mock -import pytest - from flake8 import exceptions from flake8.plugins import manager +import mock + +import pytest + TEST_NAMESPACE = "testing.plugin-type-manager" diff --git a/tests/unit/test_style_guide.py b/tests/unit/test_style_guide.py index 03bb341..973281b 100644 --- a/tests/unit/test_style_guide.py +++ b/tests/unit/test_style_guide.py @@ -1,11 +1,12 @@ """Tests for the flake8.style_guide.StyleGuide class.""" import optparse +from flake8 import style_guide from flake8.formatting import base from flake8.plugins import notifier -from flake8 import style_guide import mock + import pytest diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 28b8e02..d69d939 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,12 +1,13 @@ """Tests for flake8's utils module.""" import os + +from flake8 import utils +from flake8.plugins import manager as plugin_manager + import mock import pytest -from flake8.plugins import manager as plugin_manager -from flake8 import utils - RELATIVE_PATHS = ["flake8", "pep8", "pyflakes", "mccabe"] From 62a78f4a97ef82b3a28d1478635fc4f501762657 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 25 Feb 2016 14:43:52 -0600 Subject: [PATCH 173/204] Add note to tox.ini for others --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index d5e07cc..946d532 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,10 @@ commands = # Flake8 Configuration [flake8] # Ignore some flake8-docstrings errors +# NOTE(sigmavirus24): While we're still using flake8 2.x, this ignore line +# defaults to selecting all other errors so we do not need select=E,F,W,I,D +# Once Flake8 3.0 is released and in a good state, we can use both and it will +# work well \o/ ignore = D203 # NOTE(sigmavirus24): Once we release 3.0.0 this exclude option can be specified # across multiple lines. Presently it cannot be specified across multiple lines. From 6a15bd00b52d44830cda9eb73411423210344fc3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 26 Feb 2016 08:21:09 -0600 Subject: [PATCH 174/204] Store plugin parameters on the plugin itself This allows us to access these from the checker module as well. --- flake8/plugins/manager.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index aba5f16..c453dbe 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -35,6 +35,7 @@ class Plugin(object): self.name = name self.entry_point = entry_point self._plugin = None + self._parameters = None def __repr__(self): """Provide an easy to read description of the current plugin.""" @@ -42,6 +43,13 @@ class Plugin(object): self.name, self.entry_point ) + @property + def parameters(self): + """List of arguments that need to be passed to the plugin.""" + if self._parameters is None: + self._parameters = utils.parameters_for(self) + return self._parameters + @property def plugin(self): """The loaded (and cached) plugin associated with the entry-point. @@ -303,8 +311,7 @@ class Checkers(PluginTypeManager): Find all checker plugins that are expecting a specific argument. """ for plugin in self.plugins.values(): - parameters = utils.parameters_for(plugin) - if argument_name == parameters[0]: + if argument_name == plugin.parameters[0]: yield plugin @property From b12f531da42df816f90f43197aa01d34aefb3d4f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 26 Feb 2016 08:52:20 -0600 Subject: [PATCH 175/204] Separate the check runner from file processor This separates concerns so that the check runner can rely on the file processor to store state and such. It introduces two logical collaborators and will allow us to keep feature parity with flake8 2's plugin design (where it could request any attribute from pep8.Checker). --- flake8/checker.py | 79 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 07c90f4..bb13b13 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -179,7 +179,73 @@ class FileChecker(object): self.filename = filename self.checks = checks self.results = [] - self.lines = [] + self.processor = FileProcessor(filename) + + def report(self, error_code, line_number, column, text): + # type: (str, int, int, str) -> NoneType + """Report an error by storing it in the results list.""" + error = (error_code, self.filename, line_number, column, text) + self.results.append(error) + + def run_check(self, plugin): + """Run the check in a single plugin.""" + arguments = {} + for parameter in plugin.parameters: + arguments[parameter] = self.attributes[parameter] + return plugin.execute(**arguments) + + def run_checks(self): + """Run checks against the file.""" + self.run_ast_checks() + self.run_physical_checks() + self.run_logical_checks() + + def run_ast_checks(self): + """Run checks that require an abstract syntax tree.""" + pass + + def run_physical_checks(self): + """Run checks that require the physical line.""" + pass + + def run_logical_checks(self): + """Run checks that require the logical line from a file.""" + pass + + +class FileProcessor(object): + """Processes a file and holdes state. + + This processes a file by generating tokens, logical and physical lines, + and AST trees. This also provides a way of passing state about the file + to checks expecting that state. Any public attribute on this object can + be requested by a plugin. The known public attributes are: + + - multiline + - max_line_length + - tokens + - indent_level + - indect_char + - noqa + - verbose + - line_number + - total_lines + - previous_logical + - logical_line + - previous_indent_level + - blank_before + - blank_lines + """ + + def __init__(self, filename): + """Initialice our file processor. + + :param str filename: + Name of the file to process + """ + self.filename = filename + self.lines = self.read_lines() + self.strip_utf_bom() def read_lines(self): # type: () -> List[str] @@ -235,17 +301,6 @@ class FileChecker(object): """Read the lines from standard in.""" return utils.stdin_get_value().splitlines(True) - def report(self, error_code, line_number, column, text): - # type: (str, int, int, str) -> NoneType - """Report an error by storing it in the results list.""" - error = (error_code, self.filename, line_number, column, text) - self.results.append(error) - - def run_checks(self): - """Run checks against the file.""" - self.lines = self.read_lines() - self.strip_utf_bom() - def strip_utf_bom(self): # type: () -> NoneType """Strip the UTF bom from the lines of the file.""" From 12e71b037218904b2fdf9ff51c7a0982adde020c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 26 Feb 2016 23:16:25 -0600 Subject: [PATCH 176/204] Incorporate more parsing logic from pycodestyle Presently we're working on having two singly-responsible classes that will be easy to test and ideally easier to reason about. --- flake8/checker.py | 150 ++++++++++++++++++++++++++++++------------- flake8/defaults.py | 3 + flake8/exceptions.py | 18 ++++++ 3 files changed, 125 insertions(+), 46 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index bb13b13..bf72362 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -10,6 +10,8 @@ try: except ImportError: multiprocessing = None +from flake8 import defaults +from flake8 import exceptions from flake8 import utils LOG = logging.getLogger(__name__) @@ -128,7 +130,7 @@ class Manager(object): paths = self.arguments filename_patterns = self.options.filename self.checkers = [ - FileChecker(filename, self.checks) + FileChecker(filename, self.checks, self.options) for argument in paths for filename in utils.filenames_from(argument, self.is_path_excluded) @@ -165,8 +167,7 @@ class Manager(object): class FileChecker(object): """Manage running checks for a file and aggregate the results.""" - def __init__(self, filename, checks): - # type: (str, flake8.plugins.manager.Checkers) -> NoneType + def __init__(self, filename, checks, options): """Initialize our file checker. :param str filename: @@ -178,8 +179,24 @@ class FileChecker(object): """ self.filename = filename self.checks = checks + self.options = options self.results = [] - self.processor = FileProcessor(filename) + self.processor = self._make_processor() + + def _make_processor(self): + try: + return FileProcessor(self.filename, self.options) + except IOError: + # If we can not read the file due to an IOError (e.g., the file + # does not exist or we do not have the permissions to open it) + # then we need to format that exception for the user. + # NOTE(sigmavirus24): Historically, pep8 has always reported this + # as an E902. We probably *want* a better error code for this + # going forward. + (exc_type, exception) = sys.exc_info()[:2] + message = '{0}: {1}'.format(exc_type.__name__, exception) + self.report('E902', 0, 0, message) + return None def report(self, error_code, line_number, column, text): # type: (str, int, int, str) -> NoneType @@ -196,21 +213,12 @@ class FileChecker(object): def run_checks(self): """Run checks against the file.""" - self.run_ast_checks() - self.run_physical_checks() - self.run_logical_checks() - - def run_ast_checks(self): - """Run checks that require an abstract syntax tree.""" - pass - - def run_physical_checks(self): - """Run checks that require the physical line.""" - pass - - def run_logical_checks(self): - """Run checks that require the logical line from a file.""" - pass + try: + for token in self.processor.generate_tokens(): + pass + except exceptions.InvalidSyntax as exc: + self.report(exc.error_code, exc.line_number, exc.column_number, + exc.error_message) class FileProcessor(object): @@ -221,23 +229,23 @@ class FileProcessor(object): to checks expecting that state. Any public attribute on this object can be requested by a plugin. The known public attributes are: - - multiline - - max_line_length - - tokens - - indent_level - - indect_char - - noqa - - verbose - - line_number - - total_lines - - previous_logical - - logical_line - - previous_indent_level - blank_before - blank_lines + - indect_char + - indent_level + - line_number + - logical_line + - max_line_length + - multiline + - noqa + - previous_indent_level + - previous_logical + - tokens + - total_lines + - verbose """ - def __init__(self, filename): + def __init__(self, filename, options): """Initialice our file processor. :param str filename: @@ -246,6 +254,69 @@ class FileProcessor(object): self.filename = filename self.lines = self.read_lines() self.strip_utf_bom() + self.options = options + + # Defaults for public attributes + #: Number of preceding blank lines + self.blank_before = 0 + #: Number of blank lines + self.blank_lines = 0 + #: Character used for indentation + self.indent_char = None + #: Current level of indentation + self.indent_level = 0 + #: Line number in the file + self.line_number = 0 + #: Current logical line + self.logical_line = '' + #: Maximum line length as configured by the user + self.max_line_length = options.max_line_length + #: Whether the current physical line is multiline + self.multiline = False + #: Whether or not we're observing NoQA + self.noqa = False + #: Previous level of indentation + self.previous_indent_level = 0 + #: Previous logical line + self.previous_logical = '' + #: Current set of tokens + self.tokens = [] + #: Total number of lines in the file + self.total_lines = len(self.lines) + #: Verbosity level of Flake8 + self.verbosity = options.verbosity + + def generate_tokens(self): + """Tokenize the file and yield the tokens. + + :raises flake8.exceptions.InvalidSyntax: + If a :class:`tokenize.TokenError` is raised while generating + tokens. + """ + try: + for token in tokenize.generate_tokens(self.next_line): + if token[2][0] > self.total_lines: + break + self.tokens.append(token) + yield token + # NOTE(sigmavirus24): pycodestyle was catching both a SyntaxError + # and a tokenize.TokenError. In looking a the source on Python 2 and + # Python 3, the SyntaxError should never arise from generate_tokens. + # If we were using tokenize.tokenize, we would have to catch that. Of + # course, I'm going to be unsurprised to be proven wrong at a later + # date. + except tokenize.TokenError as exc: + raise exceptions.InvalidSyntax(exc.message, exception=exc) + + def next_line(self): + """Get the next line from the list.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in defaults.WHITESPACE: + self.indent_char = line[0] + return line def read_lines(self): # type: () -> List[str] @@ -281,20 +352,7 @@ class FileProcessor(object): readlines = self._readlines_py2 elif (3, 0) <= sys.version_info < (4, 0): readlines = self._readlines_py3 - - try: - return readlines() - except IOError: - # If we can not read the file due to an IOError (e.g., the file - # does not exist or we do not have the permissions to open it) - # then we need to format that exception for the user. - # NOTE(sigmavirus24): Historically, pep8 has always reported this - # as an E902. We probably *want* a better error code for this - # going forward. - (exc_type, exception) = sys.exc_info()[:2] - message = '{0}: {1}'.format(exc_type.__name__, exception) - self.report('E902', 0, 0, message) - return [] + return readlines() def read_lines_from_stdin(self): # type: () -> List[str] diff --git a/flake8/defaults.py b/flake8/defaults.py index ac798b5..62939b5 100644 --- a/flake8/defaults.py +++ b/flake8/defaults.py @@ -3,3 +3,6 @@ EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' IGNORE = 'E121,E123,E126,E226,E24,E704' MAX_LINE_LENGTH = 79 + +# Other consants +WHITESPACE = frozenset(' \t') diff --git a/flake8/exceptions.py b/flake8/exceptions.py index ac78769..18ee90c 100644 --- a/flake8/exceptions.py +++ b/flake8/exceptions.py @@ -23,3 +23,21 @@ class FailedToLoadPlugin(Flake8Exception): """Return a nice string for our exception.""" return self.FORMAT % {'name': self.ep_name, 'exc': self.original_exception} + + +class InvalidSyntax(Flake8Exception): + """Exception raised when tokenizing a file fails.""" + + def __init__(self, *args, **kwargs): + """Initialize our InvalidSyntax exception.""" + self.original_exception = kwargs.pop('exception') + self.error_code = 'E902' + self.line_number = 1 + self.column_number = 0 + try: + self.error_message = self.original_exception.message + except AttributeError: + # On Python 3, the IOError is an OSError which has a + # strerror attribute instead of a message attribute + self.error_message = self.original_exception.strerror + super(InvalidSyntax, self).__init__(*args, **kwargs) From 074739de276283fea9a574e08db82e404ddf4fc4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 27 Feb 2016 00:06:19 -0600 Subject: [PATCH 177/204] Accept the StyleGuide instead of options pep8's checker has the noqa logic which we've correctly placed on the StyleGuide object. By passing the StyleGuide in to our checkers we can have the checkers pass the physical line to the StyleGuide so we can eliminate our usage of linecache. --- flake8/checker.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index bf72362..9f2154d 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -36,13 +36,13 @@ class Manager(object): together and make our output deterministic. """ - def __init__(self, options, arguments, checker_plugins): + def __init__(self, style_guide, arguments, checker_plugins): """Initialize our Manager instance. - :param options: - The options parsed from config files and CLI. - :type options: - optparse.Values + :param style_guide: + The instantiated style guide for this instance of Flake8. + :type style_guide: + flake8.style_guide.StyleGuide :param list arguments: The extra arguments parsed from the CLI (if any) :param checker_plugins: @@ -51,7 +51,8 @@ class Manager(object): flake8.plugins.manager.Checkers """ self.arguments = arguments - self.options = options + self.style_guide = style_guide + self.options = style_guide.options self.checks = checker_plugins self.jobs = self._job_count() self.process_queue = None @@ -130,7 +131,7 @@ class Manager(object): paths = self.arguments filename_patterns = self.options.filename self.checkers = [ - FileChecker(filename, self.checks, self.options) + FileChecker(filename, self.checks, self.style_guide) for argument in paths for filename in utils.filenames_from(argument, self.is_path_excluded) @@ -167,7 +168,7 @@ class Manager(object): class FileChecker(object): """Manage running checks for a file and aggregate the results.""" - def __init__(self, filename, checks, options): + def __init__(self, filename, checks, style_guide): """Initialize our file checker. :param str filename: @@ -179,13 +180,13 @@ class FileChecker(object): """ self.filename = filename self.checks = checks - self.options = options + self.style_guide = style_guide self.results = [] self.processor = self._make_processor() def _make_processor(self): try: - return FileProcessor(self.filename, self.options) + return FileProcessor(self.filename, self.style_guide.options) except IOError: # If we can not read the file due to an IOError (e.g., the file # does not exist or we do not have the permissions to open it) From 6ac955dfd4db7632018211c8d55bac678081426d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 28 Feb 2016 00:55:47 -0600 Subject: [PATCH 178/204] Continue porting more logic from pep8 --- flake8/checker.py | 77 ++++++++++++++++++++++++++++++++++++++++--- flake8/style_guide.py | 6 ++-- flake8/utils.py | 22 +++++++++++++ 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 9f2154d..38a4634 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -1,4 +1,5 @@ """Checker Manager and Checker classes.""" +import contextlib import io import logging import os @@ -205,22 +206,57 @@ class FileChecker(object): error = (error_code, self.filename, line_number, column, text) self.results.append(error) - def run_check(self, plugin): + def run_check(self, plugin, **arguments): """Run the check in a single plugin.""" - arguments = {} - for parameter in plugin.parameters: - arguments[parameter] = self.attributes[parameter] + self.processor.keyword_arguments_for(plugin.parameters, arguments) return plugin.execute(**arguments) + def run_physical_checks(self, physical_line): + for plugin in self.checks.physical_line_plugins: + result = self.run_check(plugin, physical_line=physical_line) + if result is not None: + column_offset, text = result + error_code, error_text = text.split(' ', 1) + self.report( + error_code=error_code, + line_number=self.processor.line_number, + column=column_offset, + text=error_text, + ) + + self.processor.check_physical_error(error_code, physical_line) + def run_checks(self): """Run checks against the file.""" try: for token in self.processor.generate_tokens(): - pass + self.check_physical_eol(token) except exceptions.InvalidSyntax as exc: self.report(exc.error_code, exc.line_number, exc.column_number, exc.error_message) + def check_physical_eol(self, token): + """Run physical checks if and only if it is at the end of the line.""" + if utils.is_eol_token(token): + # Obviously, a newline token ends a single physical line. + self.run_physical_checks(token[4]) + elif utils.is_multiline_string(token): + # Less obviously, a string that contains newlines is a + # multiline string, either triple-quoted or with internal + # newlines backslash-escaped. Check every physical line in the + # string *except* for the last one: its newline is outside of + # the multiline string, so we consider it a regular physical + # line, and will check it like any other physical line. + # + # Subtleties: + # - have to wind self.line_number back because initially it + # points to the last line of the string, and we want + # check_physical() to give accurate feedback + line_no = token[2][0] + with self.processor.inside_multiline(line_number=line_no): + for line in self.processor.split_line(token): + self.run_physical_checks(line + '\n') + class FileProcessor(object): """Processes a file and holdes state. @@ -287,6 +323,37 @@ class FileProcessor(object): #: Verbosity level of Flake8 self.verbosity = options.verbosity + @contextlib.contextmanager + def inside_multiline(self, line_number): + """Context-manager to toggle the multiline attribute.""" + self.line_number = line_number + self.multiline = True + yield + self.multiline = False + + def split_line(self, token): + """Split a physical line's line based on new-lines. + + This also auto-increments the line number for the caller. + """ + for line in token[1].split('\n')[:-1]: + yield line + self.line_number += 1 + + def keyword_arguments_for(self, parameters, arguments=None): + """Generate the keyword arguments for a list of parameters.""" + if arguments is None: + arguments = {} + for param in parameters: + if param not in arguments: + arguments[param] = getattr(self, param) + return arguments + + def check_physical_error(self, error_code, line): + """Update attributes based on error code and line.""" + if error_code == 'E101': + self.indent_char = line[0] + def generate_tokens(self): """Tokenize the file and yield the tokens. diff --git a/flake8/style_guide.py b/flake8/style_guide.py index bbfe658..51c3a8e 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -157,14 +157,16 @@ class StyleGuide(object): LOG.debug('"%s" will be "%s"', code, decision) return decision - def is_inline_ignored(self, error): + def is_inline_ignored(self, error, physical_line=None): # type: (Error) -> bool """Determine if an comment has been added to ignore this line.""" # TODO(sigmavirus24): Determine how to handle stdin with linecache if self.options.disable_noqa: return False - physical_line = linecache.getline(error.filename, error.line_number) + if physical_line is None: + physical_line = linecache.getline(error.filename, + error.line_number) noqa_match = self.NOQA_INLINE_REGEXP.search(physical_line) if noqa_match is None: LOG.debug('%r is not inline ignored', error) diff --git a/flake8/utils.py b/flake8/utils.py index 7e08e41..a3be0ea 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -4,6 +4,7 @@ import inspect import io import os import sys +import tokenize def parse_comma_separated_list(value): @@ -183,3 +184,24 @@ def parameters_for(plugin): parameters.remove('self') return parameters + +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + + +def is_eol_token(token): + """Check if the token is an end-of-line token.""" + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' + +if COMMENT_WITH_NL: # If on Python 2.6 + def is_eol_token(token, _is_eol_token=is_eol_token): + """Check if the token is an end-of-line token.""" + return (_is_eol_token(token) or + (token[0] == tokenize.COMMENT and token[1] == token[4])) + + +def is_multiline_string(token): + """Check if this is a multiline string.""" + return token[0] == tokenize.STRING and '\n' in token[1] From 0c894cc8bfb066b8e45daa7b2c6ebde480b1eccf Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 1 Mar 2016 21:27:36 -0600 Subject: [PATCH 179/204] Pull more logic out of pep8 --- flake8/checker.py | 98 +++++++++++++++++++++++++++++++++++++++++++---- flake8/utils.py | 18 +++++++++ 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 38a4634..a6c14ec 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -201,40 +201,112 @@ class FileChecker(object): return None def report(self, error_code, line_number, column, text): - # type: (str, int, int, str) -> NoneType + # type: (str, int, int, str) -> str """Report an error by storing it in the results list.""" + if error_code is None: + error_code, text = text.split(' ', 1) error = (error_code, self.filename, line_number, column, text) self.results.append(error) + return error_code def run_check(self, plugin, **arguments): """Run the check in a single plugin.""" self.processor.keyword_arguments_for(plugin.parameters, arguments) return plugin.execute(**arguments) + def run_logical_checks(self): + """Run all checks expecting a logical line.""" + for plugin in self.checks.logical_line_plugins: + result = self.run_check(plugin) # , logical_line=logical_line) + if result is not None: + column_offset, text = result + self.report( + error_code=None, + line_number=self.processor.line_number, + column=column_offset, + text=text, + ) + def run_physical_checks(self, physical_line): + """Run all checks for a given physical line.""" for plugin in self.checks.physical_line_plugins: result = self.run_check(plugin, physical_line=physical_line) if result is not None: column_offset, text = result - error_code, error_text = text.split(' ', 1) - self.report( - error_code=error_code, + error_code = self.report( + error_code=None, line_number=self.processor.line_number, column=column_offset, - text=error_text, + text=text, ) self.processor.check_physical_error(error_code, physical_line) + def _log_token(self, token): + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + LOG.debug('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], + token[1])) + + def process_tokens(self): + """Process tokens and trigger checks. + + This can raise a :class:`flake8.exceptions.InvalidSyntax` exception. + Instead of using this directly, you should use + :meth:`flake8.checker.FileChecker.run_checks`. + """ + parens = 0 + processor = self.processor + for token in processor.generate_tokens(): + self.check_physical_eol(token) + token_type, text = token[0:2] + self._log_token(token) + if token_type == tokenize.OP: + parens = utils.count_parentheses(parens, text) + elif parens == 0: + if utils.token_is_newline(token): + self.handle_newline(token_type) + elif (utils.token_is_comment(token) and + len(processor.tokens) == 1): + self.handle_comment(token, text) + + if processor.tokens: + # If any tokens are left over, process them + self.run_physical_checks(processor.lines[-1]) + self.run_logical_checks() + def run_checks(self): """Run checks against the file.""" try: - for token in self.processor.generate_tokens(): - self.check_physical_eol(token) + self.process_tokens() except exceptions.InvalidSyntax as exc: self.report(exc.error_code, exc.line_number, exc.column_number, exc.error_message) + def handle_comment(self, token, token_text): + """Handle the logic when encountering a comment token.""" + # The comment also ends a physical line + token = list(token) + token[1] = token_text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.processor.tokens = [tuple(token)] + self.run_logical_checks() + + def handle_newline(self, token_type): + """Handle the logic when encountering a newline token.""" + if token_type == tokenize.NEWLINE: + self.check_logical() + self.processor.reset_blank_before() + elif len(self.processor.tokens) == 1: + # The physical line contains only this token. + self.processor.visited_new_blank_line() + self.processor.delete_first_token() + else: + self.run_logical_checks() + def check_physical_eol(self, token): """Run physical checks if and only if it is at the end of the line.""" if utils.is_eol_token(token): @@ -331,6 +403,18 @@ class FileProcessor(object): yield self.multiline = False + def reset_blank_before(self): + """Reset the blank_before attribute to zero.""" + self.blank_before = 0 + + def delete_first_token(self): + """Delete the first token in the list of tokens.""" + del self.tokens[0] + + def visited_new_blank_line(self): + """Note that we visited a new blank line.""" + self.blank_lines += 1 + def split_line(self, token): """Split a physical line's line based on new-lines. diff --git a/flake8/utils.py b/flake8/utils.py index a3be0ea..97b766a 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -205,3 +205,21 @@ if COMMENT_WITH_NL: # If on Python 2.6 def is_multiline_string(token): """Check if this is a multiline string.""" return token[0] == tokenize.STRING and '\n' in token[1] + + +def token_is_newline(token): + """Check if the token type is a newline token type.""" + return token[0] in NEWLINE + + +def token_is_comment(token): + """Check if the token type is a comment.""" + return COMMENT_WITH_NL and token[0] == tokenize.COMMENT + + +def count_parentheses(current_parentheses_count, token_text): + """Count the number of parentheses.""" + if token_text in '([{': + return current_parentheses_count + 1 + elif token_text in '}])': + return current_parentheses_count - 1 From 23c9091b1ab5119643f1dd742687f2ae452a6f4c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 2 Mar 2016 23:28:24 -0600 Subject: [PATCH 180/204] Slowly working through pep8.Checker.check_logical --- flake8/checker.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++- flake8/utils.py | 20 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/flake8/checker.py b/flake8/checker.py index a6c14ec..82f9e79 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -17,6 +17,9 @@ from flake8 import utils LOG = logging.getLogger(__name__) +SKIP_TOKENS = frozenset([tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, + tokenize.DEDENT]) + class Manager(object): """Manage the parallelism and checker instances for each plugin and file. @@ -216,8 +219,15 @@ class FileChecker(object): def run_logical_checks(self): """Run all checks expecting a logical line.""" + comments, logical_line, mapping = self.processor.build_logical_line() + if not mapping: + return + self.processor.update_state(mapping) + + LOG.debug('Logical line: "%s"', logical_line.rstrip()) + for plugin in self.checks.logical_line_plugins: - result = self.run_check(plugin) # , logical_line=logical_line) + result = self.run_check(plugin, logical_line=logical_line) if result is not None: column_offset, text = result self.report( @@ -415,6 +425,45 @@ class FileProcessor(object): """Note that we visited a new blank line.""" self.blank_lines += 1 + def build_logical_line_tokens(self): + """Build the mapping, comments, and logical line lists.""" + logical = [] + comments = [] + length = 0 + previous_row = previous_column = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = utils.mutate_string(text) + if previous_row: + (start_row, start_column) = start + if previous_row != start_row: + row_index = previous_row - 1 + column_index = previous_column - 1 + previous_text = self.lines[row_index][column_index] + if (previous_text == ',' or + (previous_text not in '{[(' and + text not in '}])')): + text = ' ' + text + elif previous_column != start_column: + text = line[previous_column:start_column] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (previous_row, previous_column) = end + return comments, logical, mapping + + def build_logical_line(self): + """Build a logical line from the current tokens list.""" + comments, logical, mapping_list = self.build_logical_line_tokens() + return ''.join(comments), ''.join(logical), mapping_list + def split_line(self, token): """Split a physical line's line based on new-lines. diff --git a/flake8/utils.py b/flake8/utils.py index 97b766a..fe024d1 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -223,3 +223,23 @@ def count_parentheses(current_parentheses_count, token_text): return current_parentheses_count + 1 elif token_text in '}])': return current_parentheses_count - 1 + + +def mutate_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] From f7a8b7ade769f93b6d4a9e0f976042a4e45b304b Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 4 Mar 2016 20:03:05 -0600 Subject: [PATCH 181/204] Move processor to its own module --- flake8/checker.py | 279 +++------------------------------------- flake8/processor.py | 300 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 314 insertions(+), 265 deletions(-) create mode 100644 flake8/processor.py diff --git a/flake8/checker.py b/flake8/checker.py index 82f9e79..69bab17 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -1,6 +1,4 @@ """Checker Manager and Checker classes.""" -import contextlib -import io import logging import os import sys @@ -11,15 +9,12 @@ try: except ImportError: multiprocessing = None -from flake8 import defaults from flake8 import exceptions +from flake8 import processor from flake8 import utils LOG = logging.getLogger(__name__) -SKIP_TOKENS = frozenset([tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, - tokenize.DEDENT]) - class Manager(object): """Manage the parallelism and checker instances for each plugin and file. @@ -190,7 +185,8 @@ class FileChecker(object): def _make_processor(self): try: - return FileProcessor(self.filename, self.style_guide.options) + return processor.FileProcessor(self.filename, + self.style_guide.options) except IOError: # If we can not read the file due to an IOError (e.g., the file # does not exist or we do not have the permissions to open it) @@ -252,15 +248,6 @@ class FileChecker(object): self.processor.check_physical_error(error_code, physical_line) - def _log_token(self, token): - if token[2][0] == token[3][0]: - pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) - else: - pos = 'l.%s' % token[3][0] - LOG.debug('l.%s\t%s\t%s\t%r' % - (token[2][0], pos, tokenize.tok_name[token[0]], - token[1])) - def process_tokens(self): """Process tokens and trigger checks. @@ -269,23 +256,23 @@ class FileChecker(object): :meth:`flake8.checker.FileChecker.run_checks`. """ parens = 0 - processor = self.processor - for token in processor.generate_tokens(): + file_processor = self.processor + for token in file_processor.generate_tokens(): self.check_physical_eol(token) token_type, text = token[0:2] - self._log_token(token) + processor.log_token(token) if token_type == tokenize.OP: - parens = utils.count_parentheses(parens, text) + parens = processor.count_parentheses(parens, text) elif parens == 0: - if utils.token_is_newline(token): + if processor.token_is_newline(token): self.handle_newline(token_type) - elif (utils.token_is_comment(token) and - len(processor.tokens) == 1): + elif (processor.token_is_comment(token) and + len(file_processor.tokens) == 1): self.handle_comment(token, text) - if processor.tokens: + if file_processor.tokens: # If any tokens are left over, process them - self.run_physical_checks(processor.lines[-1]) + self.run_physical_checks(file_processor.lines[-1]) self.run_logical_checks() def run_checks(self): @@ -319,10 +306,10 @@ class FileChecker(object): def check_physical_eol(self, token): """Run physical checks if and only if it is at the end of the line.""" - if utils.is_eol_token(token): + if processor.is_eol_token(token): # Obviously, a newline token ends a single physical line. self.run_physical_checks(token[4]) - elif utils.is_multiline_string(token): + elif processor.is_multiline_string(token): # Less obviously, a string that contains newlines is a # multiline string, either triple-quoted or with internal # newlines backslash-escaped. Check every physical line in the @@ -338,241 +325,3 @@ class FileChecker(object): with self.processor.inside_multiline(line_number=line_no): for line in self.processor.split_line(token): self.run_physical_checks(line + '\n') - - -class FileProcessor(object): - """Processes a file and holdes state. - - This processes a file by generating tokens, logical and physical lines, - and AST trees. This also provides a way of passing state about the file - to checks expecting that state. Any public attribute on this object can - be requested by a plugin. The known public attributes are: - - - blank_before - - blank_lines - - indect_char - - indent_level - - line_number - - logical_line - - max_line_length - - multiline - - noqa - - previous_indent_level - - previous_logical - - tokens - - total_lines - - verbose - """ - - def __init__(self, filename, options): - """Initialice our file processor. - - :param str filename: - Name of the file to process - """ - self.filename = filename - self.lines = self.read_lines() - self.strip_utf_bom() - self.options = options - - # Defaults for public attributes - #: Number of preceding blank lines - self.blank_before = 0 - #: Number of blank lines - self.blank_lines = 0 - #: Character used for indentation - self.indent_char = None - #: Current level of indentation - self.indent_level = 0 - #: Line number in the file - self.line_number = 0 - #: Current logical line - self.logical_line = '' - #: Maximum line length as configured by the user - self.max_line_length = options.max_line_length - #: Whether the current physical line is multiline - self.multiline = False - #: Whether or not we're observing NoQA - self.noqa = False - #: Previous level of indentation - self.previous_indent_level = 0 - #: Previous logical line - self.previous_logical = '' - #: Current set of tokens - self.tokens = [] - #: Total number of lines in the file - self.total_lines = len(self.lines) - #: Verbosity level of Flake8 - self.verbosity = options.verbosity - - @contextlib.contextmanager - def inside_multiline(self, line_number): - """Context-manager to toggle the multiline attribute.""" - self.line_number = line_number - self.multiline = True - yield - self.multiline = False - - def reset_blank_before(self): - """Reset the blank_before attribute to zero.""" - self.blank_before = 0 - - def delete_first_token(self): - """Delete the first token in the list of tokens.""" - del self.tokens[0] - - def visited_new_blank_line(self): - """Note that we visited a new blank line.""" - self.blank_lines += 1 - - def build_logical_line_tokens(self): - """Build the mapping, comments, and logical line lists.""" - logical = [] - comments = [] - length = 0 - previous_row = previous_column = mapping = None - for token_type, text, start, end, line in self.tokens: - if token_type in SKIP_TOKENS: - continue - if not mapping: - mapping = [(0, start)] - if token_type == tokenize.COMMENT: - comments.append(text) - continue - if token_type == tokenize.STRING: - text = utils.mutate_string(text) - if previous_row: - (start_row, start_column) = start - if previous_row != start_row: - row_index = previous_row - 1 - column_index = previous_column - 1 - previous_text = self.lines[row_index][column_index] - if (previous_text == ',' or - (previous_text not in '{[(' and - text not in '}])')): - text = ' ' + text - elif previous_column != start_column: - text = line[previous_column:start_column] + text - logical.append(text) - length += len(text) - mapping.append((length, end)) - (previous_row, previous_column) = end - return comments, logical, mapping - - def build_logical_line(self): - """Build a logical line from the current tokens list.""" - comments, logical, mapping_list = self.build_logical_line_tokens() - return ''.join(comments), ''.join(logical), mapping_list - - def split_line(self, token): - """Split a physical line's line based on new-lines. - - This also auto-increments the line number for the caller. - """ - for line in token[1].split('\n')[:-1]: - yield line - self.line_number += 1 - - def keyword_arguments_for(self, parameters, arguments=None): - """Generate the keyword arguments for a list of parameters.""" - if arguments is None: - arguments = {} - for param in parameters: - if param not in arguments: - arguments[param] = getattr(self, param) - return arguments - - def check_physical_error(self, error_code, line): - """Update attributes based on error code and line.""" - if error_code == 'E101': - self.indent_char = line[0] - - def generate_tokens(self): - """Tokenize the file and yield the tokens. - - :raises flake8.exceptions.InvalidSyntax: - If a :class:`tokenize.TokenError` is raised while generating - tokens. - """ - try: - for token in tokenize.generate_tokens(self.next_line): - if token[2][0] > self.total_lines: - break - self.tokens.append(token) - yield token - # NOTE(sigmavirus24): pycodestyle was catching both a SyntaxError - # and a tokenize.TokenError. In looking a the source on Python 2 and - # Python 3, the SyntaxError should never arise from generate_tokens. - # If we were using tokenize.tokenize, we would have to catch that. Of - # course, I'm going to be unsurprised to be proven wrong at a later - # date. - except tokenize.TokenError as exc: - raise exceptions.InvalidSyntax(exc.message, exception=exc) - - def next_line(self): - """Get the next line from the list.""" - if self.line_number >= self.total_lines: - return '' - line = self.lines[self.line_number] - self.line_number += 1 - if self.indent_char is None and line[:1] in defaults.WHITESPACE: - self.indent_char = line[0] - return line - - def read_lines(self): - # type: () -> List[str] - """Read the lines for this file checker.""" - if self.filename is None or self.filename == '-': - self.filename = 'stdin' - return self.read_lines_from_stdin() - return self.read_lines_from_filename() - - def _readlines_py2(self): - # type: () -> List[str] - with open(self.filename, 'rU') as fd: - return fd.readlines() - - def _readlines_py3(self): - # type: () -> List[str] - try: - with open(self.filename, 'rb') as fd: - (coding, lines) = tokenize.detect_encoding(fd.readline) - textfd = io.TextIOWrapper(fd, coding, line_buffering=True) - return ([l.decode(coding) for l in lines] + - textfd.readlines()) - except (LookupError, SyntaxError, UnicodeError): - # If we can't detect the codec with tokenize.detect_encoding, or - # the detected encoding is incorrect, just fallback to latin-1. - with open(self.filename, encoding='latin-1') as fd: - return fd.readlines() - - def read_lines_from_filename(self): - # type: () -> List[str] - """Read the lines for a file.""" - if (2, 6) <= sys.version_info < (3, 0): - readlines = self._readlines_py2 - elif (3, 0) <= sys.version_info < (4, 0): - readlines = self._readlines_py3 - return readlines() - - def read_lines_from_stdin(self): - # type: () -> List[str] - """Read the lines from standard in.""" - return utils.stdin_get_value().splitlines(True) - - def strip_utf_bom(self): - # type: () -> NoneType - """Strip the UTF bom from the lines of the file.""" - if not self.lines: - # If we have nothing to analyze quit early - return - - first_byte = ord(self.lines[0][0]) - if first_byte not in (0xEF, 0xFEFF): - return - - # If the first byte of the file is a UTF-8 BOM, strip it - if first_byte == 0xFEFF: - self.lines[0] = self.lines[0][1:] - elif self.lines[0][:3] == '\xEF\xBB\xBF': - self.lines[0] = self.lines[0][3:] diff --git a/flake8/processor.py b/flake8/processor.py new file mode 100644 index 0000000..e681c03 --- /dev/null +++ b/flake8/processor.py @@ -0,0 +1,300 @@ +"""Module containing our file processor that tokenizes a file for checks.""" +import contextlib +import io +import sys +import tokenize + +from flake8 import defaults +from flake8 import exceptions +from flake8 import utils + +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + +SKIP_TOKENS = frozenset([tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, + tokenize.DEDENT]) + + +class FileProcessor(object): + """Processes a file and holdes state. + + This processes a file by generating tokens, logical and physical lines, + and AST trees. This also provides a way of passing state about the file + to checks expecting that state. Any public attribute on this object can + be requested by a plugin. The known public attributes are: + + - blank_before + - blank_lines + - indect_char + - indent_level + - line_number + - logical_line + - max_line_length + - multiline + - noqa + - previous_indent_level + - previous_logical + - tokens + - total_lines + - verbose + """ + + def __init__(self, filename, options): + """Initialice our file processor. + + :param str filename: + Name of the file to process + """ + self.filename = filename + self.lines = self.read_lines() + self.strip_utf_bom() + self.options = options + + # Defaults for public attributes + #: Number of preceding blank lines + self.blank_before = 0 + #: Number of blank lines + self.blank_lines = 0 + #: Character used for indentation + self.indent_char = None + #: Current level of indentation + self.indent_level = 0 + #: Line number in the file + self.line_number = 0 + #: Current logical line + self.logical_line = '' + #: Maximum line length as configured by the user + self.max_line_length = options.max_line_length + #: Whether the current physical line is multiline + self.multiline = False + #: Whether or not we're observing NoQA + self.noqa = False + #: Previous level of indentation + self.previous_indent_level = 0 + #: Previous logical line + self.previous_logical = '' + #: Current set of tokens + self.tokens = [] + #: Total number of lines in the file + self.total_lines = len(self.lines) + #: Verbosity level of Flake8 + self.verbosity = options.verbosity + + @contextlib.contextmanager + def inside_multiline(self, line_number): + """Context-manager to toggle the multiline attribute.""" + self.line_number = line_number + self.multiline = True + yield + self.multiline = False + + def reset_blank_before(self): + """Reset the blank_before attribute to zero.""" + self.blank_before = 0 + + def delete_first_token(self): + """Delete the first token in the list of tokens.""" + del self.tokens[0] + + def visited_new_blank_line(self): + """Note that we visited a new blank line.""" + self.blank_lines += 1 + + def build_logical_line_tokens(self): + """Build the mapping, comments, and logical line lists.""" + logical = [] + comments = [] + length = 0 + previous_row = previous_column = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = utils.mutate_string(text) + if previous_row: + (start_row, start_column) = start + if previous_row != start_row: + row_index = previous_row - 1 + column_index = previous_column - 1 + previous_text = self.lines[row_index][column_index] + if (previous_text == ',' or + (previous_text not in '{[(' and + text not in '}])')): + text = ' ' + text + elif previous_column != start_column: + text = line[previous_column:start_column] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (previous_row, previous_column) = end + return comments, logical, mapping + + def build_logical_line(self): + """Build a logical line from the current tokens list.""" + comments, logical, mapping_list = self.build_logical_line_tokens() + return ''.join(comments), ''.join(logical), mapping_list + + def split_line(self, token): + """Split a physical line's line based on new-lines. + + This also auto-increments the line number for the caller. + """ + for line in token[1].split('\n')[:-1]: + yield line + self.line_number += 1 + + def keyword_arguments_for(self, parameters, arguments=None): + """Generate the keyword arguments for a list of parameters.""" + if arguments is None: + arguments = {} + for param in parameters: + if param not in arguments: + arguments[param] = getattr(self, param) + return arguments + + def check_physical_error(self, error_code, line): + """Update attributes based on error code and line.""" + if error_code == 'E101': + self.indent_char = line[0] + + def generate_tokens(self): + """Tokenize the file and yield the tokens. + + :raises flake8.exceptions.InvalidSyntax: + If a :class:`tokenize.TokenError` is raised while generating + tokens. + """ + try: + for token in tokenize.generate_tokens(self.next_line): + if token[2][0] > self.total_lines: + break + self.tokens.append(token) + yield token + # NOTE(sigmavirus24): pycodestyle was catching both a SyntaxError + # and a tokenize.TokenError. In looking a the source on Python 2 and + # Python 3, the SyntaxError should never arise from generate_tokens. + # If we were using tokenize.tokenize, we would have to catch that. Of + # course, I'm going to be unsurprised to be proven wrong at a later + # date. + except tokenize.TokenError as exc: + raise exceptions.InvalidSyntax(exc.message, exception=exc) + + def next_line(self): + """Get the next line from the list.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in defaults.WHITESPACE: + self.indent_char = line[0] + return line + + def read_lines(self): + # type: () -> List[str] + """Read the lines for this file checker.""" + if self.filename is None or self.filename == '-': + self.filename = 'stdin' + return self.read_lines_from_stdin() + return self.read_lines_from_filename() + + def _readlines_py2(self): + # type: () -> List[str] + with open(self.filename, 'rU') as fd: + return fd.readlines() + + def _readlines_py3(self): + # type: () -> List[str] + try: + with open(self.filename, 'rb') as fd: + (coding, lines) = tokenize.detect_encoding(fd.readline) + textfd = io.TextIOWrapper(fd, coding, line_buffering=True) + return ([l.decode(coding) for l in lines] + + textfd.readlines()) + except (LookupError, SyntaxError, UnicodeError): + # If we can't detect the codec with tokenize.detect_encoding, or + # the detected encoding is incorrect, just fallback to latin-1. + with open(self.filename, encoding='latin-1') as fd: + return fd.readlines() + + def read_lines_from_filename(self): + # type: () -> List[str] + """Read the lines for a file.""" + if (2, 6) <= sys.version_info < (3, 0): + readlines = self._readlines_py2 + elif (3, 0) <= sys.version_info < (4, 0): + readlines = self._readlines_py3 + return readlines() + + def read_lines_from_stdin(self): + # type: () -> List[str] + """Read the lines from standard in.""" + return utils.stdin_get_value().splitlines(True) + + def strip_utf_bom(self): + # type: () -> NoneType + """Strip the UTF bom from the lines of the file.""" + if not self.lines: + # If we have nothing to analyze quit early + return + + first_byte = ord(self.lines[0][0]) + if first_byte not in (0xEF, 0xFEFF): + return + + # If the first byte of the file is a UTF-8 BOM, strip it + if first_byte == 0xFEFF: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xEF\xBB\xBF': + self.lines[0] = self.lines[0][3:] + + +def is_eol_token(token): + """Check if the token is an end-of-line token.""" + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' + +if COMMENT_WITH_NL: # If on Python 2.6 + def is_eol_token(token, _is_eol_token=is_eol_token): + """Check if the token is an end-of-line token.""" + return (_is_eol_token(token) or + (token[0] == tokenize.COMMENT and token[1] == token[4])) + + +def is_multiline_string(token): + """Check if this is a multiline string.""" + return token[0] == tokenize.STRING and '\n' in token[1] + + +def token_is_newline(token): + """Check if the token type is a newline token type.""" + return token[0] in NEWLINE + + +def token_is_comment(token): + """Check if the token type is a comment.""" + return COMMENT_WITH_NL and token[0] == tokenize.COMMENT + + +def count_parentheses(current_parentheses_count, token_text): + """Count the number of parentheses.""" + if token_text in '([{': + return current_parentheses_count + 1 + elif token_text in '}])': + return current_parentheses_count - 1 + + +def log_token(log, token): + """Log a token to a provided logging object.""" + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + log.debug('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], + token[1])) From 36fb688f97d69d1ba0e1ea7bd6502ec7fe8b3141 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 4 Mar 2016 22:51:22 -0600 Subject: [PATCH 182/204] Refactor processor and file checker some more --- flake8/checker.py | 22 ++++++++++++--- flake8/processor.py | 68 ++++++++++++++++++++++++++++++++++++++++++++- flake8/utils.py | 60 --------------------------------------- 3 files changed, 85 insertions(+), 65 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 69bab17..c058db5 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -223,16 +223,19 @@ class FileChecker(object): LOG.debug('Logical line: "%s"', logical_line.rstrip()) for plugin in self.checks.logical_line_plugins: - result = self.run_check(plugin, logical_line=logical_line) - if result is not None: - column_offset, text = result + results = self.run_check(plugin, logical_line=logical_line) or () + for offset, text in results: + offset = find_offset(offset, mapping) + line_number, column_offset = offset self.report( error_code=None, - line_number=self.processor.line_number, + line_number=line_number, column=column_offset, text=text, ) + self.processor.next_logical_line() + def run_physical_checks(self, physical_line): """Run all checks for a given physical line.""" for plugin in self.checks.physical_line_plugins: @@ -325,3 +328,14 @@ class FileChecker(object): with self.processor.inside_multiline(line_number=line_no): for line in self.processor.split_line(token): self.run_physical_checks(line + '\n') + + +def find_offset(offset, mapping): + """Find the offset tuple for a single offset.""" + if isinstance(offset, tuple): + return offset + + for token_offset, position in mapping: + if offset <= token_offset: + break + return (position[0], position[1] + offset - token_offset) diff --git a/flake8/processor.py b/flake8/processor.py index e681c03..062006d 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -102,6 +102,25 @@ class FileProcessor(object): """Note that we visited a new blank line.""" self.blank_lines += 1 + def update_state(self, mapping): + """Update the indent level based on the logical line mapping.""" + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + + def next_logical_line(self): + """Record the previous logical line. + + This also resets the tokens list and the blank_lines count. + """ + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + self.blank_lines = 0 + self.tokens = [] + def build_logical_line_tokens(self): """Build the mapping, comments, and logical line lists.""" logical = [] @@ -117,7 +136,7 @@ class FileProcessor(object): comments.append(text) continue if token_type == tokenize.STRING: - text = utils.mutate_string(text) + text = mutate_string(text) if previous_row: (start_row, start_column) = start if previous_row != start_row: @@ -298,3 +317,50 @@ def log_token(log, token): log.debug('l.%s\t%s\t%s\t%r' % (token[2][0], pos, tokenize.tok_name[token[0]], token[1])) + + +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +def mutate_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] diff --git a/flake8/utils.py b/flake8/utils.py index fe024d1..7e08e41 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -4,7 +4,6 @@ import inspect import io import os import sys -import tokenize def parse_comma_separated_list(value): @@ -184,62 +183,3 @@ def parameters_for(plugin): parameters.remove('self') return parameters - -NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) -# Work around Python < 2.6 behaviour, which does not generate NL after -# a comment which is on a line by itself. -COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' - - -def is_eol_token(token): - """Check if the token is an end-of-line token.""" - return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' - -if COMMENT_WITH_NL: # If on Python 2.6 - def is_eol_token(token, _is_eol_token=is_eol_token): - """Check if the token is an end-of-line token.""" - return (_is_eol_token(token) or - (token[0] == tokenize.COMMENT and token[1] == token[4])) - - -def is_multiline_string(token): - """Check if this is a multiline string.""" - return token[0] == tokenize.STRING and '\n' in token[1] - - -def token_is_newline(token): - """Check if the token type is a newline token type.""" - return token[0] in NEWLINE - - -def token_is_comment(token): - """Check if the token type is a comment.""" - return COMMENT_WITH_NL and token[0] == tokenize.COMMENT - - -def count_parentheses(current_parentheses_count, token_text): - """Count the number of parentheses.""" - if token_text in '([{': - return current_parentheses_count + 1 - elif token_text in '}])': - return current_parentheses_count - 1 - - -def mutate_string(text): - """Replace contents with 'xxx' to prevent syntax matching. - - >>> mute_string('"abc"') - '"xxx"' - >>> mute_string("'''abc'''") - "'''xxx'''" - >>> mute_string("r'abc'") - "r'xxx'" - """ - # String modifiers (e.g. u or r) - start = text.index(text[-1]) + 1 - end = len(text) - 1 - # Triple quotes - if text[-3:] in ('"""', "'''"): - start += 2 - end -= 2 - return text[:start] + 'x' * (end - start) + text[end:] From da2182151773e5e37faa87ad607308694e6c1997 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 10 Mar 2016 19:00:07 -0600 Subject: [PATCH 183/204] Make flake8 actually work --- flake8/checker.py | 13 +++++++++++-- flake8/main/cli.py | 17 ++++++++++++++++- flake8/processor.py | 4 +++- setup.py | 1 + 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index c058db5..675ad0c 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -137,6 +137,14 @@ class Manager(object): if utils.fnmatch(filename, filename_patterns) ] + def run(self): + """Run checks. + + TODO(sigmavirus24): Get rid of this + """ + for checker in self.checkers: + checker.run_checks() + def is_path_excluded(self, path): # type: (str) -> bool """Check if a path is excluded. @@ -210,6 +218,7 @@ class FileChecker(object): def run_check(self, plugin, **arguments): """Run the check in a single plugin.""" + LOG.debug('Running %r with %r', plugin, arguments) self.processor.keyword_arguments_for(plugin.parameters, arguments) return plugin.execute(**arguments) @@ -263,7 +272,7 @@ class FileChecker(object): for token in file_processor.generate_tokens(): self.check_physical_eol(token) token_type, text = token[0:2] - processor.log_token(token) + processor.log_token(LOG, token) if token_type == tokenize.OP: parens = processor.count_parentheses(parens, text) elif parens == 0: @@ -298,7 +307,7 @@ class FileChecker(object): def handle_newline(self, token_type): """Handle the logic when encountering a newline token.""" if token_type == tokenize.NEWLINE: - self.check_logical() + self.run_logical_checks() self.processor.reset_blank_before() elif len(self.processor.tokens) == 1: # The physical line contains only this token. diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 15eea8e..a2be97f 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -1,5 +1,6 @@ """Command-line implementation of flake8.""" import flake8 +from flake8 import checker from flake8 import defaults from flake8 import style_guide from flake8.options import aggregator @@ -181,10 +182,11 @@ class Application(object): self.check_plugins = None self.listening_plugins = None - self.formatting_plugigns = None + self.formatting_plugins = None self.formatter = None self.listener_trie = None self.guide = None + self.file_checker_manager = None self.options = None self.args = None @@ -242,6 +244,17 @@ class Application(object): self.options, self.listener_trie, self.formatter ) + def make_file_checker_manager(self): + # type: () -> NoneType + """Initialize our FileChecker Manager.""" + if self.file_checker_manager is None: + self.file_checker_manager = checker.Manager( + style_guide=self.guide, + arguments=self.args, + checker_plugins=self.check_plugins, + ) + self.file_checker_manager.make_checkers() + def run(self, argv=None): # type: (Union[NoneType, List[str]]) -> NoneType """Run our application.""" @@ -251,6 +264,8 @@ class Application(object): self.make_formatter() self.make_notifier() self.make_guide() + self.make_file_checker_manager() + self.file_checker_manager.run() def main(argv=None): diff --git a/flake8/processor.py b/flake8/processor.py index 062006d..88514e5 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -80,7 +80,7 @@ class FileProcessor(object): #: Total number of lines in the file self.total_lines = len(self.lines) #: Verbosity level of Flake8 - self.verbosity = options.verbosity + self.verbosity = options.verbose @contextlib.contextmanager def inside_multiline(self, line_number): @@ -302,10 +302,12 @@ def token_is_comment(token): def count_parentheses(current_parentheses_count, token_text): """Count the number of parentheses.""" + current_parentheses_count = current_parentheses_count or 0 if token_text in '([{': return current_parentheses_count + 1 elif token_text in '}])': return current_parentheses_count - 1 + return current_parentheses_count def log_token(log, token): diff --git a/setup.py b/setup.py index ec40eb6..9b66694 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ setuptools.setup( 'console_scripts': ['flake8 = flake8.main.cli:main'], 'flake8.extension': [ 'F = flake8.plugins.pyflakes:FlakesChecker', + 'pep8.tabs_or_spaces = pep8:tabs_or_spaces', ], 'flake8.report': [ 'default = flake8.formatting.default:Default', From f04d8da485751bcc5fbdfccdfa6cb369019572af Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 10 Mar 2016 19:41:19 -0600 Subject: [PATCH 184/204] Add pep8 checks to plugins list --- setup.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/setup.py b/setup.py index 9b66694..f0b9b83 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,43 @@ setuptools.setup( 'console_scripts': ['flake8 = flake8.main.cli:main'], 'flake8.extension': [ 'F = flake8.plugins.pyflakes:FlakesChecker', + # PEP-0008 checks provied by PyCQA/pycodestyle 'pep8.tabs_or_spaces = pep8:tabs_or_spaces', + 'pep8.tabs_obsolete = pep8:tabs_obsolete', + 'pep8.trailing_whitespace = pep8:trailing_whitespace', + 'pep8.trailing_blank_lines = pep8:trailing_blank_lines', + 'pep8.maximum_line_length = pep8:maximum_line_length', + 'pep8.blank_lines = pep8:blank_lines', + 'pep8.extraneous_whitespace = pep8:extraneous_whitespace', + ('pep8.whitespace_around_keywords = ' + 'pep8:whitespace_around_keywords'), + 'pep8.missing_whitespace = pep8:missing_whitespace', + 'pep8.indentation = pep8:indentation', + 'pep8.continued_indentation = pep8:continued_indentation', + ('pep8.whitespace_before_parameters = ' + 'pep8:whitespace_before_parameters'), + ('pep8.whitespace_around_operator = ' + 'pep8:whitespace_around_operator'), + ('pep8.missing_whitespace_around_operator = ' + 'pep8:missing_whitespace_around_operator'), + 'pep8.whitespace_around_comma = pep8:whitespace_around_comma', + ('pep8.whitespace_around_named_parameter_equals = ' + 'pep8:whitespace_around_named_parameter_equals'), + 'pep8.whitespace_before_comment = pep8:whitespace_before_comment', + 'pep8.imports_on_separate_lines = pep8:imports_on_separate_lines', + ('pep8.module_imports_on_top_of_file = ' + 'pep8:module_imports_on_top_of_file'), + 'pep8.compound_statements = pep8:compound_statements', + 'pep8.explicit_line_join = pep8:explicit_line_join', + ('pep8.break_around_binary_operator = ' + 'pep8:break_around_binary_operator'), + 'pep8.comparison_to_singleton = pep8:comparison_to_singleton', + 'pep8.comparison_negative = pep8:comparison_negative', + 'pep8.comparison_type = pep8:comparison_type', + 'pep8.python_3000_has_key = pep8:python_3000_has_key', + 'pep8.python_3000_raise_comma = pep8:python_3000_raise_comma', + 'pep8.python_3000_not_equal = pep8:python_3000_not_equal', + 'pep8.python_3000_backticks = pep8:python_3000_backticks', ], 'flake8.report': [ 'default = flake8.formatting.default:Default', From 960f4d6af7cae10fefd5f67f9b0886b4e73021ed Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 10 Mar 2016 19:54:53 -0600 Subject: [PATCH 185/204] Fix missing attributes for pep8 plugins --- flake8/checker.py | 2 ++ flake8/processor.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/flake8/checker.py b/flake8/checker.py index 675ad0c..9fcaa3c 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -232,6 +232,7 @@ class FileChecker(object): LOG.debug('Logical line: "%s"', logical_line.rstrip()) for plugin in self.checks.logical_line_plugins: + self.processor.update_checker_state_for(plugin) results = self.run_check(plugin, logical_line=logical_line) or () for offset, text in results: offset = find_offset(offset, mapping) @@ -248,6 +249,7 @@ class FileChecker(object): def run_physical_checks(self, physical_line): """Run all checks for a given physical line.""" for plugin in self.checks.physical_line_plugins: + self.processor.update_checker_state_for(plugin) result = self.run_check(plugin, physical_line=physical_line) if result is not None: column_offset, text = result diff --git a/flake8/processor.py b/flake8/processor.py index 88514e5..52d7ab1 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -57,6 +57,12 @@ class FileProcessor(object): self.blank_before = 0 #: Number of blank lines self.blank_lines = 0 + #: Checker states for each plugin? + self._checker_states = {} + #: Current checker state + self.checker_state = None + #: User provided option for hang closing + self.hang_closing = options.hang_closing #: Character used for indentation self.indent_char = None #: Current level of indentation @@ -80,7 +86,7 @@ class FileProcessor(object): #: Total number of lines in the file self.total_lines = len(self.lines) #: Verbosity level of Flake8 - self.verbosity = options.verbose + self.verbose = options.verbose @contextlib.contextmanager def inside_multiline(self, line_number): @@ -110,6 +116,13 @@ class FileProcessor(object): if self.blank_before < self.blank_lines: self.blank_before = self.blank_lines + def update_checker_state_for(self, plugin): + """Update the checker_state attribute for the plugin.""" + if 'checker_state' in plugin.parameters: + self.checker_state = self._checker_states.setdefault( + plugin.name, {} + ) + def next_logical_line(self): """Record the previous logical line. From d331558868e7262948d169d322f044117eb871fd Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 11 Mar 2016 21:07:40 -0600 Subject: [PATCH 186/204] Send logging output to --output-file --- flake8/main/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index a2be97f..aa4f9a7 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -178,7 +178,8 @@ class Application(object): # Set the verbosity of the program preliminary_opts, _ = self.option_manager.parse_args() - flake8.configure_logging(preliminary_opts.verbose) + flake8.configure_logging(preliminary_opts.verbose, + preliminary_opts.output_file) self.check_plugins = None self.listening_plugins = None From 4c797c9ff7300b39601943e841d98dcb3cb691e3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 11 Mar 2016 21:08:41 -0600 Subject: [PATCH 187/204] Simplify how we check if a file is excluded Our typical usage always passes is_path_excluded which checks the basename and the full path --- flake8/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8/utils.py b/flake8/utils.py index 7e08e41..1ceb76f 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -117,7 +117,7 @@ def filenames_from(arg, predicate=None): for root, sub_directories, files in os.walk(arg): for filename in files: joined = os.path.join(root, filename) - if predicate(filename) or predicate(joined): + if predicate(joined): continue yield joined # NOTE(sigmavirus24): os.walk() will skip a directory if you From 2c17b4342f8a6f24d0e034b322cefec4f7f5d5df Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 11 Mar 2016 21:26:21 -0600 Subject: [PATCH 188/204] Better log format --- flake8/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index 416c951..53e56f8 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -37,7 +37,8 @@ _VERBOSITY_TO_LOG_LEVEL = { 2: logging.DEBUG, } -LOG_FORMAT = '[flake8] %(asctime)s %(levelname)s %(message)s' +LOG_FORMAT = ('[%(name)-25s]:%(threadName)s %(relativeCreated)6d ' + '%(levelname)-8s %(message)s') def configure_logging(verbosity, filename=None, logformat=LOG_FORMAT): From daf5c4d80d576b23d4b43feee356b736a216573d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 14 Mar 2016 20:23:03 -0500 Subject: [PATCH 189/204] Add ability to check if a file is ignored inline Check for ``# flake8: noqa`` lines inside a file. --- flake8/checker.py | 3 +++ flake8/processor.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/flake8/checker.py b/flake8/checker.py index 9fcaa3c..7422427 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -291,6 +291,9 @@ class FileChecker(object): def run_checks(self): """Run checks against the file.""" + if self.processor.should_ignore_file(): + return + try: self.process_tokens() except exceptions.InvalidSyntax as exc: diff --git a/flake8/processor.py b/flake8/processor.py index 52d7ab1..f2f5b00 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -1,6 +1,7 @@ """Module containing our file processor that tokenizes a file for checks.""" import contextlib import io +import re import sys import tokenize @@ -41,6 +42,8 @@ class FileProcessor(object): - verbose """ + NOQA_FILE = re.compile(r'\s*# flake8[:=]\s*noqa', re.I) + def __init__(self, filename, options): """Initialice our file processor. @@ -269,6 +272,19 @@ class FileProcessor(object): """Read the lines from standard in.""" return utils.stdin_get_value().splitlines(True) + def should_ignore_file(self): + # type: () -> bool + """Check if ``# flake8: noqa`` is in the file to be ignored. + + :returns: + True if a line matches :attr:`FileProcessor.NOQA_FILE`, + otherwise False + :rtype: + bool + """ + ignore_file = self.NOQA_FILE.search + return any(ignore_file(line) for line in self.lines) + def strip_utf_bom(self): # type: () -> NoneType """Strip the UTF bom from the lines of the file.""" From 447a6d4fccf92e20f003473f1a32cf118d1c320d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 14 Mar 2016 20:23:20 -0500 Subject: [PATCH 190/204] Fix indentation causing incorrect logical lines --- flake8/processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake8/processor.py b/flake8/processor.py index f2f5b00..30c0e96 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -163,8 +163,8 @@ class FileProcessor(object): (previous_text not in '{[(' and text not in '}])')): text = ' ' + text - elif previous_column != start_column: - text = line[previous_column:start_column] + text + elif previous_column != start_column: + text = line[previous_column:start_column] + text logical.append(text) length += len(text) mapping.append((length, end)) From e2314c38ed464812e04be5fba3f7e2091cbc5500 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 14 Mar 2016 21:37:32 -0500 Subject: [PATCH 191/204] Add a lower level for extra verbosity --- flake8/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/flake8/__init__.py b/flake8/__init__.py index 53e56f8..40336ad 100644 --- a/flake8/__init__.py +++ b/flake8/__init__.py @@ -30,14 +30,21 @@ del NullHandler __version__ = '3.0.0a1' +# There is nothing lower than logging.DEBUG (10) in the logging library, +# but we want an extra level to avoid being too verbose when using -vv. +_EXTRA_VERBOSE = 5 +logging.addLevelName(_EXTRA_VERBOSE, 'VERBOSE') + _VERBOSITY_TO_LOG_LEVEL = { # output more than warnings but not debugging info - 1: logging.INFO, - # output debugging information and everything else - 2: logging.DEBUG, + 1: logging.INFO, # INFO is a numerical level of 20 + # output debugging information + 2: logging.DEBUG, # DEBUG is a numerical level of 10 + # output extra verbose debugging information + 3: _EXTRA_VERBOSE, } -LOG_FORMAT = ('[%(name)-25s]:%(threadName)s %(relativeCreated)6d ' +LOG_FORMAT = ('%(name)-25s %(processName)-11s %(relativeCreated)6d ' '%(levelname)-8s %(message)s') @@ -54,8 +61,8 @@ def configure_logging(verbosity, filename=None, logformat=LOG_FORMAT): """ if verbosity <= 0: return - if verbosity > 2: - verbosity = 2 + if verbosity > 3: + verbosity = 3 log_level = _VERBOSITY_TO_LOG_LEVEL[verbosity] From 0d3506b45753a3115ecb14f804464c2cd58af29a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 14 Mar 2016 21:37:50 -0500 Subject: [PATCH 192/204] Log tokens at a lower level than debug --- flake8/processor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flake8/processor.py b/flake8/processor.py index 30c0e96..9ad7751 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -5,6 +5,7 @@ import re import sys import tokenize +import flake8 from flake8 import defaults from flake8 import exceptions from flake8 import utils @@ -345,9 +346,9 @@ def log_token(log, token): pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) else: pos = 'l.%s' % token[3][0] - log.debug('l.%s\t%s\t%s\t%r' % - (token[2][0], pos, tokenize.tok_name[token[0]], - token[1])) + log.log(flake8._EXTRA_VERBOSE, 'l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], + token[1])) def expand_indent(line): From d222fcb9e118bec3b051355d12da0f6aa9cce163 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 14 Mar 2016 21:38:08 -0500 Subject: [PATCH 193/204] Correct log levels around loading plugins --- flake8/plugins/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake8/plugins/manager.py b/flake8/plugins/manager.py index c453dbe..872a4f4 100644 --- a/flake8/plugins/manager.py +++ b/flake8/plugins/manager.py @@ -97,7 +97,7 @@ class Plugin(object): Nothing """ if self._plugin is None: - LOG.debug('Loading plugin "%s" from entry-point.', self.name) + LOG.info('Loading plugin "%s" from entry-point.', self.name) try: self._load(verify_requirements) except Exception as load_exception: @@ -162,12 +162,12 @@ class PluginManager(object): # pylint: disable=too-few-public-methods self._load_all_plugins() def _load_all_plugins(self): - LOG.debug('Loading entry-points for "%s".', self.namespace) + LOG.info('Loading entry-points for "%s".', self.namespace) for entry_point in pkg_resources.iter_entry_points(self.namespace): name = entry_point.name self.plugins[name] = Plugin(name, entry_point) self.names.append(name) - LOG.info('Loaded %r for plugin "%s".', self.plugins[name], name) + LOG.debug('Loaded %r for plugin "%s".', self.plugins[name], name) def map(self, func, *args, **kwargs): r"""Call ``func`` with the plugin and \*args and \**kwargs after. From 07b9ffbeb9a9e68ce9376f56595ece772ee628cf Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 14 Mar 2016 21:38:56 -0500 Subject: [PATCH 194/204] Add naive multiprocessing support --- flake8/checker.py | 56 ++++++++++++++++++++++++++++++++++++---------- flake8/main/cli.py | 3 ++- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 7422427..0398187 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -116,12 +116,27 @@ class Manager(object): def start(self): """Start checking files.""" - pass - # for i in range(self.jobs or 0): - # proc = multiprocessing.Process(target=self.process_files) - # proc.daemon = True - # proc.start() - # self.processes.append(proc) + LOG.info('Making checkers') + self.make_checkers() + if not self.using_multiprocessing: + return + + LOG.info('Populating process queue') + for checker in self.checkers: + self.process_queue.put(checker) + + def stop(self): + """Stop checking files.""" + if not self.using_multiprocessing: + return + + LOG.info('Notifying process workers of completion') + for i in range(self.jobs or 0): + self.process_queue.put('DONE') + + LOG.info('Joining process workers') + for process in self.processes: + process.join() def make_checkers(self, paths=None): # type: (List[str]) -> NoneType @@ -137,14 +152,31 @@ class Manager(object): if utils.fnmatch(filename, filename_patterns) ] - def run(self): - """Run checks. - - TODO(sigmavirus24): Get rid of this - """ - for checker in self.checkers: + def _run_checks_from_queue(self): + LOG.info('Running checks in parallel') + for checker in iter(self.process_queue.get, 'DONE'): + LOG.debug('Running checker for file "%s"', checker.filename) checker.run_checks() + def run(self): + """Run all the checkers. + + This handles starting the process workers or just simply running all + of the checks in serial. + """ + if self.using_multiprocessing: + LOG.info('Starting process workers') + for i in range(self.jobs or 0): + proc = multiprocessing.Process( + target=self._run_checks_from_queue + ) + proc.daemon = True + proc.start() + self.processes.append(proc) + else: + for checker in self.checkers: + checker.run_checks() + def is_path_excluded(self, path): # type: (str) -> bool """Check if a path is excluded. diff --git a/flake8/main/cli.py b/flake8/main/cli.py index aa4f9a7..0dfd19a 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -254,7 +254,6 @@ class Application(object): arguments=self.args, checker_plugins=self.check_plugins, ) - self.file_checker_manager.make_checkers() def run(self, argv=None): # type: (Union[NoneType, List[str]]) -> NoneType @@ -266,7 +265,9 @@ class Application(object): self.make_notifier() self.make_guide() self.make_file_checker_manager() + self.file_checker_manager.start() self.file_checker_manager.run() + self.file_checker_manager.stop() def main(argv=None): From 3b830366b6c96ec65071eb316d69dee463d91390 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 08:40:43 -0500 Subject: [PATCH 195/204] Add a results queue --- flake8/checker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake8/checker.py b/flake8/checker.py index 0398187..274dc23 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -55,6 +55,7 @@ class Manager(object): self.checks = checker_plugins self.jobs = self._job_count() self.process_queue = None + self.results_queue = None self.using_multiprocessing = False self.processes = [] self.checkers = [] @@ -62,6 +63,7 @@ class Manager(object): if self.jobs is not None and self.jobs > 1: self.using_multiprocessing = True self.process_queue = multiprocessing.Queue() + self.results_queue = multiprocessing.Queue() def _job_count(self): # type: () -> Union[int, NoneType] From 189faf68bac204625d6e0b3f3d3ff74ba924dc84 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 08:42:13 -0500 Subject: [PATCH 196/204] Reorganize methods alphabetically Add methods to report errors to the style guide A single file works fine now but not a directory --- flake8/checker.py | 157 +++++++++++++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 57 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 274dc23..7cd6d53 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -116,68 +116,38 @@ class Manager(object): # it to an integer return int(jobs) - def start(self): - """Start checking files.""" - LOG.info('Making checkers') - self.make_checkers() - if not self.using_multiprocessing: - return + def _report_after_parallel(self): + style_guide = self.style_guide + for (filename, results) in iter(self.results_queue.get, 'DONE'): + results = sorted(results, key=lambda tup: (tup[2], tup[3])) + for (error_code, line_number, column, text) in results: + style_guide.handle_error( + code=error_code, + filename=filename, + line_number=line_number, + column_number=column, + text=text + ) - LOG.info('Populating process queue') + def _report_after_serial(self): + style_guide = self.style_guide for checker in self.checkers: - self.process_queue.put(checker) - - def stop(self): - """Stop checking files.""" - if not self.using_multiprocessing: - return - - LOG.info('Notifying process workers of completion') - for i in range(self.jobs or 0): - self.process_queue.put('DONE') - - LOG.info('Joining process workers') - for process in self.processes: - process.join() - - def make_checkers(self, paths=None): - # type: (List[str]) -> NoneType - """Create checkers for each file.""" - if paths is None: - paths = self.arguments - filename_patterns = self.options.filename - self.checkers = [ - FileChecker(filename, self.checks, self.style_guide) - for argument in paths - for filename in utils.filenames_from(argument, - self.is_path_excluded) - if utils.fnmatch(filename, filename_patterns) - ] + results = sorted(checker.results, key=lambda tup: (tup[2], tup[3])) + filename = checker.filename + for (error_code, line_number, column, text) in results: + style_guide.handle_error( + code=error_code, + filename=filename, + line_number=line_number, + column_number=column, + text=text + ) def _run_checks_from_queue(self): LOG.info('Running checks in parallel') for checker in iter(self.process_queue.get, 'DONE'): LOG.debug('Running checker for file "%s"', checker.filename) - checker.run_checks() - - def run(self): - """Run all the checkers. - - This handles starting the process workers or just simply running all - of the checks in serial. - """ - if self.using_multiprocessing: - LOG.info('Starting process workers') - for i in range(self.jobs or 0): - proc = multiprocessing.Process( - target=self._run_checks_from_queue - ) - proc.daemon = True - proc.start() - self.processes.append(proc) - else: - for checker in self.checkers: - checker.run_checks() + checker.run_checks(self.results_queue) def is_path_excluded(self, path): # type: (str) -> bool @@ -205,6 +175,76 @@ class Manager(object): '' if match else 'not ') return match + def make_checkers(self, paths=None): + # type: (List[str]) -> NoneType + """Create checkers for each file.""" + if paths is None: + paths = self.arguments + filename_patterns = self.options.filename + self.checkers = [ + FileChecker(filename, self.checks, self.style_guide) + for argument in paths + for filename in utils.filenames_from(argument, + self.is_path_excluded) + if utils.fnmatch(filename, filename_patterns) + ] + + def report(self): + """Report all of the errors found in the managed file checkers. + + This iterates over each of the checkers and reports the errors sorted + by line number. + """ + if self.using_multiprocessing: + self._report_after_parallel() + else: + self._report_after_serial() + + def run(self): + """Run all the checkers. + + This handles starting the process workers or just simply running all + of the checks in serial. + """ + if self.using_multiprocessing: + LOG.info('Starting process workers') + for i in range(self.jobs or 0): + proc = multiprocessing.Process( + target=self._run_checks_from_queue + ) + proc.daemon = True + proc.start() + self.processes.append(proc) + else: + for checker in self.checkers: + checker.run_checks() + + def start(self): + """Start checking files.""" + LOG.info('Making checkers') + self.make_checkers() + if not self.using_multiprocessing: + return + + LOG.info('Populating process queue') + for checker in self.checkers: + self.process_queue.put(checker) + + def stop(self): + """Stop checking files.""" + if not self.using_multiprocessing: + return + + LOG.info('Notifying process workers of completion') + for i in range(self.jobs or 0): + self.process_queue.put('DONE') + + LOG.info('Joining process workers') + for process in self.processes: + process.join() + LOG.info('Processes joined') + self.results_queue.put('DONE') + class FileChecker(object): """Manage running checks for a file and aggregate the results.""" @@ -246,7 +286,7 @@ class FileChecker(object): """Report an error by storing it in the results list.""" if error_code is None: error_code, text = text.split(' ', 1) - error = (error_code, self.filename, line_number, column, text) + error = (error_code, line_number, column, text) self.results.append(error) return error_code @@ -323,7 +363,7 @@ class FileChecker(object): self.run_physical_checks(file_processor.lines[-1]) self.run_logical_checks() - def run_checks(self): + def run_checks(self, results_queue): """Run checks against the file.""" if self.processor.should_ignore_file(): return @@ -334,6 +374,9 @@ class FileChecker(object): self.report(exc.error_code, exc.line_number, exc.column_number, exc.error_message) + if results_queue is not None: + results_queue.put_nowait((self.filename, self.results)) + def handle_comment(self, token, token_text): """Handle the logic when encountering a comment token.""" # The comment also ends a physical line From 6eb3dee1df547b15a257ba1a666cd04df6928298 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 08:42:30 -0500 Subject: [PATCH 197/204] Reorganize our Application flow --- flake8/main/cli.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 0dfd19a..06299ec 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -255,6 +255,18 @@ class Application(object): checker_plugins=self.check_plugins, ) + def run_checks(self): + # type: () -> NoneType + """Run the actual checks with the FileChecker Manager.""" + self.file_checker_manager.start() + self.file_checker_manager.run() + self.file_checker_manager.stop() + + def report_errors(self): + # type: () -> NoneType + """Report all the errors found by flake8 3.0.""" + self.file_checker_manager.report() + def run(self, argv=None): # type: (Union[NoneType, List[str]]) -> NoneType """Run our application.""" @@ -265,9 +277,8 @@ class Application(object): self.make_notifier() self.make_guide() self.make_file_checker_manager() - self.file_checker_manager.start() - self.file_checker_manager.run() - self.file_checker_manager.stop() + self.run_checks() + self.report_errors() def main(argv=None): From 67f9e04335d1f30d8e77c50a96d57b887f9805f3 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 11:50:27 -0500 Subject: [PATCH 198/204] Only ever return an integer for job count --- flake8/checker.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 7cd6d53..33bbf79 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -60,7 +60,7 @@ class Manager(object): self.processes = [] self.checkers = [] - if self.jobs is not None and self.jobs > 1: + if self.jobs > 1: self.using_multiprocessing = True self.process_queue = multiprocessing.Queue() self.results_queue = multiprocessing.Queue() @@ -79,28 +79,28 @@ class Manager(object): if not multiprocessing: LOG.warning('The multiprocessing module is not available. ' 'Ignoring --jobs arguments.') - return None + return 0 if utils.is_windows(): LOG.warning('The --jobs option is not available on Windows. ' 'Ignoring --jobs arguments.') - return None + return 0 if utils.is_using_stdin(self.arguments): LOG.warning('The --jobs option is not compatible with supplying ' 'input using - . Ignoring --jobs arguments.') - return None + return 0 if self.options.diff: LOG.warning('The --diff option was specified with --jobs but ' 'they are not compatible. Ignoring --jobs arguments.') - return None + return 0 jobs = self.options.jobs if jobs != 'auto' and not jobs.isdigit(): LOG.warning('"%s" is not a valid parameter to --jobs. Must be one ' 'of "auto" or a numerical value, e.g., 4.', jobs) - return None + return 0 # If the value is "auto", we want to let the multiprocessing library # decide the number based on the number of CPUs. However, if that @@ -207,8 +207,8 @@ class Manager(object): of the checks in serial. """ if self.using_multiprocessing: - LOG.info('Starting process workers') - for i in range(self.jobs or 0): + LOG.info('Starting %d process workers', self.jobs) + for i in range(self.jobs): proc = multiprocessing.Process( target=self._run_checks_from_queue ) @@ -232,11 +232,7 @@ class Manager(object): def stop(self): """Stop checking files.""" - if not self.using_multiprocessing: - return - - LOG.info('Notifying process workers of completion') - for i in range(self.jobs or 0): + for i in range(self.jobs): self.process_queue.put('DONE') LOG.info('Joining process workers') From c0659d1a8ce9c4eb6a29a34f4bbf2b0ec2d2f83a Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 11:53:12 -0500 Subject: [PATCH 199/204] Catch Keyboard interruptions in our application Add logging to our main application --- flake8/main/cli.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 06299ec..ecbdde9 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -1,4 +1,6 @@ """Command-line implementation of flake8.""" +import logging + import flake8 from flake8 import checker from flake8 import defaults @@ -7,6 +9,8 @@ from flake8.options import aggregator from flake8.options import manager from flake8.plugins import manager as plugin_manager +LOG = logging.getLogger(__name__) + def register_default_options(option_manager): """Register the default options on our OptionManager.""" @@ -267,9 +271,7 @@ class Application(object): """Report all the errors found by flake8 3.0.""" self.file_checker_manager.report() - def run(self, argv=None): - # type: (Union[NoneType, List[str]]) -> NoneType - """Run our application.""" + def _run(self, argv): self.find_plugins() self.register_plugin_options() self.parse_configuration_and_cli(argv) @@ -280,6 +282,16 @@ class Application(object): self.run_checks() self.report_errors() + def run(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Run our application.""" + try: + self._run(argv) + except KeyboardInterrupt as exc: + LOG.critical('Caught keyboard interrupt from user') + LOG.exception(exc) + self.file_checker_manager._force_cleanup() + def main(argv=None): # type: (Union[NoneType, List[str]]) -> NoneType From 19062c5e9c948fd7816561b8d6dd2f5c8b60a842 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 12:18:45 -0500 Subject: [PATCH 200/204] Wrap up multiprocessing work --- flake8/checker.py | 55 ++++++++++++++++++++++++++++++++++++---------- flake8/main/cli.py | 2 ++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 33bbf79..0d6a386 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -65,6 +65,18 @@ class Manager(object): self.process_queue = multiprocessing.Queue() self.results_queue = multiprocessing.Queue() + @staticmethod + def _cleanup_queue(q): + while not q.empty(): + q.get_nowait() + + def _force_cleanup(self): + if self.using_multiprocessing: + for proc in self.processes: + proc.join(0.2) + self._cleanup_queue(self.process_queue) + self._cleanup_queue(self.results_queue) + def _job_count(self): # type: () -> Union[int, NoneType] # First we walk through all of our error cases: @@ -116,10 +128,29 @@ class Manager(object): # it to an integer return int(jobs) + def _results(self): + seen_done = 0 + while True: + LOG.info('Retrieving results') + result = self.results_queue.get() + if result == 'DONE': + seen_done += 1 + if seen_done >= self.jobs: + break + continue + + yield result + def _report_after_parallel(self): style_guide = self.style_guide - for (filename, results) in iter(self.results_queue.get, 'DONE'): - results = sorted(results, key=lambda tup: (tup[2], tup[3])) + final_results = {} + for (filename, results) in self._results(): + final_results[filename] = results + + for checker in self.checkers: + filename = checker.filename + results = sorted(final_results[filename], + key=lambda tup: (tup[1], tup[2])) for (error_code, line_number, column, text) in results: style_guide.handle_error( code=error_code, @@ -148,6 +179,7 @@ class Manager(object): for checker in iter(self.process_queue.get, 'DONE'): LOG.debug('Running checker for file "%s"', checker.filename) checker.run_checks(self.results_queue) + self.results_queue.put('DONE') def is_path_excluded(self, path): # type: (str) -> bool @@ -195,9 +227,7 @@ class Manager(object): This iterates over each of the checkers and reports the errors sorted by line number. """ - if self.using_multiprocessing: - self._report_after_parallel() - else: + if not self.using_multiprocessing: self._report_after_serial() def run(self): @@ -215,9 +245,13 @@ class Manager(object): proc.daemon = True proc.start() self.processes.append(proc) + proc = multiprocessing.Process(target=self._report_after_parallel) + proc.start() + LOG.info('Started process to report errors') + self.processes.append(proc) else: for checker in self.checkers: - checker.run_checks() + checker.run_checks(self.results_queue) def start(self): """Start checking files.""" @@ -235,11 +269,8 @@ class Manager(object): for i in range(self.jobs): self.process_queue.put('DONE') - LOG.info('Joining process workers') - for process in self.processes: - process.join() - LOG.info('Processes joined') - self.results_queue.put('DONE') + for proc in self.processes: + proc.join() class FileChecker(object): @@ -371,7 +402,7 @@ class FileChecker(object): exc.error_message) if results_queue is not None: - results_queue.put_nowait((self.filename, self.results)) + results_queue.put((self.filename, self.results)) def handle_comment(self, token, token_text): """Handle the logic when encountering a comment token.""" diff --git a/flake8/main/cli.py b/flake8/main/cli.py index ecbdde9..5bf6144 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -264,11 +264,13 @@ class Application(object): """Run the actual checks with the FileChecker Manager.""" self.file_checker_manager.start() self.file_checker_manager.run() + LOG.info('Finished running') self.file_checker_manager.stop() def report_errors(self): # type: () -> NoneType """Report all the errors found by flake8 3.0.""" + LOG.info('Reporting errors') self.file_checker_manager.report() def _run(self, argv): From 666e1c2b06151b7f6f2d290d82aa38b13bafb033 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 12:27:05 -0500 Subject: [PATCH 201/204] Ensure the logical line is tracked on the processor --- flake8/processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake8/processor.py b/flake8/processor.py index 9ad7751..d75256b 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -175,7 +175,8 @@ class FileProcessor(object): def build_logical_line(self): """Build a logical line from the current tokens list.""" comments, logical, mapping_list = self.build_logical_line_tokens() - return ''.join(comments), ''.join(logical), mapping_list + self.logical_line = ''.join(logical) + return ''.join(comments), self.logical_line, mapping_list def split_line(self, token): """Split a physical line's line based on new-lines. From 576d1f6c85b321f4a9897a386af8815a7ca779d4 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 14:10:05 -0500 Subject: [PATCH 202/204] Bypass linecache by recording physical line --- flake8/checker.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 0d6a386..c449a9b 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -151,13 +151,14 @@ class Manager(object): filename = checker.filename results = sorted(final_results[filename], key=lambda tup: (tup[1], tup[2])) - for (error_code, line_number, column, text) in results: + for (error_code, line_number, column, text, line) in results: style_guide.handle_error( code=error_code, filename=filename, line_number=line_number, column_number=column, - text=text + text=text, + physical_line=line, ) def _report_after_serial(self): @@ -165,13 +166,14 @@ class Manager(object): for checker in self.checkers: results = sorted(checker.results, key=lambda tup: (tup[2], tup[3])) filename = checker.filename - for (error_code, line_number, column, text) in results: + for (error_code, line_number, column, text, line) in results: style_guide.handle_error( code=error_code, filename=filename, line_number=line_number, column_number=column, - text=text + text=text, + physical_line=line, ) def _run_checks_from_queue(self): @@ -313,7 +315,8 @@ class FileChecker(object): """Report an error by storing it in the results list.""" if error_code is None: error_code, text = text.split(' ', 1) - error = (error_code, line_number, column, text) + physical_line = self.processor.line_for(line_number) + error = (error_code, line_number, column, text, physical_line) self.results.append(error) return error_code From 0b063a10249558ede919bcd7f67c6aa563ba74ab Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 14:10:20 -0500 Subject: [PATCH 203/204] Run checks expecting an AST --- flake8/checker.py | 33 +++++++++++++++++++++++++++++++++ flake8/main/cli.py | 3 +++ flake8/processor.py | 9 +++++++++ flake8/style_guide.py | 5 +++-- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index c449a9b..905cade 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -326,6 +326,37 @@ class FileChecker(object): self.processor.keyword_arguments_for(plugin.parameters, arguments) return plugin.execute(**arguments) + def run_ast_checks(self): + """Run all checks expecting an abstract syntax tree.""" + try: + ast = self.processor.build_ast() + except (ValueError, SyntaxError, TypeError): + (exc_type, exception) = sys.exc_info()[:2] + if len(exception.args) > 1: + offset = exception.args[1] + if len(offset) > 2: + offset = offset[1:3] + else: + offset = (1, 0) + + self.report('E999', offset[0], offset[1], '%s: %s' % + (exc_type.__name__, exception.args[0])) + return + + for plugin in self.checks.ast_plugins: + checker = self.run_check(plugin, tree=ast) + # NOTE(sigmavirus24): If we want to allow for AST plugins that are + # not classes exclusively, we can do the following: + # retrieve_results = getattr(checker, 'run', lambda: checker) + # Otherwise, we just call run on the checker + for (line_number, offset, text, check) in checker.run(): + self.report( + error_code=None, + line_number=line_number, + column=offset, + text=text, + ) + def run_logical_checks(self): """Run all checks expecting a logical line.""" comments, logical_line, mapping = self.processor.build_logical_line() @@ -404,6 +435,8 @@ class FileChecker(object): self.report(exc.error_code, exc.line_number, exc.column_number, exc.error_message) + self.run_ast_checks() + if results_queue is not None: results_queue.put((self.filename, self.results)) diff --git a/flake8/main/cli.py b/flake8/main/cli.py index 5bf6144..dbd9603 100644 --- a/flake8/main/cli.py +++ b/flake8/main/cli.py @@ -227,6 +227,9 @@ class Application(object): self.option_manager, argv ) + self.check_plugins.provide_options(self.option_manager, self.options, + self.args) + def make_formatter(self): # type: () -> NoneType """Initialize a formatter based on the parsed options.""" diff --git a/flake8/processor.py b/flake8/processor.py index d75256b..1dc27a1 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -10,6 +10,7 @@ from flake8 import defaults from flake8 import exceptions from flake8 import utils +PyCF_ONLY_AST = 1024 NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) # Work around Python < 2.6 behaviour, which does not generate NL after # a comment which is on a line by itself. @@ -172,6 +173,10 @@ class FileProcessor(object): (previous_row, previous_column) = end return comments, logical, mapping + def build_ast(self): + """Build an abstract syntax tree from the list of lines.""" + return compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + def build_logical_line(self): """Build a logical line from the current tokens list.""" comments, logical, mapping_list = self.build_logical_line_tokens() @@ -223,6 +228,10 @@ class FileProcessor(object): except tokenize.TokenError as exc: raise exceptions.InvalidSyntax(exc.message, exception=exc) + def line_for(self, line_number): + """Retrieve the physical line at the specified line number.""" + return self.lines[line_number - 1] + def next_line(self): """Get the next line from the list.""" if self.line_number >= self.total_lines: diff --git a/flake8/style_guide.py b/flake8/style_guide.py index 51c3a8e..2c18c9d 100644 --- a/flake8/style_guide.py +++ b/flake8/style_guide.py @@ -187,12 +187,13 @@ class StyleGuide(object): error, codes_str) return False - def handle_error(self, code, filename, line_number, column_number, text): + def handle_error(self, code, filename, line_number, column_number, text, + physical_line=None): # type: (str, str, int, int, str) -> NoneType """Handle an error reported by a check.""" error = Error(code, filename, line_number, column_number, text) if (self.should_report_error(error.code) is Decision.Selected and - self.is_inline_ignored(error) is False): + self.is_inline_ignored(error, physical_line) is False): self.formatter.handle(error) self.listener.notify(error.code, error) From ee18ac981e3dbaf3365a817663834c7b547f83cb Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 15 Mar 2016 14:39:43 -0500 Subject: [PATCH 204/204] Handle case where file was ignored --- flake8/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8/checker.py b/flake8/checker.py index 905cade..becde97 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -149,7 +149,7 @@ class Manager(object): for checker in self.checkers: filename = checker.filename - results = sorted(final_results[filename], + results = sorted(final_results.get(filename, []), key=lambda tup: (tup[1], tup[2])) for (error_code, line_number, column, text, line) in results: style_guide.handle_error(