From 20b8c5962ece6143c85fe9ba959fb6cd6ae843a5 Mon Sep 17 00:00:00 2001 From: Florent Xicluna Date: Wed, 13 Feb 2013 19:18:49 +0100 Subject: [PATCH] Refactor to use extensions --- flake8/engine.py | 56 +++++++++++++++++ flake8/main.py | 131 ++++++++++---------------------------- flake8/util.py | 160 ++++++++++++----------------------------------- 3 files changed, 130 insertions(+), 217 deletions(-) create mode 100644 flake8/engine.py diff --git a/flake8/engine.py b/flake8/engine.py new file mode 100644 index 0000000..332c8c8 --- /dev/null +++ b/flake8/engine.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +import pep8 + +from flake8 import __version__ +from flake8.util import OrderedSet + + +def _register_extensions(): + """Register all the extensions.""" + extensions = OrderedSet() + extensions.add(('pep8', pep8.__version__)) + parser_hooks = [] + options_hooks = [] + try: + from pkg_resources import iter_entry_points + except ImportError: + pass + else: + for entry in iter_entry_points('flake8.extension'): + checker = entry.load() + pep8.register_check(checker, codes=[entry.name]) + extensions.add((checker.name, checker.version)) + if hasattr(checker, 'add_options'): + parser_hooks.append(checker.add_options) + if hasattr(checker, 'parse_options'): + options_hooks.append(checker.parse_options) + return extensions, parser_hooks, options_hooks + + +def get_parser(): + (extensions, parser_hooks, options_hooks) = _register_extensions() + details = ', '.join(['%s: %s' % ext for ext in extensions]) + parser = pep8.get_parser('flake8', '%s (%s)' % (__version__, details)) + for opt in ('--repeat', '--testsuite', '--doctest'): + try: + parser.remove_option(opt) + except ValueError: + pass + parser.add_option('--exit-zero', action='store_true', + help="exit with code 0 even if there are errors") + for parser_hook in parser_hooks: + parser_hook(parser) + parser.add_option('--install-hook', default=False, action='store_true', + help='Install the appropriate hook for this ' + 'repository.', dest='install_hook') + return parser, options_hooks + + +def get_style_guide(**kwargs): + """Parse the options and configure the checker.""" + kwargs['parser'], options_hooks = get_parser() + styleguide = pep8.StyleGuide(**kwargs) + options = styleguide.options + for options_hook in options_hooks: + options_hook(options) + return styleguide diff --git a/flake8/main.py b/flake8/main.py index 38e712d..4f8e120 100644 --- a/flake8/main.py +++ b/flake8/main.py @@ -1,91 +1,58 @@ import os import sys -import pep8 -import pyflakes.api -import pyflakes.checker import select -from flake8 import mccabe -from flake8.util import skip_file, get_parser, Flake8Reporter + +from flake8.engine import get_style_guide pep8style = None if sys.platform.startswith('win'): - pep8.DEFAULT_CONFIG = os.path.expanduser(r'~\.flake8') + DEFAULT_CONFIG = os.path.expanduser(r'~\.flake8') else: - pep8.DEFAULT_CONFIG = os.path.join( + DEFAULT_CONFIG = os.path.join( os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), 'flake8' ) -PROJECT_CONFIG = ['.flake8'] -PROJECT_CONFIG.extend(pep8.PROJECT_CONFIG) -pep8.PROJECT_CONFIG = tuple(PROJECT_CONFIG) - def main(): - global pep8style - # parse out our flags so pep8 doesn't get confused - parser = get_parser() + """Parse options and run checks on Python source.""" + # Prepare + flake8_style = get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG) + options = flake8_style.options - # We then have to re-parse argv to make sure pep8 is properly initialized - pep8style = pep8.StyleGuide(parse_argv=True, config_file=True, - parser=parser) - opts = pep8style.options - - if opts.install_hook: + if options.install_hook: from flake8.hooks import install_hook install_hook() - warnings = 0 - stdin = None + # Run the checkers + report = flake8_style.check_files() - complexity = opts.max_complexity - builtins = set(opts.builtins.split(',')) - if builtins: - orig_builtins = set(pyflakes.checker._MAGIC_GLOBALS) - pyflakes.checker._MAGIC_GLOBALS = orig_builtins | builtins - - # This is needed so we can ignore some items - pyflakes_reporter = Flake8Reporter(opts.ignore) - - if pep8style.paths and opts.filename is not None: - for path in _get_python_files(pep8style.paths): - if path == '-': - if stdin is None: - stdin = read_stdin() - warnings += check_code(stdin, opts.ignore, complexity) - else: - warnings += check_file(path, opts.ignore, complexity, - pyflakes_reporter) - else: - stdin = read_stdin() - warnings += check_code(stdin, opts.ignore, complexity, - pyflakes_reporter) - - if opts.exit_zero: - raise SystemExit(0) - - raise SystemExit(warnings > 0) + # Print the final report + options = flake8_style.options + if options.statistics: + report.print_statistics() + if options.benchmark: + report.print_benchmark() + if report.total_errors: + if options.count: + sys.stderr.write(str(report.total_errors) + '\n') + if not options.exit_zero: + raise SystemExit(1) def check_file(path, ignore=(), complexity=-1, reporter=None): - if pep8style.excluded(path): - return 0 - warnings = pyflakes.api.checkPath(path, reporter) - warnings -= reporter.ignored_warnings - warnings += pep8style.input_file(path) - if complexity > -1: - warnings += mccabe.get_module_complexity(path, complexity) - return warnings + flake8_style = get_style_guide( + config_file=DEFAULT_CONFIG, + ignore=ignore, max_complexity=complexity, reporter=reporter) + return flake8_style.input_file(path) def check_code(code, ignore=(), complexity=-1, reporter=None): - warnings = pyflakes.api.check(code, '', reporter) - warnings -= reporter.ignored_warnings - warnings += pep8style.input_file('-', lines=code.split('\n')) - if complexity > -1: - warnings += mccabe.get_code_complexity(code, complexity) - return warnings + flake8_style = get_style_guide( + config_file=DEFAULT_CONFIG, + ignore=ignore, max_complexity=complexity, reporter=reporter) + return flake8_style.input_file('-', lines=code.split('\n')) def read_stdin(): @@ -123,37 +90,7 @@ else: yield "%s.py" % filename def run(self): - # XXX the setuptools command is currently broken - # _initpep8() - - # _get_python_files can produce the same file several - # times, if one of its paths is a parent of another. Keep - # a set of checked files to de-duplicate. - checked = set() - - warnings = 0 - for path in _get_python_files(self.distribution_files()): - if path not in checked: - warnings += check_file(path) - checked.add(path) - - raise SystemExit(warnings > 0) - - -def _get_python_files(paths): - for path in paths: - if os.path.isdir(path): - for dirpath, dirnames, filenames in os.walk(path): - if pep8style.excluded(dirpath): - dirnames[:] = [] - continue - for filename in filenames: - if not filename.endswith('.py'): - continue - fullpath = os.path.join(dirpath, filename) - if not skip_file(fullpath) or pep8style.excluded(fullpath): - yield fullpath - - else: - if not skip_file(path) or pep8style.excluded(path): - yield path + flake8_style = get_style_guide(config_file=DEFAULT_CONFIG) + paths = self.distribution_files() + report = flake8_style.check_files(paths) + raise SystemExit(report.get_file_results() > 0) diff --git a/flake8/util.py b/flake8/util.py index 23ee811..3354274 100644 --- a/flake8/util.py +++ b/flake8/util.py @@ -1,73 +1,45 @@ +# -*- coding: utf-8 -*- from __future__ import with_statement +import os.path import re -import os -import sys -from io import StringIO -import pep8 -import pyflakes -from pyflakes import reporter, messages -pep8style = None +__all__ = ['ast', 'iter_child_nodes', 'OrderedSet', 'skip_file'] + +try: + import ast + iter_child_nodes = ast.iter_child_nodes +except ImportError: # Python 2.5 + import _ast as ast + + if 'decorator_list' not in ast.ClassDef._fields: + # Patch the missing attribute 'decorator_list' + ast.ClassDef.decorator_list = () + ast.FunctionDef.decorator_list = property(lambda s: s.decorators) + + def iter_child_nodes(node): + """ + Yield all direct child nodes of *node*, that is, all fields that + are nodes and all items of fields that are lists of nodes. + """ + if not node._fields: + return + for name in node._fields: + field = getattr(node, name, None) + if isinstance(field, ast.AST): + yield field + elif isinstance(field, list): + for item in field: + if isinstance(item, ast.AST): + yield item -def get_parser(): - """Create a custom OptionParser""" - from flake8 import __version__ - parser = pep8.get_parser() - - def version(option, opt, value, parser): - parser.print_usage() - parser.print_version() - sys.exit(0) - - parser.version = '{0} (pep8: {1}, pyflakes: {2})'.format( - __version__, pep8.__version__, pyflakes.__version__) - parser.remove_option('--version') - parser.add_option('--builtins', default='', dest='builtins', - help="append builtin functions to pyflakes' " - "_MAGIC_BUILTINS") - parser.add_option('--exit-zero', action='store_true', default=False, - help='Exit with status 0 even if there are errors') - parser.add_option('--max-complexity', default=-1, action='store', - type='int', help='McCabe complexity threshold') - parser.add_option('--install-hook', default=False, action='store_true', - help='Install the appropriate hook for this ' - 'repository.', dest='install_hook') - # don't overlap with pep8's verbose option - parser.add_option('-V', '--version', action='callback', - callback=version, - help='Print the version info for flake8') - parser.prog = os.path.basename(sys.argv[0]) - return parser - - -def skip_warning(warning, ignore=[]): - # XXX quick dirty hack, just need to keep the line in the warning - if not hasattr(warning, 'message') or ignore is None: - # McCabe's warnings cannot be skipped afaik, and they're all strings. - # And we'll get a TypeError otherwise - return False - if warning.message.split()[0] in ignore: - return True - if not os.path.isfile(warning.filename): - return False - - # XXX should cache the file in memory - with open(warning.filename) as f: - line = f.readlines()[warning.lineno - 1] - - return skip_line(line) - - -def skip_line(line): - def _noqa(line): - return line.strip().lower().endswith('# noqa') - skip = _noqa(line) - if not skip: - i = line.rfind(' #') - skip = _noqa(line[:i]) if i > 0 else False - return skip +class OrderedSet(list): + """List without duplicates.""" + __slots__ = () + def add(self, value): + if value not in self: + self.append(value) _NOQA = re.compile(r'flake8[:=]\s*noqa', re.I | re.M) @@ -78,61 +50,9 @@ def skip_file(path, source=None): # flake8: noqa """ if os.path.isfile(path): - f = open(path) - elif source: - f = StringIO(source) - else: + with open(path) as f: + source = f.read() + elif not source: return False - try: - content = f.read() - finally: - f.close() - return _NOQA.search(content) is not None - - -error_mapping = { - 'W402': (messages.UnusedImport,), - 'W403': (messages.ImportShadowedByLoopVar,), - 'W404': (messages.ImportStarUsed,), - 'W405': (messages.LateFutureImport,), - 'W801': (messages.RedefinedWhileUnused, - messages.RedefinedInListComp,), - 'W802': (messages.UndefinedName,), - 'W803': (messages.UndefinedExport,), - 'W804': (messages.UndefinedLocal, - messages.UnusedVariable,), - 'W805': (messages.DuplicateArgument,), - 'W806': (messages.Redefined,), -} - - -class Flake8Reporter(reporter.Reporter): - """Our own instance of a Reporter so that we can silence some messages.""" - class_mapping = dict((k, c) for (c, v) in error_mapping.items() for k in v) - - def __init__(self, ignore=None): - super(Flake8Reporter, self).__init__(sys.stdout, sys.stderr) - self.ignore = ignore or [] - self.ignored_warnings = 0 - - def flake(self, message): - classes = [error_mapping[i] for i in self.ignore if i in error_mapping] - - if (any(isinstance(message, c) for c in classes) or - skip_warning(message)): - self.ignored_warnings += 1 - return - m = self.to_str(message) - i = m.rfind(':') + 1 - message = '{0} {1}{2}'.format( - m[:i], self.class_mapping[message.__class__], m[i:] - ) - - super(Flake8Reporter, self).flake(message) - - def to_str(self, message): - try: - return unicode(message) - except NameError: - return str(message) + return _NOQA.search(source) is not None