From 36fb688f97d69d1ba0e1ea7bd6502ec7fe8b3141 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 4 Mar 2016 22:51:22 -0600 Subject: [PATCH] Refactor processor and file checker some more --- flake8/checker.py | 22 ++++++++++++--- flake8/processor.py | 68 ++++++++++++++++++++++++++++++++++++++++++++- flake8/utils.py | 60 --------------------------------------- 3 files changed, 85 insertions(+), 65 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 69bab17..c058db5 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -223,16 +223,19 @@ class FileChecker(object): LOG.debug('Logical line: "%s"', logical_line.rstrip()) for plugin in self.checks.logical_line_plugins: - result = self.run_check(plugin, logical_line=logical_line) - if result is not None: - column_offset, text = result + results = self.run_check(plugin, logical_line=logical_line) or () + for offset, text in results: + offset = find_offset(offset, mapping) + line_number, column_offset = offset self.report( error_code=None, - line_number=self.processor.line_number, + line_number=line_number, column=column_offset, text=text, ) + self.processor.next_logical_line() + def run_physical_checks(self, physical_line): """Run all checks for a given physical line.""" for plugin in self.checks.physical_line_plugins: @@ -325,3 +328,14 @@ class FileChecker(object): with self.processor.inside_multiline(line_number=line_no): for line in self.processor.split_line(token): self.run_physical_checks(line + '\n') + + +def find_offset(offset, mapping): + """Find the offset tuple for a single offset.""" + if isinstance(offset, tuple): + return offset + + for token_offset, position in mapping: + if offset <= token_offset: + break + return (position[0], position[1] + offset - token_offset) diff --git a/flake8/processor.py b/flake8/processor.py index e681c03..062006d 100644 --- a/flake8/processor.py +++ b/flake8/processor.py @@ -102,6 +102,25 @@ class FileProcessor(object): """Note that we visited a new blank line.""" self.blank_lines += 1 + def update_state(self, mapping): + """Update the indent level based on the logical line mapping.""" + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + + def next_logical_line(self): + """Record the previous logical line. + + This also resets the tokens list and the blank_lines count. + """ + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + self.blank_lines = 0 + self.tokens = [] + def build_logical_line_tokens(self): """Build the mapping, comments, and logical line lists.""" logical = [] @@ -117,7 +136,7 @@ class FileProcessor(object): comments.append(text) continue if token_type == tokenize.STRING: - text = utils.mutate_string(text) + text = mutate_string(text) if previous_row: (start_row, start_column) = start if previous_row != start_row: @@ -298,3 +317,50 @@ def log_token(log, token): log.debug('l.%s\t%s\t%s\t%r' % (token[2][0], pos, tokenize.tok_name[token[0]], token[1])) + + +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +def mutate_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] diff --git a/flake8/utils.py b/flake8/utils.py index fe024d1..7e08e41 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -4,7 +4,6 @@ import inspect import io import os import sys -import tokenize def parse_comma_separated_list(value): @@ -184,62 +183,3 @@ def parameters_for(plugin): parameters.remove('self') return parameters - -NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) -# Work around Python < 2.6 behaviour, which does not generate NL after -# a comment which is on a line by itself. -COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' - - -def is_eol_token(token): - """Check if the token is an end-of-line token.""" - return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' - -if COMMENT_WITH_NL: # If on Python 2.6 - def is_eol_token(token, _is_eol_token=is_eol_token): - """Check if the token is an end-of-line token.""" - return (_is_eol_token(token) or - (token[0] == tokenize.COMMENT and token[1] == token[4])) - - -def is_multiline_string(token): - """Check if this is a multiline string.""" - return token[0] == tokenize.STRING and '\n' in token[1] - - -def token_is_newline(token): - """Check if the token type is a newline token type.""" - return token[0] in NEWLINE - - -def token_is_comment(token): - """Check if the token type is a comment.""" - return COMMENT_WITH_NL and token[0] == tokenize.COMMENT - - -def count_parentheses(current_parentheses_count, token_text): - """Count the number of parentheses.""" - if token_text in '([{': - return current_parentheses_count + 1 - elif token_text in '}])': - return current_parentheses_count - 1 - - -def mutate_string(text): - """Replace contents with 'xxx' to prevent syntax matching. - - >>> mute_string('"abc"') - '"xxx"' - >>> mute_string("'''abc'''") - "'''xxx'''" - >>> mute_string("r'abc'") - "r'xxx'" - """ - # String modifiers (e.g. u or r) - start = text.index(text[-1]) + 1 - end = len(text) - 1 - # Triple quotes - if text[-3:] in ('"""', "'''"): - start += 2 - end -= 2 - return text[:start] + 'x' * (end - start) + text[end:]