Refactor processor and file checker some more

flake8/checker.py

@@ -223,16 +223,19 @@ class FileChecker(object):
         LOG.debug('Logical line: "%s"', logical_line.rstrip())
 
         for plugin in self.checks.logical_line_plugins:
-            result = self.run_check(plugin, logical_line=logical_line)
+            results = self.run_check(plugin, logical_line=logical_line) or ()
-            if result is not None:
+            for offset, text in results:
-                column_offset, text = result
+                offset = find_offset(offset, mapping)
+                line_number, column_offset = offset
                 self.report(
                     error_code=None,
-                    line_number=self.processor.line_number,
+                    line_number=line_number,
                     column=column_offset,
                     text=text,
                 )
 
+        self.processor.next_logical_line()
+
     def run_physical_checks(self, physical_line):
         """Run all checks for a given physical line."""
         for plugin in self.checks.physical_line_plugins:
@@ -325,3 +328,14 @@ class FileChecker(object):
             with self.processor.inside_multiline(line_number=line_no):
                 for line in self.processor.split_line(token):
                     self.run_physical_checks(line + '\n')
+
+
+def find_offset(offset, mapping):
+    """Find the offset tuple for a single offset."""
+    if isinstance(offset, tuple):
+        return offset
+
+    for token_offset, position in mapping:
+        if offset <= token_offset:
+            break
+    return (position[0], position[1] + offset - token_offset)

flake8/processor.py

@@ -102,6 +102,25 @@ class FileProcessor(object):
         """Note that we visited a new blank line."""
         self.blank_lines += 1
 
+    def update_state(self, mapping):
+        """Update the indent level based on the logical line mapping."""
+        (start_row, start_col) = mapping[0][1]
+        start_line = self.lines[start_row - 1]
+        self.indent_level = expand_indent(start_line[:start_col])
+        if self.blank_before < self.blank_lines:
+            self.blank_before = self.blank_lines
+
+    def next_logical_line(self):
+        """Record the previous logical line.
+
+        This also resets the tokens list and the blank_lines count.
+        """
+        if self.logical_line:
+            self.previous_indent_level = self.indent_level
+            self.previous_logical = self.logical_line
+        self.blank_lines = 0
+        self.tokens = []
+
     def build_logical_line_tokens(self):
         """Build the mapping, comments, and logical line lists."""
         logical = []
@@ -117,7 +136,7 @@ class FileProcessor(object):
                 comments.append(text)
                 continue
             if token_type == tokenize.STRING:
-                text = utils.mutate_string(text)
+                text = mutate_string(text)
             if previous_row:
                 (start_row, start_column) = start
                 if previous_row != start_row:
@@ -298,3 +317,50 @@ def log_token(log, token):
     log.debug('l.%s\t%s\t%s\t%r' %
               (token[2][0], pos, tokenize.tok_name[token[0]],
                   token[1]))
+
+
+def expand_indent(line):
+    r"""Return the amount of indentation.
+
+    Tabs are expanded to the next multiple of 8.
+
+    >>> expand_indent('    ')
+    4
+    >>> expand_indent('\t')
+    8
+    >>> expand_indent('       \t')
+    8
+    >>> expand_indent('        \t')
+    16
+    """
+    if '\t' not in line:
+        return len(line) - len(line.lstrip())
+    result = 0
+    for char in line:
+        if char == '\t':
+            result = result // 8 * 8 + 8
+        elif char == ' ':
+            result += 1
+        else:
+            break
+    return result
+
+
+def mutate_string(text):
+    """Replace contents with 'xxx' to prevent syntax matching.
+
+    >>> mute_string('"abc"')
+    '"xxx"'
+    >>> mute_string("'''abc'''")
+    "'''xxx'''"
+    >>> mute_string("r'abc'")
+    "r'xxx'"
+    """
+    # String modifiers (e.g. u or r)
+    start = text.index(text[-1]) + 1
+    end = len(text) - 1
+    # Triple quotes
+    if text[-3:] in ('"""', "'''"):
+        start += 2
+        end -= 2
+    return text[:start] + 'x' * (end - start) + text[end:]

flake8/utils.py

@@ -4,7 +4,6 @@ import inspect
 import io
 import os
 import sys
-import tokenize
 
 
 def parse_comma_separated_list(value):
@@ -184,62 +183,3 @@ def parameters_for(plugin):
         parameters.remove('self')
 
     return parameters
-
-NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE])
-# Work around Python < 2.6 behaviour, which does not generate NL after
-# a comment which is on a line by itself.
-COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n'
-
-
-def is_eol_token(token):
-    """Check if the token is an end-of-line token."""
-    return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n'
-
-if COMMENT_WITH_NL:  # If on Python 2.6
-    def is_eol_token(token, _is_eol_token=is_eol_token):
-        """Check if the token is an end-of-line token."""
-        return (_is_eol_token(token) or
-                (token[0] == tokenize.COMMENT and token[1] == token[4]))
-
-
-def is_multiline_string(token):
-    """Check if this is a multiline string."""
-    return token[0] == tokenize.STRING and '\n' in token[1]
-
-
-def token_is_newline(token):
-    """Check if the token type is a newline token type."""
-    return token[0] in NEWLINE
-
-
-def token_is_comment(token):
-    """Check if the token type is a comment."""
-    return COMMENT_WITH_NL and token[0] == tokenize.COMMENT
-
-
-def count_parentheses(current_parentheses_count, token_text):
-    """Count the number of parentheses."""
-    if token_text in '([{':
-        return current_parentheses_count + 1
-    elif token_text in '}])':
-        return current_parentheses_count - 1
-
-
-def mutate_string(text):
-    """Replace contents with 'xxx' to prevent syntax matching.
-
-    >>> mute_string('"abc"')
-    '"xxx"'
-    >>> mute_string("'''abc'''")
-    "'''xxx'''"
-    >>> mute_string("r'abc'")
-    "r'xxx'"
-    """
-    # String modifiers (e.g. u or r)
-    start = text.index(text[-1]) + 1
-    end = len(text) - 1
-    # Triple quotes
-    if text[-3:] in ('"""', "'''"):
-        start += 2
-        end -= 2
-    return text[:start] + 'x' * (end - start) + text[end:]