From bc5e7f2d72818d1ffc6ed019cf1ddf93086a454e Mon Sep 17 00:00:00 2001 From: Lucas Cimon Date: Thu, 18 Aug 2016 16:39:06 +0200 Subject: [PATCH 1/4] trailing-whitespace hook: restoring original file in case of failure - fixes #134 --- pre_commit_hooks/trailing_whitespace_fixer.py | 22 ++++++++++++------- tests/trailing_whitespace_fixer_test.py | 10 +++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index c159071..22c41ba 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -9,7 +9,7 @@ from pre_commit_hooks.util import cmd_output def _fix_file(filename, markdown=False): - for line in fileinput.input([filename], inplace=True): + for line in fileinput.input([filename], inplace=True, backup='.bak'): # preserve trailing two-space for non-blank lines in markdown files if markdown and (not line.isspace()) and (line.endswith(" \n")): line = line.rstrip(' \n') @@ -64,14 +64,20 @@ def fix_trailing_whitespace(argv=None): .format(ext) ) - if bad_whitespace_files: - for bad_whitespace_file in bad_whitespace_files: - print('Fixing {0}'.format(bad_whitespace_file)) - _, extension = os.path.splitext(bad_whitespace_file.lower()) + return_code = 0 + for bad_whitespace_file in bad_whitespace_files: + print('Fixing {0}'.format(bad_whitespace_file)) + _, extension = os.path.splitext(bad_whitespace_file.lower()) + try: _fix_file(bad_whitespace_file, all_markdown or extension in md_exts) - return 1 - else: - return 0 + return_code = 1 + # pylint: disable=broad-except + except Exception as error: # pragma: no cover + # e.g. error can be a UnicodeDecodeError in Python 3 + print('Ignoring {} that caused a {}'.format(bad_whitespace_file, error.__class__)) + os.remove(bad_whitespace_file) + os.rename(bad_whitespace_file + '.bak', bad_whitespace_file) + return return_code if __name__ == '__main__': diff --git a/tests/trailing_whitespace_fixer_test.py b/tests/trailing_whitespace_fixer_test.py index 6f4fdfd..78e6e73 100644 --- a/tests/trailing_whitespace_fixer_test.py +++ b/tests/trailing_whitespace_fixer_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import sys + import pytest from pre_commit_hooks.trailing_whitespace_fixer import fix_trailing_whitespace @@ -103,3 +105,11 @@ def test_no_markdown_linebreak_ext_opt(filename, input_s, output, tmpdir): def test_returns_zero_for_no_changes(): assert fix_trailing_whitespace([__file__]) == 0 + + +def test_preserve_non_utf8_file(tmpdir): + path = tmpdir.join('file.txt') + path.write_binary(b'\xe9 \n') + ret = fix_trailing_whitespace([path.strpath]) + assert ret == (1 if sys.version_info[0] < 3 else 0) # a UnicodeDecodeError is only triggered in Python 3 + assert path.size() > 0 From eaad923dd4c56d0ab1919948e634cd7b4fbf3807 Mon Sep 17 00:00:00 2001 From: Lucas Cimon Date: Fri, 19 Aug 2016 22:40:15 +0100 Subject: [PATCH 2/4] trailing-whitespace hook: Switching from using fileinput to a tempfile and whitespace substitution in binary mode --- pre_commit_hooks/trailing_whitespace_fixer.py | 38 +++++++++---------- tests/trailing_whitespace_fixer_test.py | 9 ++--- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 22c41ba..e39a119 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -1,24 +1,29 @@ from __future__ import print_function import argparse -import fileinput import os import sys +import tempfile from pre_commit_hooks.util import cmd_output def _fix_file(filename, markdown=False): - for line in fileinput.input([filename], inplace=True, backup='.bak'): - # preserve trailing two-space for non-blank lines in markdown files - if markdown and (not line.isspace()) and (line.endswith(" \n")): - line = line.rstrip(' \n') - # only preserve if there are no trailing tabs or unusual whitespace - if not line[-1].isspace(): - print(line + " ") - continue - - print(line.rstrip()) + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + with open(filename, 'rb') as original_file: + for line in original_file.readlines(): + # preserve trailing two-space for non-blank lines in markdown files + if markdown and (not line.isspace()) and line.endswith(b' \n'): + line = line.rstrip(b' \n') # restricted stripping: e.g. \t are not stripped + # only preserve if there are no trailing tabs or unusual whitespace + if not line[-1:].isspace(): + tmp_file.write(line + b' \n') + else: + tmp_file.write(line.rstrip() + b'\n') + else: + tmp_file.write(line.rstrip() + b'\n') + os.remove(filename) + os.rename(tmp_file.name, filename) def fix_trailing_whitespace(argv=None): @@ -68,15 +73,8 @@ def fix_trailing_whitespace(argv=None): for bad_whitespace_file in bad_whitespace_files: print('Fixing {0}'.format(bad_whitespace_file)) _, extension = os.path.splitext(bad_whitespace_file.lower()) - try: - _fix_file(bad_whitespace_file, all_markdown or extension in md_exts) - return_code = 1 - # pylint: disable=broad-except - except Exception as error: # pragma: no cover - # e.g. error can be a UnicodeDecodeError in Python 3 - print('Ignoring {} that caused a {}'.format(bad_whitespace_file, error.__class__)) - os.remove(bad_whitespace_file) - os.rename(bad_whitespace_file + '.bak', bad_whitespace_file) + _fix_file(bad_whitespace_file, all_markdown or extension in md_exts) + return_code = 1 return return_code diff --git a/tests/trailing_whitespace_fixer_test.py b/tests/trailing_whitespace_fixer_test.py index 78e6e73..3498edd 100644 --- a/tests/trailing_whitespace_fixer_test.py +++ b/tests/trailing_whitespace_fixer_test.py @@ -1,8 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -import sys - import pytest from pre_commit_hooks.trailing_whitespace_fixer import fix_trailing_whitespace @@ -108,8 +106,9 @@ def test_returns_zero_for_no_changes(): def test_preserve_non_utf8_file(tmpdir): + non_utf8_bytes_content = b'\xe9 \n\n' path = tmpdir.join('file.txt') - path.write_binary(b'\xe9 \n') + path.write_binary(non_utf8_bytes_content) ret = fix_trailing_whitespace([path.strpath]) - assert ret == (1 if sys.version_info[0] < 3 else 0) # a UnicodeDecodeError is only triggered in Python 3 - assert path.size() > 0 + assert ret == 1 + assert path.size() == (len(non_utf8_bytes_content) - 1) From c3c870c398e871171203897f592d7c15d03ca9f7 Mon Sep 17 00:00:00 2001 From: Lucas Cimon Date: Tue, 30 Aug 2016 10:34:06 +0100 Subject: [PATCH 3/4] trailing-whitespace hook: support for CRLFs --- pre_commit_hooks/trailing_whitespace_fixer.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index e39a119..7ac5479 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -9,19 +9,18 @@ from pre_commit_hooks.util import cmd_output def _fix_file(filename, markdown=False): - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - with open(filename, 'rb') as original_file: + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as tmp_file: + with open(filename, mode='rb') as original_file: for line in original_file.readlines(): # preserve trailing two-space for non-blank lines in markdown files - if markdown and (not line.isspace()) and line.endswith(b' \n'): - line = line.rstrip(b' \n') # restricted stripping: e.g. \t are not stripped + eol = b'\r\n' if line[-2:] == b'\r\n' else b'\n' + if markdown and (not line.isspace()) and line.endswith(b' ' + eol): + line = line.rstrip(b' \r\n') # restricted stripping: e.g. \t are not stripped # only preserve if there are no trailing tabs or unusual whitespace if not line[-1:].isspace(): - tmp_file.write(line + b' \n') - else: - tmp_file.write(line.rstrip() + b'\n') - else: - tmp_file.write(line.rstrip() + b'\n') + tmp_file.write(line + b' ' + eol) + continue + tmp_file.write(line.rstrip() + eol) os.remove(filename) os.rename(tmp_file.name, filename) From cb23c48b0dcf7d88dac2cecaec272fd6b0775525 Mon Sep 17 00:00:00 2001 From: Lucas Cimon Date: Wed, 31 Aug 2016 02:02:33 +0100 Subject: [PATCH 4/4] Post-review fixes --- pre_commit_hooks/trailing_whitespace_fixer.py | 31 +++++++++---------- tests/trailing_whitespace_fixer_test.py | 2 +- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 7ac5479..fa9b7dd 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -3,26 +3,25 @@ from __future__ import print_function import argparse import os import sys -import tempfile from pre_commit_hooks.util import cmd_output -def _fix_file(filename, markdown=False): - with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as tmp_file: - with open(filename, mode='rb') as original_file: - for line in original_file.readlines(): - # preserve trailing two-space for non-blank lines in markdown files - eol = b'\r\n' if line[-2:] == b'\r\n' else b'\n' - if markdown and (not line.isspace()) and line.endswith(b' ' + eol): - line = line.rstrip(b' \r\n') # restricted stripping: e.g. \t are not stripped - # only preserve if there are no trailing tabs or unusual whitespace - if not line[-1:].isspace(): - tmp_file.write(line + b' ' + eol) - continue - tmp_file.write(line.rstrip() + eol) - os.remove(filename) - os.rename(tmp_file.name, filename) +def _fix_file(filename, is_markdown): + with open(filename, mode='rb') as file_processed: + lines = file_processed.readlines() + lines = [_process_line(line, is_markdown) for line in lines] + with open(filename, mode='wb') as file_processed: + for line in lines: + file_processed.write(line) + + +def _process_line(line, is_markdown): + # preserve trailing two-space for non-blank lines in markdown files + eol = b'\r\n' if line[-2:] == b'\r\n' else b'\n' + if is_markdown and (not line.isspace()) and line.endswith(b' ' + eol): + return line.rstrip() + b' ' + eol + return line.rstrip() + eol def fix_trailing_whitespace(argv=None): diff --git a/tests/trailing_whitespace_fixer_test.py b/tests/trailing_whitespace_fixer_test.py index 3498edd..3a72ccb 100644 --- a/tests/trailing_whitespace_fixer_test.py +++ b/tests/trailing_whitespace_fixer_test.py @@ -42,7 +42,7 @@ def test_fixes_trailing_markdown_whitespace(filename, input_s, output, tmpdir): MD_TESTS_2 = ( ('foo.txt', 'foo \nbar \n \n', 'foo \nbar\n\n'), ('bar.Markdown', 'bar \nbaz\t\n\t\n', 'bar \nbaz\n\n'), - ('bar.MD', 'bar \nbaz\t \n\t\n', 'bar \nbaz\n\n'), + ('bar.MD', 'bar \nbaz\t \n\t\n', 'bar \nbaz \n\n'), ('.txt', 'baz \nquux \t\n\t\n', 'baz\nquux\n\n'), ('txt', 'foo \nbaz \n\t\n', 'foo\nbaz\n\n'), )