From 886dfc42053f63ce64f382f3637f56919889979b Mon Sep 17 00:00:00 2001 From: iconmaster5326 Date: Fri, 25 Oct 2019 11:12:49 -0400 Subject: [PATCH 1/7] trailing-whitespace: add option for custom chars to strip --- pre_commit_hooks/trailing_whitespace_fixer.py | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 2ccc003..88b3f93 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -7,10 +7,15 @@ from typing import Optional from typing import Sequence -def _fix_file(filename, is_markdown): # type: (str, bool) -> bool +def _fix_file(filename, is_markdown, chars_to_strip): + # type: (str, bool, Optional[bytes]) -> bool with open(filename, mode='rb') as file_processed: lines = file_processed.readlines() - newlines = [_process_line(line, is_markdown) for line in lines] + newlines = [ + _process_line(line, is_markdown, chars_to_strip) + for line + in lines + ] if newlines != lines: with open(filename, mode='wb') as file_processed: for line in newlines: @@ -20,7 +25,8 @@ def _fix_file(filename, is_markdown): # type: (str, bool) -> bool return False -def _process_line(line, is_markdown): # type: (bytes, bool) -> bytes +def _process_line(line, is_markdown, chars_to_strip): + # type: (bytes, bool, Optional[bytes]) -> bytes if line[-2:] == b'\r\n': eol = b'\r\n' elif line[-1:] == b'\n': @@ -29,8 +35,8 @@ def _process_line(line, is_markdown): # type: (bytes, bool) -> bytes eol = b'' # preserve trailing two-space for non-blank lines in markdown files if is_markdown and (not line.isspace()) and line.endswith(b' ' + eol): - return line.rstrip() + b' ' + eol - return line.rstrip() + eol + return line.rstrip(chars_to_strip) + b' ' + eol + return line.rstrip(chars_to_strip) + eol def main(argv=None): # type: (Optional[Sequence[str]]) -> int @@ -50,6 +56,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int 'default: %(default)s' ), ) + parser.add_argument( + '--chars', + help='The set of characters to strip from the end of lines. ' + 'Defaults to all whitespace characters.', + ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -78,7 +89,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int for filename in args.filenames: _, extension = os.path.splitext(filename.lower()) md = all_markdown or extension in md_exts - if _fix_file(filename, md): + if _fix_file( + filename, + md, + None if args.chars is None else bytes(args.chars, 'utf-8'), + ): print('Fixing {}'.format(filename)) return_code = 1 return return_code From 08663c91f2362aa70fe0b5e9dc389823a46c66bf Mon Sep 17 00:00:00 2001 From: iconmaster5326 Date: Fri, 25 Oct 2019 11:15:48 -0400 Subject: [PATCH 2/7] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 584b53d..e579976 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ Add this to your `.pre-commit-config.yaml` use `args: [--markdown-linebreak-ext=md]` (or other extensions used by your markdownfiles). If for some reason you want to treat all files as markdown, use `--markdown-linebreak-ext=*`. + - By default, this hook trims all whitespace from the ends of lines. + To specify a custom set of characters to trim instead, use `args: [--chars,""]`. ### Deprecated / replaced hooks From a2f836a23b095c19744fb5a73a3584cffa90bf8d Mon Sep 17 00:00:00 2001 From: iconmaster5326 Date: Fri, 25 Oct 2019 11:34:26 -0400 Subject: [PATCH 3/7] fix-whitespace: Added test for custom charsets --- pre_commit_hooks/trailing_whitespace_fixer.py | 6 ++++-- tests/trailing_whitespace_fixer_test.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 88b3f93..70ab382 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -29,13 +29,15 @@ def _process_line(line, is_markdown, chars_to_strip): # type: (bytes, bool, Optional[bytes]) -> bytes if line[-2:] == b'\r\n': eol = b'\r\n' + line = line[:-2] elif line[-1:] == b'\n': eol = b'\n' + line = line[:-1] else: eol = b'' # preserve trailing two-space for non-blank lines in markdown files - if is_markdown and (not line.isspace()) and line.endswith(b' ' + eol): - return line.rstrip(chars_to_strip) + b' ' + eol + if is_markdown and (not line.isspace()) and line.endswith(b' '): + return line[:-2].rstrip(chars_to_strip) + b' ' + eol return line.rstrip(chars_to_strip) + eol diff --git a/tests/trailing_whitespace_fixer_test.py b/tests/trailing_whitespace_fixer_test.py index 82c9b6d..ec4c918 100644 --- a/tests/trailing_whitespace_fixer_test.py +++ b/tests/trailing_whitespace_fixer_test.py @@ -78,3 +78,19 @@ def test_preserve_non_utf8_file(tmpdir): ret = main([path.strpath]) assert ret == 1 assert path.size() == (len(non_utf8_bytes_content) - 1) + + +def test_custom_charset_change(tmpdir): + # strip spaces only, no tabs + path = tmpdir.join('file.txt') + path.write('\ta \t \n') + ret = main([path.strpath, '--chars', ' ']) + assert ret == 1 + assert path.read() == '\ta \t\n' + + +def test_custom_charset_no_change(tmpdir): + path = tmpdir.join('file.txt') + path.write('\ta \t\n') + ret = main([path.strpath, '--chars', ' ']) + assert ret == 0 From c431f09ac30a903d74e47257bc3d1561a22f06e2 Mon Sep 17 00:00:00 2001 From: iconmaster5326 Date: Fri, 25 Oct 2019 11:54:00 -0400 Subject: [PATCH 4/7] Fix failure on Python 2 --- pre_commit_hooks/trailing_whitespace_fixer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 70ab382..6063684 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -94,7 +94,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int if _fix_file( filename, md, - None if args.chars is None else bytes(args.chars, 'utf-8'), + None if args.chars is None else bytes(args.chars.encode('utf-8')), ): print('Fixing {}'.format(filename)) return_code = 1 From dcbf43489c86a47ce0ec731c15613f7daf551181 Mon Sep 17 00:00:00 2001 From: Iconmaster Date: Fri, 25 Oct 2019 12:15:11 -0400 Subject: [PATCH 5/7] Apply suggestion: the `bytes(...)` call does nothing here --- pre_commit_hooks/trailing_whitespace_fixer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 6063684..b74b753 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -94,7 +94,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int if _fix_file( filename, md, - None if args.chars is None else bytes(args.chars.encode('utf-8')), + None if args.chars is None else args.chars.encode('utf-8'), ): print('Fixing {}'.format(filename)) return_code = 1 From a33a8f0d8aa87a1090427091182644dca880cb21 Mon Sep 17 00:00:00 2001 From: iconmaster5326 Date: Fri, 25 Oct 2019 12:20:04 -0400 Subject: [PATCH 6/7] Change parameter name to "chars" and move encoding outside loop --- pre_commit_hooks/trailing_whitespace_fixer.py | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index b74b753..a21b54f 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -7,15 +7,11 @@ from typing import Optional from typing import Sequence -def _fix_file(filename, is_markdown, chars_to_strip): +def _fix_file(filename, is_markdown, chars): # type: (str, bool, Optional[bytes]) -> bool with open(filename, mode='rb') as file_processed: lines = file_processed.readlines() - newlines = [ - _process_line(line, is_markdown, chars_to_strip) - for line - in lines - ] + newlines = [_process_line(line, is_markdown, chars) for line in lines] if newlines != lines: with open(filename, mode='wb') as file_processed: for line in newlines: @@ -25,7 +21,7 @@ def _fix_file(filename, is_markdown, chars_to_strip): return False -def _process_line(line, is_markdown, chars_to_strip): +def _process_line(line, is_markdown, chars): # type: (bytes, bool, Optional[bytes]) -> bytes if line[-2:] == b'\r\n': eol = b'\r\n' @@ -37,8 +33,8 @@ def _process_line(line, is_markdown, chars_to_strip): eol = b'' # preserve trailing two-space for non-blank lines in markdown files if is_markdown and (not line.isspace()) and line.endswith(b' '): - return line[:-2].rstrip(chars_to_strip) + b' ' + eol - return line.rstrip(chars_to_strip) + eol + return line[:-2].rstrip(chars) + b' ' + eol + return line.rstrip(chars) + eol def main(argv=None): # type: (Optional[Sequence[str]]) -> int @@ -60,8 +56,10 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int ) parser.add_argument( '--chars', - help='The set of characters to strip from the end of lines. ' - 'Defaults to all whitespace characters.', + help=( + 'The set of characters to strip from the end of lines. ' + 'Defaults to all whitespace characters.' + ), ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -86,16 +84,12 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int " (probably filename; use '--markdown-linebreak-ext=EXT')" .format(ext), ) - + chars = None if args.chars is None else args.chars.encode('utf-8') return_code = 0 for filename in args.filenames: _, extension = os.path.splitext(filename.lower()) md = all_markdown or extension in md_exts - if _fix_file( - filename, - md, - None if args.chars is None else args.chars.encode('utf-8'), - ): + if _fix_file(filename, md, chars): print('Fixing {}'.format(filename)) return_code = 1 return return_code From 0114962a7492186863997724384f6a77f09179f1 Mon Sep 17 00:00:00 2001 From: iconmaster5326 Date: Fri, 25 Oct 2019 12:28:50 -0400 Subject: [PATCH 7/7] Added test for --markdown-linebreak-ext and --chars together --- tests/trailing_whitespace_fixer_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/trailing_whitespace_fixer_test.py b/tests/trailing_whitespace_fixer_test.py index ec4c918..97f9aef 100644 --- a/tests/trailing_whitespace_fixer_test.py +++ b/tests/trailing_whitespace_fixer_test.py @@ -94,3 +94,11 @@ def test_custom_charset_no_change(tmpdir): path.write('\ta \t\n') ret = main([path.strpath, '--chars', ' ']) assert ret == 0 + + +def test_markdown_with_custom_charset(tmpdir): + path = tmpdir.join('file.md') + path.write('\ta \t \n') + ret = main([path.strpath, '--chars', ' ', '--markdown-linebreak-ext', '*']) + assert ret == 1 + assert path.read() == '\ta \t \n'