From 56d45435f40ea708eb8fd135e6b3de143a772b76 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 13 Jun 2017 21:38:14 +0200 Subject: [PATCH 01/42] Add mixed-line-ending test case --- testing/resources/mixed_line_ending.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 testing/resources/mixed_line_ending.txt diff --git a/testing/resources/mixed_line_ending.txt b/testing/resources/mixed_line_ending.txt new file mode 100644 index 0000000..a4d8dc0 --- /dev/null +++ b/testing/resources/mixed_line_ending.txt @@ -0,0 +1,11 @@ +This line ends with 'LF' +This line ends with 'CRLF' +This line ends with 'LF' +This line ends with 'CRLF' +This line ends with 'LF' +This line ends with 'CRLF' +This line ends with 'LF' +This line ends with 'CRLF' +This line ends with 'LF' +This line ends with 'CRLF' +This line ends with 'LF' From 3d4fb41d8a3912b8973066a2b16132c9d172d7d5 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 13 Jun 2017 21:37:03 +0200 Subject: [PATCH 02/42] Add mixed-line-ending processor --- pre_commit_hooks/mixed_line_ending.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 pre_commit_hooks/mixed_line_ending.py diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py new file mode 100644 index 0000000..612cdc9 --- /dev/null +++ b/pre_commit_hooks/mixed_line_ending.py @@ -0,0 +1,27 @@ +import argparse +import sys + + +def mixed_line_ending(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument( + '-f', + '--fix', + choices=['auto', 'crlf', 'lf', 'no'], + default='auto', + help='Replace line ending with the specified. Default is "auto"', + nargs=1) + parser.add_argument( + '-v', + '--verbose', + action="store_true", + help='Increase output verbosity') + args = parser.parse_args(argv) + + print(args.fix) + + return 0 + + +if __name__ == '__main__': + sys.exit(mixed_line_ending()) From 16b7c7af5d4ca5d7befe8d9a586c9e8c76bfa662 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 15 Jun 2017 20:29:28 +0200 Subject: [PATCH 03/42] Use Enum to list line ending types in mixed_line_ending --- pre_commit_hooks/mixed_line_ending.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 612cdc9..bc863be 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -1,13 +1,25 @@ import argparse import sys +from enum import Enum + + +class LineEnding(Enum): + CRLF = '\r\n', '\\r\\n', 'crlf' + LF = '\n', '\\n', 'lf' + + def __init__(self, str, strPrint, optName): + self.str = str + self.strPrint = strPrint + self.optName = optName + def mixed_line_ending(argv=None): parser = argparse.ArgumentParser() parser.add_argument( '-f', '--fix', - choices=['auto', 'crlf', 'lf', 'no'], + choices=['auto', 'no'] + [m.optName for m in LineEnding], default='auto', help='Replace line ending with the specified. Default is "auto"', nargs=1) From afaa97cd11ab58c88bf74fdaa8ddd3eb109ad740 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sat, 17 Jun 2017 19:45:39 +0200 Subject: [PATCH 04/42] Add enum to mixed_line_ending `--fix` option --- pre_commit_hooks/mixed_line_ending.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index bc863be..7b7cabe 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -4,22 +4,34 @@ import sys from enum import Enum -class LineEnding(Enum): +class CLIOption(Enum): + def __init__(self, optName): + self.optName = optName + + +class LineEnding(CLIOption): CRLF = '\r\n', '\\r\\n', 'crlf' LF = '\n', '\\n', 'lf' - def __init__(self, str, strPrint, optName): - self.str = str + def __init__(self, string, strPrint, optName): + self.string = string self.strPrint = strPrint self.optName = optName +class MixedLineEndingOption(CLIOption): + AUTO = 'auto' + NO = 'no' + CRLF = LineEnding.CRLF.optName + LF = LineEnding.LF.optName + + def mixed_line_ending(argv=None): parser = argparse.ArgumentParser() parser.add_argument( '-f', '--fix', - choices=['auto', 'no'] + [m.optName for m in LineEnding], + choices=[m.optName for m in MixedLineEndingOption], default='auto', help='Replace line ending with the specified. Default is "auto"', nargs=1) From 51866649a6119fb8d473153233311168ccfc6057 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sun, 18 Jun 2017 12:23:39 +0200 Subject: [PATCH 05/42] Split argument parsing from main mixed line ending function --- pre_commit_hooks/mixed_line_ending.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 7b7cabe..946ecf6 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -27,6 +27,14 @@ class MixedLineEndingOption(CLIOption): def mixed_line_ending(argv=None): + args = _parse_arguments(argv) + + print(args.fix) + + return 0 + + +def _parse_arguments(argv=None): parser = argparse.ArgumentParser() parser.add_argument( '-f', @@ -42,9 +50,7 @@ def mixed_line_ending(argv=None): help='Increase output verbosity') args = parser.parse_args(argv) - print(args.fix) - - return 0 + return args if __name__ == '__main__': From 93194b9c6a517564c27d45eb2960d9d6d0580535 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sun, 18 Jun 2017 12:31:22 +0200 Subject: [PATCH 06/42] Change --fix option from mixed_line_ending To make it easier to use in the program (single string instead of a list of strings) --- pre_commit_hooks/mixed_line_ending.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 946ecf6..323708b 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -41,8 +41,7 @@ def _parse_arguments(argv=None): '--fix', choices=[m.optName for m in MixedLineEndingOption], default='auto', - help='Replace line ending with the specified. Default is "auto"', - nargs=1) + help='Replace line ending with the specified. Default is "auto"') parser.add_argument( '-v', '--verbose', From b2b0d5929ac65f96b19b68579a46b8925c845fd2 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 26 Jun 2017 21:31:44 +0200 Subject: [PATCH 07/42] Use enum instead of raw argparse result --- pre_commit_hooks/mixed_line_ending.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 323708b..6de9a4e 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -27,9 +27,9 @@ class MixedLineEndingOption(CLIOption): def mixed_line_ending(argv=None): - args = _parse_arguments(argv) + options = _parse_arguments(argv) - print(args.fix) + print(options) return 0 @@ -49,7 +49,19 @@ def _parse_arguments(argv=None): help='Increase output verbosity') args = parser.parse_args(argv) - return args + fix = None + if args.fix == 'auto': + fix = MixedLineEndingOption.AUTO + elif args.fix == 'no': + fix = MixedLineEndingOption.NO + elif args.fix == 'crlf': + fix = MixedLineEndingOption.CRLF + elif args.fix == 'lf': + fix = MixedLineEndingOption.LF + + options = {'fix': fix, 'verbose': args.verbose} + + return options if __name__ == '__main__': From ad0062a3bb8c6bed18d7ea86da41bf9415030423 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 26 Jun 2017 21:42:27 +0200 Subject: [PATCH 08/42] Add filenames option --- pre_commit_hooks/mixed_line_ending.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 6de9a4e..0710cf8 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -42,6 +42,7 @@ def _parse_arguments(argv=None): choices=[m.optName for m in MixedLineEndingOption], default='auto', help='Replace line ending with the specified. Default is "auto"') + parser.add_argument('filenames', nargs='*', help='Filenames to fix') parser.add_argument( '-v', '--verbose', @@ -59,7 +60,7 @@ def _parse_arguments(argv=None): elif args.fix == 'lf': fix = MixedLineEndingOption.LF - options = {'fix': fix, 'verbose': args.verbose} + options = {'fix': fix, 'filenames': args.filenames, 'verbose': args.verbose} return options From 466f9e173285bbdc90379d8878d50a753582505d Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 26 Jun 2017 23:06:57 +0200 Subject: [PATCH 09/42] Add line ending detection --- pre_commit_hooks/mixed_line_ending.py | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 0710cf8..7b6a4bf 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -1,4 +1,6 @@ import argparse +import os +import re import sys from enum import Enum @@ -10,6 +12,7 @@ class CLIOption(Enum): class LineEnding(CLIOption): + CR = '\r', '\\r', 'cr' CRLF = '\r\n', '\\r\\n', 'crlf' LF = '\n', '\\n', 'lf' @@ -26,11 +29,33 @@ class MixedLineEndingOption(CLIOption): LF = LineEnding.LF.optName +class MixedLineDetection(Enum): + MIXED_MOSTLY_CRLF = True, LineEnding.CRLF.string + MIXED_MOSTLY_LF = True, LineEnding.LF.string + NOT_MIXED = False, None + UNKNOWN = False, None + + def __init__(self, conversion, line_ending_char): + self.conversion = conversion + self.line_ending_char = line_ending_char + + +# Matches CRLF +CRLF_PATTERN = re.compile(LineEnding.CRLF.string, re.DOTALL) +# Matches LF (without preceding CR) +LF_PATTERN = re.compile('(? 0 + lf_found = lf_nb > 0 + + if crlf_nb == lf_nb: + return MixedLineDetection.UNKNOWN + + if crlf_found ^ lf_found: + return MixedLineDetection.NOT_MIXED + + if crlf_nb > lf_nb: + return MixedLineDetection.MIXED_MOSTLY_CRLF + else: + return MixedLineDetection.MIXED_MOSTLY_LF + + if __name__ == '__main__': sys.exit(mixed_line_ending()) From aaf134c2bc41760715222ccb933b356550f4b063 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Wed, 28 Jun 2017 22:15:31 +0200 Subject: [PATCH 10/42] Add line ending conversion --- pre_commit_hooks/mixed_line_ending.py | 49 +++++++++++++++++++++------ 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 7b6a4bf..c4b2a6e 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -12,14 +12,15 @@ class CLIOption(Enum): class LineEnding(CLIOption): - CR = '\r', '\\r', 'cr' - CRLF = '\r\n', '\\r\\n', 'crlf' - LF = '\n', '\\n', 'lf' + CR = '\r', '\\r', 'cr', re.compile(r'\r', re.DOTALL) + CRLF = '\r\n', '\\r\\n', 'crlf', re.compile(r'\r\n', re.DOTALL) + LF = '\n', '\\n', 'lf', re.compile(r'(? 0 lf_found = lf_nb > 0 @@ -118,5 +130,20 @@ def _detect_line_ending(filename): return MixedLineDetection.MIXED_MOSTLY_LF +def _convert_line_ending(filename, line_ending): + # read the file + fin = open(filename, 'r') + bufin = fin.read() + fin.close() + + # convert line ending + bufout = ANY_LINE_ENDING_PATTERN.sub(line_ending, bufin) + + # write the result in the file + fout = open(filename, 'w') + fout.write(bufout) + fout.close() + + if __name__ == '__main__': sys.exit(mixed_line_ending()) From 0a8b929f075afe8707b04a319090bf76272d8d3a Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 29 Jun 2017 21:16:17 +0200 Subject: [PATCH 11/42] Change files according to --fix option --- pre_commit_hooks/mixed_line_ending.py | 33 +++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index c4b2a6e..b116f50 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -24,10 +24,14 @@ class LineEnding(CLIOption): class MixedLineEndingOption(CLIOption): - AUTO = 'auto' - NO = 'no' - CRLF = LineEnding.CRLF.optName - LF = LineEnding.LF.optName + AUTO = 'auto', None + NO = 'no', None + CRLF = LineEnding.CRLF.optName, LineEnding.CRLF + LF = LineEnding.LF.optName, LineEnding.LF + + def __init__(self, opt_name, line_ending_enum): + self.opt_name = opt_name + self.line_ending_enum = line_ending_enum class MixedLineDetection(Enum): @@ -61,11 +65,22 @@ def mixed_line_ending(argv=None): _check_filenames(options['filenames']) - for filename in options['filenames']: - detect_result = _detect_line_ending(filename) + fix_option = options['fix'] - if detect_result.conversion: - _convert_line_ending(filename, detect_result.line_ending_char) + if fix_option == MixedLineEndingOption.NO: + pass + elif fix_option == MixedLineEndingOption.AUTO: + for filename in options['filenames']: + detect_result = _detect_line_ending(filename) + + if detect_result.conversion: + _convert_line_ending(filename, detect_result.line_ending_char) + # when a line ending character is forced with --fix option + else: + line_ending_enum = fix_option.line_ending_enum + + for filename in options['filenames']: + _convert_line_ending(filename, line_ending_enum.string) return 0 @@ -75,7 +90,7 @@ def _parse_arguments(argv=None): parser.add_argument( '-f', '--fix', - choices=[m.optName for m in MixedLineEndingOption], + choices=[m.opt_name for m in MixedLineEndingOption], default='auto', help='Replace line ending with the specified. Default is "auto"') parser.add_argument('filenames', nargs='*', help='Filenames to fix') From 2b28f4f0514626b9a0eaca4b309b5a53147003c5 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 29 Jun 2017 21:42:19 +0200 Subject: [PATCH 12/42] Rename variable names --- pre_commit_hooks/mixed_line_ending.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index b116f50..59d78ed 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -6,28 +6,23 @@ import sys from enum import Enum -class CLIOption(Enum): - def __init__(self, optName): - self.optName = optName - - -class LineEnding(CLIOption): +class LineEnding(Enum): CR = '\r', '\\r', 'cr', re.compile(r'\r', re.DOTALL) CRLF = '\r\n', '\\r\\n', 'crlf', re.compile(r'\r\n', re.DOTALL) LF = '\n', '\\n', 'lf', re.compile(r'(? Date: Thu, 29 Jun 2017 21:46:25 +0200 Subject: [PATCH 13/42] Reuse variable definition --- pre_commit_hooks/mixed_line_ending.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 59d78ed..b55f5f5 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -86,7 +86,7 @@ def _parse_arguments(argv=None): '-f', '--fix', choices=[m.opt_name for m in MixedLineEndingOption], - default='auto', + default=MixedLineEndingOption.AUTO.opt_name, help='Replace line ending with the specified. Default is "auto"') parser.add_argument('filenames', nargs='*', help='Filenames to fix') parser.add_argument( @@ -97,13 +97,13 @@ def _parse_arguments(argv=None): args = parser.parse_args(argv) fix = None - if args.fix == 'auto': + if args.fix == MixedLineEndingOption.AUTO.opt_name: fix = MixedLineEndingOption.AUTO - elif args.fix == 'no': + elif args.fix == MixedLineEndingOption.NO.opt_name: fix = MixedLineEndingOption.NO - elif args.fix == 'crlf': + elif args.fix == MixedLineEndingOption.CRLF.opt_name: fix = MixedLineEndingOption.CRLF - elif args.fix == 'lf': + elif args.fix == MixedLineEndingOption.LF.opt_name: fix = MixedLineEndingOption.LF options = {'fix': fix, 'filenames': args.filenames, 'verbose': args.verbose} From f477582ae6af18d30ba667b41510aaeb401c7fb4 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 29 Jun 2017 23:20:59 +0200 Subject: [PATCH 14/42] Add logs to mixed_line_ending.py --- pre_commit_hooks/mixed_line_ending.py | 55 +++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index b55f5f5..c389898 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -1,4 +1,5 @@ import argparse +import logging import os import re import sys @@ -30,14 +31,21 @@ class MixedLineEndingOption(Enum): class MixedLineDetection(Enum): - MIXED_MOSTLY_CRLF = True, LineEnding.CRLF.string - MIXED_MOSTLY_LF = True, LineEnding.LF.string + MIXED_MOSTLY_CRLF = True, LineEnding.CRLF + MIXED_MOSTLY_LF = True, LineEnding.LF NOT_MIXED = False, None UNKNOWN = False, None - def __init__(self, conversion, line_ending_char): + def __init__(self, conversion, line_ending_enum): self.conversion = conversion - self.line_ending_char = line_ending_char + self.line_ending_enum = line_ending_enum + + +VERBOSE_OPTION_TO_LOGGING_SEVERITY = { + 0: logging.WARNING, + 1: logging.INFO, + 2: logging.DEBUG, +} ANY_LINE_ENDING_PATTERN = re.compile( @@ -56,24 +64,44 @@ ANY_LINE_ENDING_PATTERN = re.compile( def mixed_line_ending(argv=None): options = _parse_arguments(argv) - print(options) + logging.basicConfig(format='%(levelname)s: %(message)s', + level=options['logging_severity']) + logging.debug('mixed_line_ending: options = %s', options) _check_filenames(options['filenames']) fix_option = options['fix'] if fix_option == MixedLineEndingOption.NO: + logging.info('No conversion asked') pass elif fix_option == MixedLineEndingOption.AUTO: for filename in options['filenames']: detect_result = _detect_line_ending(filename) + logging.debug('mixed_line_ending: detect_result = %s', + detect_result) + if detect_result.conversion: - _convert_line_ending(filename, detect_result.line_ending_char) + le_enum = detect_result.line_ending_enum + + logging.info('The file %s has mixed line ending with a ' + 'majority of "%s". Converting to "%s"', filename, + le_enum.str_print, le_enum.str_print) + + _convert_line_ending(filename, le_enum.string) + elif detect_result == MixedLineDetection.NOT_MIXED: + logging.info('The file %s has no mixed line ending', filename) + elif detect_result == MixedLineDetection.UNKNOWN: + logging.info('Could not define most frequent line ending in ' + 'file %s. File skiped.', filename) + # when a line ending character is forced with --fix option else: line_ending_enum = fix_option.line_ending_enum + logging.info('Force line ending to "%s"', line_ending_enum.str_print) + for filename in options['filenames']: _convert_line_ending(filename, line_ending_enum.string) @@ -92,7 +120,8 @@ def _parse_arguments(argv=None): parser.add_argument( '-v', '--verbose', - action="store_true", + action="count", + default=0, help='Increase output verbosity') args = parser.parse_args(argv) @@ -106,12 +135,18 @@ def _parse_arguments(argv=None): elif args.fix == MixedLineEndingOption.LF.opt_name: fix = MixedLineEndingOption.LF - options = {'fix': fix, 'filenames': args.filenames, 'verbose': args.verbose} + args.verbose = min(args.verbose, 2) + severity = VERBOSE_OPTION_TO_LOGGING_SEVERITY.get(args.verbose) + + options = {'fix': fix, 'filenames': args.filenames, + 'logging_severity': severity} return options def _check_filenames(filenames): + logging.debug('_check_filenames: filenames = %s', filenames) + for filename in filenames: if not os.path.isfile(filename): raise IOError('The file "{}" does not exist'.format(filename)) @@ -128,6 +163,10 @@ def _detect_line_ending(filename): crlf_found = crlf_nb > 0 lf_found = lf_nb > 0 + logging.debug('_detect_line_ending: crlf_nb = %d, lf_nb = %d, ' + 'crlf_found = %s, lf_found = %s', + crlf_nb, lf_nb, crlf_found, lf_found) + if crlf_nb == lf_nb: return MixedLineDetection.UNKNOWN From b1294b86144f3e95744d681cdc92971dce53c5e9 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 3 Jul 2017 19:57:43 +0200 Subject: [PATCH 15/42] Add unit test for mixed_line_ending --- pre_commit_hooks/mixed_line_ending.py | 11 ++++++++++- tests/mixed_line_ending_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/mixed_line_ending_test.py diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index c389898..ddc25eb 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -74,7 +74,8 @@ def mixed_line_ending(argv=None): if fix_option == MixedLineEndingOption.NO: logging.info('No conversion asked') - pass + + return 0 elif fix_option == MixedLineEndingOption.AUTO: for filename in options['filenames']: detect_result = _detect_line_ending(filename) @@ -90,12 +91,18 @@ def mixed_line_ending(argv=None): le_enum.str_print, le_enum.str_print) _convert_line_ending(filename, le_enum.string) + + return 1 elif detect_result == MixedLineDetection.NOT_MIXED: logging.info('The file %s has no mixed line ending', filename) + + return 0 elif detect_result == MixedLineDetection.UNKNOWN: logging.info('Could not define most frequent line ending in ' 'file %s. File skiped.', filename) + return 0 + # when a line ending character is forced with --fix option else: line_ending_enum = fix_option.line_ending_enum @@ -105,6 +112,8 @@ def mixed_line_ending(argv=None): for filename in options['filenames']: _convert_line_ending(filename, line_ending_enum.string) + return 1 + return 0 diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py new file mode 100644 index 0000000..9de14a5 --- /dev/null +++ b/tests/mixed_line_ending_test.py @@ -0,0 +1,26 @@ +import pytest + +from pre_commit_hooks.mixed_line_ending import mixed_line_ending + +# Input, expected return value, expected output +TESTS_FIX_AUTO = ( + # only 'LF' + (b'foo\nbar\nbaz\n', 0, b'foo\nbar\nbaz\n'), + # only 'CRLF' + (b'foo\r\nbar\r\nbaz\r\n', 0, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with majority of 'LF' + (b'foo\r\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), + # mixed with majority of 'CRLF' + (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), +) + + +@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), + TESTS_FIX_AUTO) +def test_mixed_line_ending_fix_auto(input_s, expected_retval, output, tmpdir): + path = tmpdir.join('file.txt') + path.write(input_s) + ret = mixed_line_ending(('--fix=auto', '-vv', path.strpath)) + + assert ret == expected_retval + assert path.read() == output From 4270b56e500acf20f0de5a20fa664eabefd45353 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 20:49:11 +0200 Subject: [PATCH 16/42] Refactor MixedLineDetection --- pre_commit_hooks/mixed_line_ending.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index ddc25eb..b136a8f 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -36,8 +36,8 @@ class MixedLineDetection(Enum): NOT_MIXED = False, None UNKNOWN = False, None - def __init__(self, conversion, line_ending_enum): - self.conversion = conversion + def __init__(self, mle_found, line_ending_enum): + self.mle_found = mle_found self.line_ending_enum = line_ending_enum @@ -83,7 +83,7 @@ def mixed_line_ending(argv=None): logging.debug('mixed_line_ending: detect_result = %s', detect_result) - if detect_result.conversion: + if detect_result.mle_found: le_enum = detect_result.line_ending_enum logging.info('The file %s has mixed line ending with a ' From a1ffbfaa5922281bd38e307392cb5b863bc1fe07 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 20:50:49 +0200 Subject: [PATCH 17/42] Add mixed line detection --- pre_commit_hooks/mixed_line_ending.py | 33 +++++++++++++++++++++++---- tests/mixed_line_ending_test.py | 24 +++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index b136a8f..c4aa9be 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -68,14 +68,13 @@ def mixed_line_ending(argv=None): level=options['logging_severity']) logging.debug('mixed_line_ending: options = %s', options) - _check_filenames(options['filenames']) - + filenames = options['filenames'] fix_option = options['fix'] - if fix_option == MixedLineEndingOption.NO: - logging.info('No conversion asked') + _check_filenames(filenames) - return 0 + if fix_option == MixedLineEndingOption.NO: + return _process_no_fix(filenames) elif fix_option == MixedLineEndingOption.AUTO: for filename in options['filenames']: detect_result = _detect_line_ending(filename) @@ -188,6 +187,30 @@ def _detect_line_ending(filename): return MixedLineDetection.MIXED_MOSTLY_LF +def _process_no_fix(filenames): + logging.info('Checking if the files have mixed line ending.') + + mle_found = False + mle_filenames = [] + for filename in filenames: + detect_result = _detect_line_ending(filename) + logging.debug('mixed_line_ending: detect_result = %s', + detect_result) + + if detect_result.mle_found: + mle_found = True + mle_filenames.append(filename) + logging.debug(filename) + + logging.debug(mle_found) + logging.debug(str(mle_filenames)) + if mle_filenames: + logging.info('The following files have mixed line endings:\n\t%s', + '\n\t'.join(mle_filenames)) + + return 1 if mle_found else 0 + + def _convert_line_ending(filename, line_ending): # read the file fin = open(filename, 'r') diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 9de14a5..0b33c92 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -24,3 +24,27 @@ def test_mixed_line_ending_fix_auto(input_s, expected_retval, output, tmpdir): assert ret == expected_retval assert path.read() == output + + +# Input, expected return value, expected output +TESTS_NO_FIX = ( + # only 'LF' + (b'foo\nbar\nbaz\n', 0, b'foo\nbar\nbaz\n'), + # only 'CRLF' + (b'foo\r\nbar\r\nbaz\r\n', 0, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with majority of 'LF' + (b'foo\r\nbar\nbaz\n', 1, b'foo\r\nbar\nbaz\n'), + # mixed with majority of 'CRLF' + (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\nbaz\r\n'), +) + + +@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), + TESTS_NO_FIX) +def test_detect_mixed_line_ending(input_s, expected_retval, output, tmpdir): + path = tmpdir.join('file.txt') + path.write(input_s) + ret = mixed_line_ending(('--fix=no', '-vv', path.strpath)) + + assert ret == expected_retval + assert path.read() == output From c6c4c4a2fa3829a6567b609b3b41639d30d3aa77 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 21:07:00 +0200 Subject: [PATCH 18/42] Refactor mixed_line_ending --- pre_commit_hooks/mixed_line_ending.py | 54 ++++++++++++++------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index c4aa9be..b2419e7 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -76,31 +76,7 @@ def mixed_line_ending(argv=None): if fix_option == MixedLineEndingOption.NO: return _process_no_fix(filenames) elif fix_option == MixedLineEndingOption.AUTO: - for filename in options['filenames']: - detect_result = _detect_line_ending(filename) - - logging.debug('mixed_line_ending: detect_result = %s', - detect_result) - - if detect_result.mle_found: - le_enum = detect_result.line_ending_enum - - logging.info('The file %s has mixed line ending with a ' - 'majority of "%s". Converting to "%s"', filename, - le_enum.str_print, le_enum.str_print) - - _convert_line_ending(filename, le_enum.string) - - return 1 - elif detect_result == MixedLineDetection.NOT_MIXED: - logging.info('The file %s has no mixed line ending', filename) - - return 0 - elif detect_result == MixedLineDetection.UNKNOWN: - logging.info('Could not define most frequent line ending in ' - 'file %s. File skiped.', filename) - - return 0 + return _process_fix_auto(filenames) # when a line ending character is forced with --fix option else: @@ -211,6 +187,34 @@ def _process_no_fix(filenames): return 1 if mle_found else 0 +def _process_fix_auto(filenames): + for filename in filenames: + detect_result = _detect_line_ending(filename) + + logging.debug('mixed_line_ending: detect_result = %s', + detect_result) + + if detect_result.mle_found: + le_enum = detect_result.line_ending_enum + + logging.info('The file %s has mixed line ending with a ' + 'majority of "%s". Converting to "%s"', filename, + le_enum.str_print, le_enum.str_print) + + _convert_line_ending(filename, le_enum.string) + + return 1 + elif detect_result == MixedLineDetection.NOT_MIXED: + logging.info('The file %s has no mixed line ending', filename) + + return 0 + elif detect_result == MixedLineDetection.UNKNOWN: + logging.info('Could not define most frequent line ending in ' + 'file %s. File skiped.', filename) + + return 0 + + def _convert_line_ending(filename, line_ending): # read the file fin = open(filename, 'r') From 614893f36def2c36a2f60a6d8f213c55d2ada3a4 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 21:16:27 +0200 Subject: [PATCH 19/42] Fix _process_fix_auto to return the right value --- pre_commit_hooks/mixed_line_ending.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index b2419e7..c890976 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -77,7 +77,6 @@ def mixed_line_ending(argv=None): return _process_no_fix(filenames) elif fix_option == MixedLineEndingOption.AUTO: return _process_fix_auto(filenames) - # when a line ending character is forced with --fix option else: line_ending_enum = fix_option.line_ending_enum @@ -188,6 +187,8 @@ def _process_no_fix(filenames): def _process_fix_auto(filenames): + converted_found = False + for filename in filenames: detect_result = _detect_line_ending(filename) @@ -198,21 +199,26 @@ def _process_fix_auto(filenames): le_enum = detect_result.line_ending_enum logging.info('The file %s has mixed line ending with a ' - 'majority of "%s". Converting to "%s"', filename, - le_enum.str_print, le_enum.str_print) + 'majority of "%s". Converting...', filename, + le_enum.str_print) _convert_line_ending(filename, le_enum.string) + converted_found = True + + logging.info('The file %s has been converted to "%s" line ' + 'ending.', filename, le_enum.str_print) - return 1 elif detect_result == MixedLineDetection.NOT_MIXED: logging.info('The file %s has no mixed line ending', filename) - return 0 + converted_found |= False elif detect_result == MixedLineDetection.UNKNOWN: logging.info('Could not define most frequent line ending in ' 'file %s. File skiped.', filename) - return 0 + converted_found |= False + + return 1 if converted_found else 0 def _convert_line_ending(filename, line_ending): From a1e1421a9973a7160f98301ce2753bad3ff5789e Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 21:23:03 +0200 Subject: [PATCH 20/42] Refactor mixed_line_ending --- pre_commit_hooks/mixed_line_ending.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index c890976..8f3e0b1 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -79,14 +79,7 @@ def mixed_line_ending(argv=None): return _process_fix_auto(filenames) # when a line ending character is forced with --fix option else: - line_ending_enum = fix_option.line_ending_enum - - logging.info('Force line ending to "%s"', line_ending_enum.str_print) - - for filename in options['filenames']: - _convert_line_ending(filename, line_ending_enum.string) - - return 1 + return _process_fix_force(filenames, fix_option.line_ending_enum) return 0 @@ -221,6 +214,15 @@ def _process_fix_auto(filenames): return 1 if converted_found else 0 +def _process_fix_force(filenames, line_ending_enum): + logging.info('Force line ending to "%s"', line_ending_enum.str_print) + + for filename in filenames: + _convert_line_ending(filename, line_ending_enum.string) + + return 1 + + def _convert_line_ending(filename, line_ending): # read the file fin = open(filename, 'r') From 609d01178ccd9cfaca4bd787b7e459c5611f9c52 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 21:26:02 +0200 Subject: [PATCH 21/42] Improve logging for force line ending --- pre_commit_hooks/mixed_line_ending.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 8f3e0b1..89451e7 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -215,11 +215,12 @@ def _process_fix_auto(filenames): def _process_fix_force(filenames, line_ending_enum): - logging.info('Force line ending to "%s"', line_ending_enum.str_print) - for filename in filenames: _convert_line_ending(filename, line_ending_enum.string) + logging.info('The file %s has been forced to "%s" line ending.', + filename, line_ending_enum.str_print) + return 1 From 63bb1fd1f5a56d84f2cc6f517a020b5a5a49e00a Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 21:51:08 +0200 Subject: [PATCH 22/42] Add unit test for mixed_line_ending --fix={cr,crlf} --- tests/mixed_line_ending_test.py | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 0b33c92..3da24a7 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -48,3 +48,53 @@ def test_detect_mixed_line_ending(input_s, expected_retval, output, tmpdir): assert ret == expected_retval assert path.read() == output + + +# Input, expected return value, expected output +TESTS_FIX_FORCE_LF = ( + # only 'LF' + (b'foo\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), + # only 'CRLF' + (b'foo\r\nbar\r\nbaz\r\n', 1, b'foo\nbar\nbaz\n'), + # mixed with majority of 'LF' + (b'foo\r\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), + # mixed with majority of 'CRLF' + (b'foo\r\nbar\nbaz\r\n', 1, b'foo\nbar\nbaz\n'), +) + + +@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), + TESTS_FIX_FORCE_LF) +def test_mixed_line_ending_fix_force_lf(input_s, expected_retval, output, + tmpdir): + path = tmpdir.join('file.txt') + path.write(input_s) + ret = mixed_line_ending(('--fix=lf', '-vv', path.strpath)) + + assert ret == expected_retval + assert path.read() == output + + +# Input, expected return value, expected output +TESTS_FIX_FORCE_CRLF = ( + # only 'LF' + (b'foo\nbar\nbaz\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # only 'CRLF' + (b'foo\r\nbar\r\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with majority of 'LF' + (b'foo\r\nbar\nbaz\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with majority of 'CRLF' + (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), +) + + +@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), + TESTS_FIX_FORCE_CRLF) +def test_mixed_line_ending_fix_force_crlf(input_s, expected_retval, output, + tmpdir): + path = tmpdir.join('file.txt') + path.write(input_s) + ret = mixed_line_ending(('--fix=crlf', '-vv', path.strpath)) + + assert ret == expected_retval + assert path.read() == output From 3dbeeeefe53bc536099eb3fd717ee9579bcda5f6 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 4 Jul 2017 22:24:24 +0200 Subject: [PATCH 23/42] Improve test coverage --- pre_commit_hooks/mixed_line_ending.py | 47 +++++++++++++-------------- tests/mixed_line_ending_test.py | 7 ++++ 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 89451e7..8b3dbff 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -31,12 +31,14 @@ class MixedLineEndingOption(Enum): class MixedLineDetection(Enum): - MIXED_MOSTLY_CRLF = True, LineEnding.CRLF - MIXED_MOSTLY_LF = True, LineEnding.LF - NOT_MIXED = False, None - UNKNOWN = False, None + MIXED_MOSTLY_CRLF = 1, True, LineEnding.CRLF + MIXED_MOSTLY_LF = 2, True, LineEnding.LF + NOT_MIXED = 3, False, None + UNKNOWN = 4, False, None - def __init__(self, mle_found, line_ending_enum): + def __init__(self, index, mle_found, line_ending_enum): + # TODO hack to prevent enum overriding + self.index = index self.mle_found = mle_found self.line_ending_enum = line_ending_enum @@ -81,8 +83,6 @@ def mixed_line_ending(argv=None): else: return _process_fix_force(filenames, fix_option.line_ending_enum) - return 0 - def _parse_arguments(argv=None): parser = argparse.ArgumentParser() @@ -102,14 +102,14 @@ def _parse_arguments(argv=None): args = parser.parse_args(argv) fix = None - if args.fix == MixedLineEndingOption.AUTO.opt_name: - fix = MixedLineEndingOption.AUTO - elif args.fix == MixedLineEndingOption.NO.opt_name: + if args.fix == MixedLineEndingOption.NO.opt_name: fix = MixedLineEndingOption.NO elif args.fix == MixedLineEndingOption.CRLF.opt_name: fix = MixedLineEndingOption.CRLF elif args.fix == MixedLineEndingOption.LF.opt_name: fix = MixedLineEndingOption.LF + else: + fix = MixedLineEndingOption.AUTO args.verbose = min(args.verbose, 2) severity = VERBOSE_OPTION_TO_LOGGING_SEVERITY.get(args.verbose) @@ -180,7 +180,7 @@ def _process_no_fix(filenames): def _process_fix_auto(filenames): - converted_found = False + mle_found = False for filename in filenames: detect_result = _detect_line_ending(filename) @@ -188,7 +188,16 @@ def _process_fix_auto(filenames): logging.debug('mixed_line_ending: detect_result = %s', detect_result) - if detect_result.mle_found: + if detect_result == MixedLineDetection.NOT_MIXED: + logging.info('The file %s has no mixed line ending', filename) + + mle_found |= False + elif detect_result == MixedLineDetection.UNKNOWN: + logging.info('Could not define most frequent line ending in ' + 'file %s. File skiped.', filename) + + mle_found = True + else: le_enum = detect_result.line_ending_enum logging.info('The file %s has mixed line ending with a ' @@ -196,22 +205,12 @@ def _process_fix_auto(filenames): le_enum.str_print) _convert_line_ending(filename, le_enum.string) - converted_found = True + mle_found = True logging.info('The file %s has been converted to "%s" line ' 'ending.', filename, le_enum.str_print) - elif detect_result == MixedLineDetection.NOT_MIXED: - logging.info('The file %s has no mixed line ending', filename) - - converted_found |= False - elif detect_result == MixedLineDetection.UNKNOWN: - logging.info('Could not define most frequent line ending in ' - 'file %s. File skiped.', filename) - - converted_found |= False - - return 1 if converted_found else 0 + return 1 if mle_found else 0 def _process_fix_force(filenames, line_ending_enum): diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 3da24a7..85081ac 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -12,6 +12,8 @@ TESTS_FIX_AUTO = ( (b'foo\r\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), # mixed with majority of 'CRLF' (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with as much 'LF' as 'CRLF' + (b'foo\r\nbar\nbaz', 1, b'foo\r\nbar\nbaz'), ) @@ -98,3 +100,8 @@ def test_mixed_line_ending_fix_force_crlf(input_s, expected_retval, output, assert ret == expected_retval assert path.read() == output + + +def test_check_filenames(): + with pytest.raises(IOError): + mixed_line_ending(['/dev/null']) From d0016c5be3ad9a738e76a0ff50e2aaa4f0fce76a Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sat, 8 Jul 2017 10:21:14 +0200 Subject: [PATCH 24/42] Ignore .cache/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2626934..f2bb40f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.iml *.py[co] .*.sw[a-z] +.cache/ .coverage .idea .project From ba63d1b9db3d8615e729c8374021b48b92ec97ca Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sat, 8 Jul 2017 10:30:09 +0200 Subject: [PATCH 25/42] Refactor _process_no_fix --- pre_commit_hooks/mixed_line_ending.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 8b3dbff..7a01af9 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -158,7 +158,6 @@ def _detect_line_ending(filename): def _process_no_fix(filenames): logging.info('Checking if the files have mixed line ending.') - mle_found = False mle_filenames = [] for filename in filenames: detect_result = _detect_line_ending(filename) @@ -166,13 +165,11 @@ def _process_no_fix(filenames): detect_result) if detect_result.mle_found: - mle_found = True mle_filenames.append(filename) - logging.debug(filename) - logging.debug(mle_found) - logging.debug(str(mle_filenames)) - if mle_filenames: + mle_found = len(mle_filenames) > 0 + + if mle_found: logging.info('The following files have mixed line endings:\n\t%s', '\n\t'.join(mle_filenames)) From 2b6ad978dee401387fa5bd00416dd4950a45ac1c Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sun, 9 Jul 2017 09:24:21 +0200 Subject: [PATCH 26/42] Fix 13 tests for Python 3.4 & 3.5 --- tests/mixed_line_ending_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 85081ac..6c83067 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -25,7 +25,7 @@ def test_mixed_line_ending_fix_auto(input_s, expected_retval, output, tmpdir): ret = mixed_line_ending(('--fix=auto', '-vv', path.strpath)) assert ret == expected_retval - assert path.read() == output + assert path.read_binary() == output # Input, expected return value, expected output @@ -49,7 +49,7 @@ def test_detect_mixed_line_ending(input_s, expected_retval, output, tmpdir): ret = mixed_line_ending(('--fix=no', '-vv', path.strpath)) assert ret == expected_retval - assert path.read() == output + assert path.read_binary() == output # Input, expected return value, expected output @@ -74,7 +74,7 @@ def test_mixed_line_ending_fix_force_lf(input_s, expected_retval, output, ret = mixed_line_ending(('--fix=lf', '-vv', path.strpath)) assert ret == expected_retval - assert path.read() == output + assert path.read_binary() == output # Input, expected return value, expected output @@ -99,7 +99,7 @@ def test_mixed_line_ending_fix_force_crlf(input_s, expected_retval, output, ret = mixed_line_ending(('--fix=crlf', '-vv', path.strpath)) assert ret == expected_retval - assert path.read() == output + assert path.read_binary() == output def test_check_filenames(): From d16d04a4d74f9aa60976fc52cd0a9ef36962733a Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Sun, 9 Jul 2017 10:51:50 +0200 Subject: [PATCH 27/42] Fix the 5 remaining tests for Python 3.4 & 3.5 Fixes nagromc/pre-commit-hooks#1 --- pre_commit_hooks/mixed_line_ending.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 7a01af9..05460f1 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -8,9 +8,9 @@ from enum import Enum class LineEnding(Enum): - CR = '\r', '\\r', 'cr', re.compile(r'\r', re.DOTALL) - CRLF = '\r\n', '\\r\\n', 'crlf', re.compile(r'\r\n', re.DOTALL) - LF = '\n', '\\n', 'lf', re.compile(r'(? Date: Sun, 9 Jul 2017 11:10:10 +0200 Subject: [PATCH 28/42] Refactor file opening --- pre_commit_hooks/mixed_line_ending.py | 52 +++++++++++++-------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 05460f1..98b532a 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -129,30 +129,29 @@ def _check_filenames(filenames): def _detect_line_ending(filename): - f = open(filename, 'rb') - buf = f.read() - f.close() + with open(filename, 'rb') as f: + buf = f.read() - crlf_nb = len(LineEnding.CRLF.regex.findall(buf)) - lf_nb = len(LineEnding.LF.regex.findall(buf)) + crlf_nb = len(LineEnding.CRLF.regex.findall(buf)) + lf_nb = len(LineEnding.LF.regex.findall(buf)) - crlf_found = crlf_nb > 0 - lf_found = lf_nb > 0 + crlf_found = crlf_nb > 0 + lf_found = lf_nb > 0 - logging.debug('_detect_line_ending: crlf_nb = %d, lf_nb = %d, ' - 'crlf_found = %s, lf_found = %s', - crlf_nb, lf_nb, crlf_found, lf_found) + logging.debug('_detect_line_ending: crlf_nb = %d, lf_nb = %d, ' + 'crlf_found = %s, lf_found = %s', + crlf_nb, lf_nb, crlf_found, lf_found) - if crlf_nb == lf_nb: - return MixedLineDetection.UNKNOWN + if crlf_nb == lf_nb: + return MixedLineDetection.UNKNOWN - if crlf_found ^ lf_found: - return MixedLineDetection.NOT_MIXED + if crlf_found ^ lf_found: + return MixedLineDetection.NOT_MIXED - if crlf_nb > lf_nb: - return MixedLineDetection.MIXED_MOSTLY_CRLF - else: - return MixedLineDetection.MIXED_MOSTLY_LF + if crlf_nb > lf_nb: + return MixedLineDetection.MIXED_MOSTLY_CRLF + else: + return MixedLineDetection.MIXED_MOSTLY_LF def _process_no_fix(filenames): @@ -222,17 +221,16 @@ def _process_fix_force(filenames, line_ending_enum): def _convert_line_ending(filename, line_ending): # read the file - fin = open(filename, 'rb') - bufin = fin.read() - fin.close() + with open(filename, 'rb+') as f: + bufin = f.read() - # convert line ending - bufout = ANY_LINE_ENDING_PATTERN.sub(line_ending, bufin) + # convert line ending + bufout = ANY_LINE_ENDING_PATTERN.sub(line_ending, bufin) - # write the result in the file - fout = open(filename, 'wb') - fout.write(bufout) - fout.close() + # write the result in the file + f.seek(0) + f.write(bufout) + f.truncate() if __name__ == '__main__': From 4fc9624b6a0e3e9d3f9a9ce8a6eff55b4719f8b7 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 10 Jul 2017 21:27:09 +0200 Subject: [PATCH 29/42] Refactor _detect_line_ending --- pre_commit_hooks/mixed_line_ending.py | 42 +++++++++++++++++---------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 98b532a..16cac56 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -8,7 +8,7 @@ from enum import Enum class LineEnding(Enum): - CR = b'\r', '\\r', 'cr', re.compile(b'\r', re.DOTALL) + CR = b'\r', '\\r', 'cr', re.compile(b'\r(?!\n)', re.DOTALL) CRLF = b'\r\n', '\\r\\n', 'crlf', re.compile(b'\r\n', re.DOTALL) LF = b'\n', '\\n', 'lf', re.compile(b'(? 0 - lf_found = lf_nb > 0 + logging.debug('_detect_line_ending: le_counts = ' + str(le_counts)) - logging.debug('_detect_line_ending: crlf_nb = %d, lf_nb = %d, ' - 'crlf_found = %s, lf_found = %s', - crlf_nb, lf_nb, crlf_found, lf_found) + mixed = False + le_found_previously = False + most_le = None + max_le_count = 0 - if crlf_nb == lf_nb: - return MixedLineDetection.UNKNOWN + for le, le_count in le_counts.items(): + le_found_cur = le_count > 0 - if crlf_found ^ lf_found: + mixed |= le_found_previously and le_found_cur + le_found_previously |= le_found_cur + + if le_count == max_le_count: + most_le = None + elif le_count > max_le_count: + max_le_count = le_count + most_le = le + + if not mixed: return MixedLineDetection.NOT_MIXED - if crlf_nb > lf_nb: - return MixedLineDetection.MIXED_MOSTLY_CRLF - else: - return MixedLineDetection.MIXED_MOSTLY_LF + for mld in MixedLineDetection: + if mld.line_ending_enum is not None \ + and mld.line_ending_enum == most_le: + return mld + + return MixedLineDetection.UNKNOWN def _process_no_fix(filenames): From 19377889df7533b73e4242f60e64a19c552582b6 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 10 Jul 2017 21:50:27 +0200 Subject: [PATCH 30/42] Add support for CR line ending for mixed_line_ending.py '--fix=cr' is not yet implemented though --- pre_commit_hooks/mixed_line_ending.py | 9 +++--- tests/mixed_line_ending_test.py | 40 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 16cac56..6bcfb24 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -31,10 +31,11 @@ class MixedLineEndingOption(Enum): class MixedLineDetection(Enum): - MIXED_MOSTLY_CRLF = 1, True, LineEnding.CRLF - MIXED_MOSTLY_LF = 2, True, LineEnding.LF - NOT_MIXED = 3, False, None - UNKNOWN = 4, False, None + NOT_MIXED = 1, False, None + UNKNOWN = 2, False, None + MIXED_MOSTLY_CRLF = 3, True, LineEnding.CRLF + MIXED_MOSTLY_LF = 4, True, LineEnding.LF + MIXED_MOSTLY_CR = 5, True, LineEnding.CR def __init__(self, index, mle_found, line_ending_enum): # TODO hack to prevent enum overriding diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 6c83067..00f5a84 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -8,12 +8,22 @@ TESTS_FIX_AUTO = ( (b'foo\nbar\nbaz\n', 0, b'foo\nbar\nbaz\n'), # only 'CRLF' (b'foo\r\nbar\r\nbaz\r\n', 0, b'foo\r\nbar\r\nbaz\r\n'), + # only 'CR' + (b'foo\rbar\rbaz\r', 0, b'foo\rbar\rbaz\r'), # mixed with majority of 'LF' (b'foo\r\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), # mixed with majority of 'CRLF' (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with majority of 'CR' + (b'foo\rbar\nbaz\r', 1, b'foo\rbar\rbaz\r'), # mixed with as much 'LF' as 'CRLF' (b'foo\r\nbar\nbaz', 1, b'foo\r\nbar\nbaz'), + # mixed with as much 'LF' as 'CR' + (b'foo\rbar\nbaz', 1, b'foo\rbar\nbaz'), + # mixed with as much 'CRLF' as 'CR' + (b'foo\r\nbar\nbaz', 1, b'foo\r\nbar\nbaz'), + # mixed with as much 'CRLF' as 'LF' as 'CR' + (b'foo\r\nbar\nbaz\r', 1, b'foo\r\nbar\nbaz\r'), ) @@ -34,10 +44,20 @@ TESTS_NO_FIX = ( (b'foo\nbar\nbaz\n', 0, b'foo\nbar\nbaz\n'), # only 'CRLF' (b'foo\r\nbar\r\nbaz\r\n', 0, b'foo\r\nbar\r\nbaz\r\n'), + # only 'CR' + (b'foo\rbar\rbaz\r', 0, b'foo\rbar\rbaz\r'), # mixed with majority of 'LF' (b'foo\r\nbar\nbaz\n', 1, b'foo\r\nbar\nbaz\n'), # mixed with majority of 'CRLF' (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\nbaz\r\n'), + # mixed with majority of 'CR' + (b'foo\rbar\nbaz\r', 1, b'foo\rbar\nbaz\r'), + # mixed with as much 'LF' as 'CR' + (b'foo\rbar\nbaz', 0, b'foo\rbar\nbaz'), + # mixed with as much 'CRLF' as 'CR' + (b'foo\r\nbar\nbaz', 0, b'foo\r\nbar\nbaz'), + # mixed with as much 'CRLF' as 'LF' as 'CR' + (b'foo\r\nbar\nbaz\r', 0, b'foo\r\nbar\nbaz\r'), ) @@ -58,10 +78,20 @@ TESTS_FIX_FORCE_LF = ( (b'foo\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), # only 'CRLF' (b'foo\r\nbar\r\nbaz\r\n', 1, b'foo\nbar\nbaz\n'), + # only 'CR' + (b'foo\rbar\rbaz\r', 1, b'foo\nbar\nbaz\n'), # mixed with majority of 'LF' (b'foo\r\nbar\nbaz\n', 1, b'foo\nbar\nbaz\n'), # mixed with majority of 'CRLF' (b'foo\r\nbar\nbaz\r\n', 1, b'foo\nbar\nbaz\n'), + # mixed with majority of 'CR' + (b'foo\rbar\nbaz\r', 1, b'foo\nbar\nbaz\n'), + # mixed with as much 'LF' as 'CR' + (b'foo\rbar\nbaz', 1, b'foo\nbar\nbaz'), + # mixed with as much 'CRLF' as 'CR' + (b'foo\r\nbar\nbaz', 1, b'foo\nbar\nbaz'), + # mixed with as much 'CRLF' as 'LF' as 'CR' + (b'foo\r\nbar\nbaz\r', 1, b'foo\nbar\nbaz\n'), ) @@ -83,10 +113,20 @@ TESTS_FIX_FORCE_CRLF = ( (b'foo\nbar\nbaz\n', 1, b'foo\r\nbar\r\nbaz\r\n'), # only 'CRLF' (b'foo\r\nbar\r\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # only 'CR' + (b'foo\rbar\rbaz\r', 1, b'foo\r\nbar\r\nbaz\r\n'), # mixed with majority of 'LF' (b'foo\r\nbar\nbaz\n', 1, b'foo\r\nbar\r\nbaz\r\n'), # mixed with majority of 'CRLF' (b'foo\r\nbar\nbaz\r\n', 1, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with majority of 'CR' + (b'foo\rbar\nbaz\r', 1, b'foo\r\nbar\r\nbaz\r\n'), + # mixed with as much 'LF' as 'CR' + (b'foo\rbar\nbaz', 1, b'foo\r\nbar\r\nbaz'), + # mixed with as much 'CRLF' as 'CR' + (b'foo\r\nbar\nbaz', 1, b'foo\r\nbar\r\nbaz'), + # mixed with as much 'CRLF' as 'LF' as 'CR' + (b'foo\r\nbar\nbaz\r', 1, b'foo\r\nbar\r\nbaz\r\n'), ) From 560e1c2e1bd8e1f97ec65235957ec2ed79942363 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 10 Jul 2017 22:06:48 +0200 Subject: [PATCH 31/42] Add mixed-line-ending hook declaration --- .pre-commit-hooks.yaml | 6 ++++++ README.md | 5 +++++ hooks.yaml | 6 ++++++ setup.py | 1 + 4 files changed, 18 insertions(+) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index bda3f76..227abfa 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -123,6 +123,12 @@ entry: forbid-new-submodules description: Prevent addition of new git submodules files: '' +- id: mixed-line-ending + name: Mixed line ending + language: python + entry: mixed-line-ending + description: Replaces or checks mixed line ending + files: '' - id: name-tests-test name: Tests should end in _test.py description: This verifies that test files are named correctly diff --git a/README.md b/README.md index 3b62234..848f348 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,11 @@ Add this to your `.pre-commit-config.yaml` - To remove the coding pragma pass `--remove` (useful in a python3-only codebase) - `flake8` - Run flake8 on your python files. - `forbid-new-submodules` - Prevent addition of new git submodules. +- `mixed-line-ending` - Replaces or checks mixed line ending. + - `--fix={auto,crlf,lf,no}` + - `auto` - Replace automatically the most frequent line ending. + - `crlf`, `lf` - Force to replace line ending by respectively CRLF and LF. + - `no` - Checks if there is any mixed line ending. - `name-tests-test` - Assert that files in tests/ end in `_test.py`. - Use `args: ['--django']` to match `test*.py` instead. - `no-commit-to-branch` - Protect specific branches from direct checkins. diff --git a/hooks.yaml b/hooks.yaml index bda3f76..227abfa 100644 --- a/hooks.yaml +++ b/hooks.yaml @@ -123,6 +123,12 @@ entry: forbid-new-submodules description: Prevent addition of new git submodules files: '' +- id: mixed-line-ending + name: Mixed line ending + language: python + entry: mixed-line-ending + description: Replaces or checks mixed line ending + files: '' - id: name-tests-test name: Tests should end in _test.py description: This verifies that test files are named correctly diff --git a/setup.py b/setup.py index 4abb7a2..d7bde34 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ setup( 'end-of-file-fixer = pre_commit_hooks.end_of_file_fixer:end_of_file_fixer', 'fix-encoding-pragma = pre_commit_hooks.fix_encoding_pragma:main', 'forbid-new-submodules = pre_commit_hooks.forbid_new_submodules:main', + 'mixed-line-ending = pre_commit_hooks.mixed_line_ending:mixed_line_ending', 'name-tests-test = pre_commit_hooks.tests_should_end_in_test:validate_files', 'no-commit-to-branch = pre_commit_hooks.no_commit_to_branch:main', 'pretty-format-json = pre_commit_hooks.pretty_format_json:pretty_format_json', From 0335ebfb866f36fa85746cf923984802535f4626 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 17 Jul 2017 21:30:20 +0200 Subject: [PATCH 32/42] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 848f348..29e0566 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ Add this to your `.pre-commit-config.yaml` - `forbid-new-submodules` - Prevent addition of new git submodules. - `mixed-line-ending` - Replaces or checks mixed line ending. - `--fix={auto,crlf,lf,no}` - - `auto` - Replace automatically the most frequent line ending. - - `crlf`, `lf` - Force to replace line ending by respectively CRLF and LF. + - `auto` - Replaces automatically the most frequent line ending. This is the default argument. + - `crlf`, `lf` - Forces to replace line ending by respectively CRLF and LF. - `no` - Checks if there is any mixed line ending. - `name-tests-test` - Assert that files in tests/ end in `_test.py`. - Use `args: ['--django']` to match `test*.py` instead. From ab2a849052ca7f02e42a3db6fa10d2201676cd79 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 18 Jul 2017 20:17:06 +0200 Subject: [PATCH 33/42] Clean up against add-trailing-comma --- pre_commit_hooks/mixed_line_ending.py | 64 ++++++++++++++++++--------- tests/mixed_line_ending_test.py | 36 ++++++++++----- 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 6bcfb24..5982705 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -60,15 +60,17 @@ ANY_LINE_ENDING_PATTERN = re.compile( b'|' + LineEnding.LF.regex.pattern + # or \r b'|' + LineEnding.CR.regex.pattern + - b')' + b')', ) def mixed_line_ending(argv=None): options = _parse_arguments(argv) - logging.basicConfig(format='%(levelname)s: %(message)s', - level=options['logging_severity']) + logging.basicConfig( + format='%(levelname)s: %(message)s', + level=options['logging_severity'], + ) logging.debug('mixed_line_ending: options = %s', options) filenames = options['filenames'] @@ -92,14 +94,16 @@ def _parse_arguments(argv=None): '--fix', choices=[m.opt_name for m in MixedLineEndingOption], default=MixedLineEndingOption.AUTO.opt_name, - help='Replace line ending with the specified. Default is "auto"') + help='Replace line ending with the specified. Default is "auto"', + ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') parser.add_argument( '-v', '--verbose', action="count", default=0, - help='Increase output verbosity') + help='Increase output verbosity', + ) args = parser.parse_args(argv) fix = None @@ -115,8 +119,10 @@ def _parse_arguments(argv=None): args.verbose = min(args.verbose, 2) severity = VERBOSE_OPTION_TO_LOGGING_SEVERITY.get(args.verbose) - options = {'fix': fix, 'filenames': args.filenames, - 'logging_severity': severity} + options = { + 'fix': fix, 'filenames': args.filenames, + 'logging_severity': severity, + } return options @@ -173,8 +179,10 @@ def _process_no_fix(filenames): mle_filenames = [] for filename in filenames: detect_result = _detect_line_ending(filename) - logging.debug('mixed_line_ending: detect_result = %s', - detect_result) + logging.debug( + 'mixed_line_ending: detect_result = %s', + detect_result, + ) if detect_result.mle_found: mle_filenames.append(filename) @@ -182,8 +190,10 @@ def _process_no_fix(filenames): mle_found = len(mle_filenames) > 0 if mle_found: - logging.info('The following files have mixed line endings:\n\t%s', - '\n\t'.join(mle_filenames)) + logging.info( + 'The following files have mixed line endings:\n\t%s', + '\n\t'.join(mle_filenames), + ) return 1 if mle_found else 0 @@ -194,30 +204,38 @@ def _process_fix_auto(filenames): for filename in filenames: detect_result = _detect_line_ending(filename) - logging.debug('mixed_line_ending: detect_result = %s', - detect_result) + logging.debug( + 'mixed_line_ending: detect_result = %s', + detect_result, + ) if detect_result == MixedLineDetection.NOT_MIXED: logging.info('The file %s has no mixed line ending', filename) mle_found |= False elif detect_result == MixedLineDetection.UNKNOWN: - logging.info('Could not define most frequent line ending in ' - 'file %s. File skiped.', filename) + logging.info( + 'Could not define most frequent line ending in ' + 'file %s. File skiped.', filename, + ) mle_found = True else: le_enum = detect_result.line_ending_enum - logging.info('The file %s has mixed line ending with a ' - 'majority of "%s". Converting...', filename, - le_enum.str_print) + logging.info( + 'The file %s has mixed line ending with a ' + 'majority of "%s". Converting...', filename, + le_enum.str_print, + ) _convert_line_ending(filename, le_enum.string) mle_found = True - logging.info('The file %s has been converted to "%s" line ' - 'ending.', filename, le_enum.str_print) + logging.info( + 'The file %s has been converted to "%s" line ' + 'ending.', filename, le_enum.str_print, + ) return 1 if mle_found else 0 @@ -226,8 +244,10 @@ def _process_fix_force(filenames, line_ending_enum): for filename in filenames: _convert_line_ending(filename, line_ending_enum.string) - logging.info('The file %s has been forced to "%s" line ending.', - filename, line_ending_enum.str_print) + logging.info( + 'The file %s has been forced to "%s" line ending.', + filename, line_ending_enum.str_print, + ) return 1 diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 00f5a84..c1dadd0 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -27,8 +27,10 @@ TESTS_FIX_AUTO = ( ) -@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), - TESTS_FIX_AUTO) +@pytest.mark.parametrize( + ('input_s', 'expected_retval', 'output'), + TESTS_FIX_AUTO, +) def test_mixed_line_ending_fix_auto(input_s, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write(input_s) @@ -61,8 +63,10 @@ TESTS_NO_FIX = ( ) -@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), - TESTS_NO_FIX) +@pytest.mark.parametrize( + ('input_s', 'expected_retval', 'output'), + TESTS_NO_FIX, +) def test_detect_mixed_line_ending(input_s, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write(input_s) @@ -95,10 +99,14 @@ TESTS_FIX_FORCE_LF = ( ) -@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), - TESTS_FIX_FORCE_LF) -def test_mixed_line_ending_fix_force_lf(input_s, expected_retval, output, - tmpdir): +@pytest.mark.parametrize( + ('input_s', 'expected_retval', 'output'), + TESTS_FIX_FORCE_LF, +) +def test_mixed_line_ending_fix_force_lf( + input_s, expected_retval, output, + tmpdir, +): path = tmpdir.join('file.txt') path.write(input_s) ret = mixed_line_ending(('--fix=lf', '-vv', path.strpath)) @@ -130,10 +138,14 @@ TESTS_FIX_FORCE_CRLF = ( ) -@pytest.mark.parametrize(('input_s', 'expected_retval', 'output'), - TESTS_FIX_FORCE_CRLF) -def test_mixed_line_ending_fix_force_crlf(input_s, expected_retval, output, - tmpdir): +@pytest.mark.parametrize( + ('input_s', 'expected_retval', 'output'), + TESTS_FIX_FORCE_CRLF, +) +def test_mixed_line_ending_fix_force_crlf( + input_s, expected_retval, output, + tmpdir, +): path = tmpdir.join('file.txt') path.write(input_s) ret = mixed_line_ending(('--fix=crlf', '-vv', path.strpath)) From 41ff0e10a8642a328d6cfe5c2b641e9dc434759b Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 18 Jul 2017 20:24:08 +0200 Subject: [PATCH 34/42] Use new features from pre-commit 0.15.0 --- .pre-commit-hooks.yaml | 7 +++++-- hooks.yaml | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 3637ec7..147dd45 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -193,10 +193,13 @@ minimum_pre_commit_version: 0.15.0 - id: mixed-line-ending name: Mixed line ending - language: python - entry: mixed-line-ending description: Replaces or checks mixed line ending + entry: mixed-line-ending + language: python + types: [text] + # for backward compatibility files: '' + minimum_pre_commit_version: 0.15.0 - id: name-tests-test name: Tests should end in _test.py description: This verifies that test files are named correctly diff --git a/hooks.yaml b/hooks.yaml index 25f71ac..59cc320 100644 --- a/hooks.yaml +++ b/hooks.yaml @@ -131,11 +131,11 @@ files: '' minimum_pre_commit_version: 0.15.0 - id: mixed-line-ending - name: Mixed line ending - language: python - entry: mixed-line-ending - description: Replaces or checks mixed line ending + language: system + name: upgrade-your-pre-commit-version + entry: upgrade-your-pre-commit-version files: '' + minimum_pre_commit_version: 0.15.0 - id: name-tests-test language: system name: upgrade-your-pre-commit-version From eb0c3ba0da60784760f1e14b0d3749ad95555591 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Tue, 18 Jul 2017 21:47:27 +0200 Subject: [PATCH 35/42] Update README.md based on maintainer's wish --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33744f6..b42a982 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Add this to your `.pre-commit-config.yaml` - `--fix={auto,crlf,lf,no}` - `auto` - Replaces automatically the most frequent line ending. This is the default argument. - `crlf`, `lf` - Forces to replace line ending by respectively CRLF and LF. - - `no` - Checks if there is any mixed line ending. + - `no` - Checks if there is any mixed line ending without modifying any file. - `name-tests-test` - Assert that files in tests/ end in `_test.py`. - Use `args: ['--django']` to match `test*.py` instead. - `no-commit-to-branch` - Protect specific branches from direct checkins. From f795097861de5382f82339348c6740217910101a Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 20 Jul 2017 18:45:47 +0200 Subject: [PATCH 36/42] Add Pyhton 2.7 'enum34' dependency declaration --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 55dee45..c0fa969 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ setup( 'simplejson', 'six', ], + extras_require={':python_version=="2.7"': ['enum34']}, entry_points={ 'console_scripts': [ 'autopep8-wrapper = pre_commit_hooks.autopep8_wrapper:main', From f58b552c37cf4b23492615265a5298ff5f3d1a0e Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 20 Jul 2017 18:54:56 +0200 Subject: [PATCH 37/42] Refactor pre_commit_hooks/mixed_line_ending.py --- pre_commit_hooks/mixed_line_ending.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 5982705..f918d4f 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -8,13 +8,13 @@ from enum import Enum class LineEnding(Enum): - CR = b'\r', '\\r', 'cr', re.compile(b'\r(?!\n)', re.DOTALL) - CRLF = b'\r\n', '\\r\\n', 'crlf', re.compile(b'\r\n', re.DOTALL) - LF = b'\n', '\\n', 'lf', re.compile(b'(? Date: Thu, 20 Jul 2017 18:59:38 +0200 Subject: [PATCH 38/42] Remove non relevant comments and make others more explicit --- pre_commit_hooks/mixed_line_ending.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index f918d4f..31e8eda 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -52,15 +52,9 @@ VERBOSE_OPTION_TO_LOGGING_SEVERITY = { ANY_LINE_ENDING_PATTERN = re.compile( - # match either - b'(' + - # \r\n - LineEnding.CRLF.regex.pattern + - # or \n + b'(' + LineEnding.CRLF.regex.pattern + b'|' + LineEnding.LF.regex.pattern + - # or \r - b'|' + LineEnding.CR.regex.pattern + - b')', + b'|' + LineEnding.CR.regex.pattern + b')', ) @@ -252,14 +246,13 @@ def _process_fix_force(filenames, line_ending_enum): def _convert_line_ending(filename, line_ending): - # read the file with open(filename, 'rb+') as f: bufin = f.read() # convert line ending bufout = ANY_LINE_ENDING_PATTERN.sub(line_ending, bufin) - # write the result in the file + # write the result in the file replacing the existing content f.seek(0) f.write(bufout) f.truncate() From ef4a3237289854c746cd9b1376bbd0066d639c62 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 20 Jul 2017 19:26:03 +0200 Subject: [PATCH 39/42] Refactor pre_commit_hooks/mixed_line_ending.py --- pre_commit_hooks/mixed_line_ending.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 31e8eda..7be4fc5 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -100,15 +100,11 @@ def _parse_arguments(argv=None): ) args = parser.parse_args(argv) - fix = None - if args.fix == MixedLineEndingOption.NO.opt_name: - fix = MixedLineEndingOption.NO - elif args.fix == MixedLineEndingOption.CRLF.opt_name: - fix = MixedLineEndingOption.CRLF - elif args.fix == MixedLineEndingOption.LF.opt_name: - fix = MixedLineEndingOption.LF - else: - fix = MixedLineEndingOption.AUTO + fix, = ( + member for name, member + in MixedLineEndingOption.__members__.items() + if member.opt_name == args.fix + ) args.verbose = min(args.verbose, 2) severity = VERBOSE_OPTION_TO_LOGGING_SEVERITY.get(args.verbose) From 4d3d8e18317959b2a69d01353bd7052e336aa6e2 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 20 Jul 2017 20:24:32 +0200 Subject: [PATCH 40/42] Remove -v/--verbose option on mixed_line_ending.py --- pre_commit_hooks/mixed_line_ending.py | 52 ++++----------------------- tests/mixed_line_ending_test.py | 8 ++--- 2 files changed, 11 insertions(+), 49 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 7be4fc5..21f6b2f 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -1,5 +1,4 @@ import argparse -import logging import os import re import sys @@ -44,13 +43,6 @@ class MixedLineDetection(Enum): self.line_ending_enum = line_ending_enum -VERBOSE_OPTION_TO_LOGGING_SEVERITY = { - 0: logging.WARNING, - 1: logging.INFO, - 2: logging.DEBUG, -} - - ANY_LINE_ENDING_PATTERN = re.compile( b'(' + LineEnding.CRLF.regex.pattern + b'|' + LineEnding.LF.regex.pattern + @@ -61,12 +53,6 @@ ANY_LINE_ENDING_PATTERN = re.compile( def mixed_line_ending(argv=None): options = _parse_arguments(argv) - logging.basicConfig( - format='%(levelname)s: %(message)s', - level=options['logging_severity'], - ) - logging.debug('mixed_line_ending: options = %s', options) - filenames = options['filenames'] fix_option = options['fix'] @@ -91,13 +77,6 @@ def _parse_arguments(argv=None): help='Replace line ending with the specified. Default is "auto"', ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') - parser.add_argument( - '-v', - '--verbose', - action="count", - default=0, - help='Increase output verbosity', - ) args = parser.parse_args(argv) fix, = ( @@ -106,20 +85,14 @@ def _parse_arguments(argv=None): if member.opt_name == args.fix ) - args.verbose = min(args.verbose, 2) - severity = VERBOSE_OPTION_TO_LOGGING_SEVERITY.get(args.verbose) - options = { 'fix': fix, 'filenames': args.filenames, - 'logging_severity': severity, } return options def _check_filenames(filenames): - logging.debug('_check_filenames: filenames = %s', filenames) - for filename in filenames: if not os.path.isfile(filename): raise IOError('The file "{}" does not exist'.format(filename)) @@ -133,8 +106,6 @@ def _detect_line_ending(filename): for le_enum in LineEnding: le_counts[le_enum] = len(le_enum.regex.findall(buf)) - logging.debug('_detect_line_ending: le_counts = ' + str(le_counts)) - mixed = False le_found_previously = False most_le = None @@ -164,15 +135,11 @@ def _detect_line_ending(filename): def _process_no_fix(filenames): - logging.info('Checking if the files have mixed line ending.') + print('Checking if the files have mixed line ending.') mle_filenames = [] for filename in filenames: detect_result = _detect_line_ending(filename) - logging.debug( - 'mixed_line_ending: detect_result = %s', - detect_result, - ) if detect_result.mle_found: mle_filenames.append(filename) @@ -180,7 +147,7 @@ def _process_no_fix(filenames): mle_found = len(mle_filenames) > 0 if mle_found: - logging.info( + print( 'The following files have mixed line endings:\n\t%s', '\n\t'.join(mle_filenames), ) @@ -194,17 +161,12 @@ def _process_fix_auto(filenames): for filename in filenames: detect_result = _detect_line_ending(filename) - logging.debug( - 'mixed_line_ending: detect_result = %s', - detect_result, - ) - if detect_result == MixedLineDetection.NOT_MIXED: - logging.info('The file %s has no mixed line ending', filename) + print('The file %s has no mixed line ending', filename) mle_found |= False elif detect_result == MixedLineDetection.UNKNOWN: - logging.info( + print( 'Could not define most frequent line ending in ' 'file %s. File skiped.', filename, ) @@ -213,7 +175,7 @@ def _process_fix_auto(filenames): else: le_enum = detect_result.line_ending_enum - logging.info( + print( 'The file %s has mixed line ending with a ' 'majority of %s. Converting...', filename, le_enum.str_print, ) @@ -221,7 +183,7 @@ def _process_fix_auto(filenames): _convert_line_ending(filename, le_enum.string) mle_found = True - logging.info( + print( 'The file %s has been converted to %s line ending.', filename, le_enum.str_print, ) @@ -233,7 +195,7 @@ def _process_fix_force(filenames, line_ending_enum): for filename in filenames: _convert_line_ending(filename, line_ending_enum.string) - logging.info( + print( 'The file %s has been forced to %s line ending.', filename, line_ending_enum.str_print, ) diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index c1dadd0..b2ac90a 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -34,7 +34,7 @@ TESTS_FIX_AUTO = ( def test_mixed_line_ending_fix_auto(input_s, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write(input_s) - ret = mixed_line_ending(('--fix=auto', '-vv', path.strpath)) + ret = mixed_line_ending(('--fix=auto', path.strpath)) assert ret == expected_retval assert path.read_binary() == output @@ -70,7 +70,7 @@ TESTS_NO_FIX = ( def test_detect_mixed_line_ending(input_s, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write(input_s) - ret = mixed_line_ending(('--fix=no', '-vv', path.strpath)) + ret = mixed_line_ending(('--fix=no', path.strpath)) assert ret == expected_retval assert path.read_binary() == output @@ -109,7 +109,7 @@ def test_mixed_line_ending_fix_force_lf( ): path = tmpdir.join('file.txt') path.write(input_s) - ret = mixed_line_ending(('--fix=lf', '-vv', path.strpath)) + ret = mixed_line_ending(('--fix=lf', path.strpath)) assert ret == expected_retval assert path.read_binary() == output @@ -148,7 +148,7 @@ def test_mixed_line_ending_fix_force_crlf( ): path = tmpdir.join('file.txt') path.write(input_s) - ret = mixed_line_ending(('--fix=crlf', '-vv', path.strpath)) + ret = mixed_line_ending(('--fix=crlf', path.strpath)) assert ret == expected_retval assert path.read_binary() == output From f9915cbbe23a535e81c8434cd4ab66055bf31f56 Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 20 Jul 2017 20:32:04 +0200 Subject: [PATCH 41/42] Remove _check_filenames in mixed_line_ending.py --- pre_commit_hooks/mixed_line_ending.py | 9 --------- tests/mixed_line_ending_test.py | 5 ----- 2 files changed, 14 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 21f6b2f..0beab60 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -1,5 +1,4 @@ import argparse -import os import re import sys @@ -56,8 +55,6 @@ def mixed_line_ending(argv=None): filenames = options['filenames'] fix_option = options['fix'] - _check_filenames(filenames) - if fix_option == MixedLineEndingOption.NO: return _process_no_fix(filenames) elif fix_option == MixedLineEndingOption.AUTO: @@ -92,12 +89,6 @@ def _parse_arguments(argv=None): return options -def _check_filenames(filenames): - for filename in filenames: - if not os.path.isfile(filename): - raise IOError('The file "{}" does not exist'.format(filename)) - - def _detect_line_ending(filename): with open(filename, 'rb') as f: buf = f.read() diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index b2ac90a..9b49b5e 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -152,8 +152,3 @@ def test_mixed_line_ending_fix_force_crlf( assert ret == expected_retval assert path.read_binary() == output - - -def test_check_filenames(): - with pytest.raises(IOError): - mixed_line_ending(['/dev/null']) From 0e223bcb15d59675459c4b8bb5cd564787a0acbb Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Thu, 20 Jul 2017 20:58:53 +0200 Subject: [PATCH 42/42] Refactor mixed_line_ending.py --- pre_commit_hooks/mixed_line_ending.py | 49 ++++++++++++++------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 0beab60..76512a7 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -94,35 +94,38 @@ def _detect_line_ending(filename): buf = f.read() le_counts = {} - for le_enum in LineEnding: - le_counts[le_enum] = len(le_enum.regex.findall(buf)) - mixed = False - le_found_previously = False - most_le = None - max_le_count = 0 + for le_enum in LineEnding: + le_counts[le_enum] = len(le_enum.regex.findall(buf)) - for le, le_count in le_counts.items(): - le_found_cur = le_count > 0 + mixed = False + le_found_previously = False + most_le = None + max_le_count = 0 - mixed |= le_found_previously and le_found_cur - le_found_previously |= le_found_cur + for le, le_count in le_counts.items(): + le_found_cur = le_count > 0 - if le_count == max_le_count: - most_le = None - elif le_count > max_le_count: - max_le_count = le_count - most_le = le + mixed |= le_found_previously and le_found_cur + le_found_previously |= le_found_cur - if not mixed: - return MixedLineDetection.NOT_MIXED + if le_count == max_le_count: + most_le = None + elif le_count > max_le_count: + max_le_count = le_count + most_le = le - for mld in MixedLineDetection: - if mld.line_ending_enum is not None \ - and mld.line_ending_enum == most_le: - return mld + if not mixed: + return MixedLineDetection.NOT_MIXED - return MixedLineDetection.UNKNOWN + for mld in MixedLineDetection: + if ( + mld.line_ending_enum is not None and + mld.line_ending_enum == most_le + ): + return mld + + return MixedLineDetection.UNKNOWN def _process_no_fix(filenames): @@ -154,8 +157,6 @@ def _process_fix_auto(filenames): if detect_result == MixedLineDetection.NOT_MIXED: print('The file %s has no mixed line ending', filename) - - mle_found |= False elif detect_result == MixedLineDetection.UNKNOWN: print( 'Could not define most frequent line ending in '