From 845a3d5bdf05579b02e49ef9ea25c7f66edcb38b Mon Sep 17 00:00:00 2001 From: dmlb2000 Date: Thu, 3 Nov 2016 09:41:23 -0700 Subject: [PATCH 1/6] adds top keys list of keys in hashes to put at the top of a hash This adds custom sorting to preferencially add a list of top keys at the start of any json hash in the json document --- pre_commit_hooks/pretty_format_json.py | 107 ++++++++++++++++++++++--- tests/pretty_format_json_test.py | 7 ++ 2 files changed, 105 insertions(+), 9 deletions(-) diff --git a/pre_commit_hooks/pretty_format_json.py b/pre_commit_hooks/pretty_format_json.py index 36a90eb..3e0ab32 100644 --- a/pre_commit_hooks/pretty_format_json.py +++ b/pre_commit_hooks/pretty_format_json.py @@ -6,17 +6,95 @@ from collections import OrderedDict import simplejson +class SortableOrderedDict(OrderedDict): + """Performs an in-place sort of the keys if you want.""" + def sort(*args, **kwds): + self = args[0] + args = args[1:] + if 'key' not in kwds: + kwds['key'] = lambda x: x[0] + if len(args): + raise TypeError('expected no positional arguments got {0}'.format(len(args))) + sorted_od = sorted([x for x in self.items()], **kwds) + self.clear() + self.update(sorted_od) -def _get_pretty_format(contents, indent, sort_keys=True): - return simplejson.dumps( - simplejson.loads( - contents, - object_pairs_hook=None if sort_keys else OrderedDict, - ), - sort_keys=sort_keys, - indent=indent - ) + "\n" # dumps don't end with a newline +class TrackedSod(SortableOrderedDict): + """Tracks instances of the SortableOrderedDict.""" + _instances = [] + def __init__(self, *args, **kwds): + super(TrackedSod, self).__init__(*args, **kwds) + self.__track(self) + @classmethod + def __track(cls, obj): + cls._instances.append(obj) + + +def _get_pretty_format(contents, indent, sort_keys=True, top_keys=[]): + class KeyToCmp(object): + def __init__(self, obj, *args): + self.obj = obj[0] + def __lt__(self, other): + if self.obj in top_keys and other.obj in top_keys: + return top_keys.index(self.obj) < top_keys.index(other.obj) + elif self.obj in top_keys and other.obj not in top_keys: + return True + elif self.obj not in top_keys and other.obj in top_keys: + return False + else: + return self.obj < other.obj + def __gt__(self, other): + if self.obj in top_keys and other.obj in top_keys: + return top_keys.index(self.obj) > top_keys.index(other.obj) + elif self.obj in top_keys and other.obj not in top_keys: + return False + elif self.obj not in top_keys and other.obj in top_keys: + return True + else: + return self.obj > other.obj + def __eq__(self, other): + if self.obj in top_keys and other.obj in top_keys: + return top_keys.index(self.obj) == top_keys.index(other.obj) + elif self.obj in top_keys and other.obj not in top_keys: + return False + elif self.obj not in top_keys and other.obj in top_keys: + return False + else: + return self.obj == other.obj + def __le__(self, other): + if self.obj in top_keys and other.obj in top_keys: + return top_keys.index(self.obj) <= top_keys.index(other.obj) + elif self.obj in top_keys and other.obj not in top_keys: + return True + elif self.obj not in top_keys and other.obj in top_keys: + return False + else: + return self.obj <= other.obj + def __ge__(self, other): + if self.obj in top_keys and other.obj in top_keys: + return top_keys.index(self.obj) >= top_keys.index(other.obj) + elif self.obj in top_keys and other.obj not in top_keys: + return False + elif self.obj not in top_keys and other.obj in top_keys: + return True + else: + return self.obj >= other.obj + def __ne__(self, other): + if self.obj in top_keys and other.obj in top_keys: + return top_keys.index(self.obj) != top_keys.index(other.obj) + elif self.obj in top_keys and other.obj not in top_keys: + return False + elif self.obj not in top_keys and other.obj in top_keys: + return False + else: + return self.obj != other.obj + py_obj = simplejson.loads(contents, object_pairs_hook=TrackedSod) + if sort_keys: + for tsod in TrackedSod._instances: + tsod.sort(key=KeyToCmp) + # dumps don't end with a newline + return simplejson.dumps(py_obj, indent=indent) + "\n" def _autofix(filename, new_contents): print("Fixing file {0}".format(filename)) @@ -43,6 +121,9 @@ def parse_indent(s): 'Negative integer supplied to construct JSON indentation delimiter. ', ) +def parse_topkeys(s): + # type: (str) -> array + return s.split(',') def pretty_format_json(argv=None): parser = argparse.ArgumentParser() @@ -65,6 +146,13 @@ def pretty_format_json(argv=None): default=False, help='Keep JSON nodes in the same order', ) + parser.add_argument( + '--top-keys', + type=parse_topkeys, + dest='top_keys', + default=[], + help='Ordered list of keys to keep at the top of JSON hashes', + ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -78,6 +166,7 @@ def pretty_format_json(argv=None): try: pretty_contents = _get_pretty_format( contents, args.indent, sort_keys=not args.no_sort_keys, + top_keys=args.top_keys ) if contents != pretty_contents: diff --git a/tests/pretty_format_json_test.py b/tests/pretty_format_json_test.py index d9061d3..387a1bf 100644 --- a/tests/pretty_format_json_test.py +++ b/tests/pretty_format_json_test.py @@ -64,6 +64,13 @@ def test_autofix_pretty_format_json(tmpdir): ret = pretty_format_json([srcfile.strpath]) assert ret == 0 +def test_orderfile_get_pretty_format(): + ret = pretty_format_json(['--top-keys=alist', get_resource_path('pretty_formatted_json.json')]) + assert ret == 0 + +def test_orderfile_get_pretty_format(): + ret = pretty_format_json(['--top-keys=blah', get_resource_path('pretty_formatted_json.json')]) + assert ret == 1 def test_badfile_pretty_format_json(): ret = pretty_format_json([get_resource_path('ok_yaml.yaml')]) From d06a515ce1c2804d0a779d796c5f7d6773f6fec7 Mon Sep 17 00:00:00 2001 From: dmlb2000 Date: Thu, 3 Nov 2016 15:47:21 -0700 Subject: [PATCH 2/6] this is much cleaner and might actually get all the coverage with out a bunch of work --- pre_commit_hooks/pretty_format_json.py | 100 ++++--------------------- 1 file changed, 14 insertions(+), 86 deletions(-) diff --git a/pre_commit_hooks/pretty_format_json.py b/pre_commit_hooks/pretty_format_json.py index 3e0ab32..1058e21 100644 --- a/pre_commit_hooks/pretty_format_json.py +++ b/pre_commit_hooks/pretty_format_json.py @@ -6,95 +6,23 @@ from collections import OrderedDict import simplejson -class SortableOrderedDict(OrderedDict): - """Performs an in-place sort of the keys if you want.""" - def sort(*args, **kwds): - self = args[0] - args = args[1:] - if 'key' not in kwds: - kwds['key'] = lambda x: x[0] - if len(args): - raise TypeError('expected no positional arguments got {0}'.format(len(args))) - sorted_od = sorted([x for x in self.items()], **kwds) - self.clear() - self.update(sorted_od) - -class TrackedSod(SortableOrderedDict): - """Tracks instances of the SortableOrderedDict.""" - _instances = [] - def __init__(self, *args, **kwds): - super(TrackedSod, self).__init__(*args, **kwds) - self.__track(self) - - @classmethod - def __track(cls, obj): - cls._instances.append(obj) def _get_pretty_format(contents, indent, sort_keys=True, top_keys=[]): - class KeyToCmp(object): - def __init__(self, obj, *args): - self.obj = obj[0] - def __lt__(self, other): - if self.obj in top_keys and other.obj in top_keys: - return top_keys.index(self.obj) < top_keys.index(other.obj) - elif self.obj in top_keys and other.obj not in top_keys: - return True - elif self.obj not in top_keys and other.obj in top_keys: - return False - else: - return self.obj < other.obj - def __gt__(self, other): - if self.obj in top_keys and other.obj in top_keys: - return top_keys.index(self.obj) > top_keys.index(other.obj) - elif self.obj in top_keys and other.obj not in top_keys: - return False - elif self.obj not in top_keys and other.obj in top_keys: - return True - else: - return self.obj > other.obj - def __eq__(self, other): - if self.obj in top_keys and other.obj in top_keys: - return top_keys.index(self.obj) == top_keys.index(other.obj) - elif self.obj in top_keys and other.obj not in top_keys: - return False - elif self.obj not in top_keys and other.obj in top_keys: - return False - else: - return self.obj == other.obj - def __le__(self, other): - if self.obj in top_keys and other.obj in top_keys: - return top_keys.index(self.obj) <= top_keys.index(other.obj) - elif self.obj in top_keys and other.obj not in top_keys: - return True - elif self.obj not in top_keys and other.obj in top_keys: - return False - else: - return self.obj <= other.obj - def __ge__(self, other): - if self.obj in top_keys and other.obj in top_keys: - return top_keys.index(self.obj) >= top_keys.index(other.obj) - elif self.obj in top_keys and other.obj not in top_keys: - return False - elif self.obj not in top_keys and other.obj in top_keys: - return True - else: - return self.obj >= other.obj - def __ne__(self, other): - if self.obj in top_keys and other.obj in top_keys: - return top_keys.index(self.obj) != top_keys.index(other.obj) - elif self.obj in top_keys and other.obj not in top_keys: - return False - elif self.obj not in top_keys and other.obj in top_keys: - return False - else: - return self.obj != other.obj - py_obj = simplejson.loads(contents, object_pairs_hook=TrackedSod) - if sort_keys: - for tsod in TrackedSod._instances: - tsod.sort(key=KeyToCmp) - # dumps don't end with a newline - return simplejson.dumps(py_obj, indent=indent) + "\n" + def pairs_first(pairs): + before = [pair for pair in pairs if pair[0] in top_keys] + before = sorted(before, key=lambda x: top_keys.index(x[0])) + after = [pair for pair in pairs if pair[0] not in top_keys] + if sort_keys: + after = sorted(after, key=lambda x: x[0]) + return OrderedDict(before + after) + return simplejson.dumps( + simplejson.loads( + contents, + object_pairs_hook=pairs_first, + ), + indent=indent + ) + "\n" # dumps don't end with a newline def _autofix(filename, new_contents): print("Fixing file {0}".format(filename)) From c7ab1976450968562b61ab9975d5573efe90c08a Mon Sep 17 00:00:00 2001 From: dmlb2000 Date: Thu, 3 Nov 2016 15:49:04 -0700 Subject: [PATCH 3/6] don't need to blow away the space here --- pre_commit_hooks/pretty_format_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pre_commit_hooks/pretty_format_json.py b/pre_commit_hooks/pretty_format_json.py index 1058e21..a1dccc0 100644 --- a/pre_commit_hooks/pretty_format_json.py +++ b/pre_commit_hooks/pretty_format_json.py @@ -24,6 +24,7 @@ def _get_pretty_format(contents, indent, sort_keys=True, top_keys=[]): indent=indent ) + "\n" # dumps don't end with a newline + def _autofix(filename, new_contents): print("Fixing file {0}".format(filename)) with open(filename, 'w') as f: From 7f057b0bd5b25b888c3d181a2f65626580a26cbd Mon Sep 17 00:00:00 2001 From: dmlb2000 Date: Thu, 3 Nov 2016 15:51:24 -0700 Subject: [PATCH 4/6] change the name to show both working and not instead of overwriting the function name --- tests/pretty_format_json_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pretty_format_json_test.py b/tests/pretty_format_json_test.py index 387a1bf..df2152a 100644 --- a/tests/pretty_format_json_test.py +++ b/tests/pretty_format_json_test.py @@ -68,7 +68,7 @@ def test_orderfile_get_pretty_format(): ret = pretty_format_json(['--top-keys=alist', get_resource_path('pretty_formatted_json.json')]) assert ret == 0 -def test_orderfile_get_pretty_format(): +def test_not_orderfile_get_pretty_format(): ret = pretty_format_json(['--top-keys=blah', get_resource_path('pretty_formatted_json.json')]) assert ret == 1 From 84b1fb6827ebcd539f60ffdd6f95229650aa056c Mon Sep 17 00:00:00 2001 From: dmlb2000 Date: Thu, 3 Nov 2016 15:54:48 -0700 Subject: [PATCH 5/6] let pre-commit fix some stuff --- pre_commit_hooks/pretty_format_json.py | 3 ++- tests/pretty_format_json_test.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pre_commit_hooks/pretty_format_json.py b/pre_commit_hooks/pretty_format_json.py index a1dccc0..91dae8d 100644 --- a/pre_commit_hooks/pretty_format_json.py +++ b/pre_commit_hooks/pretty_format_json.py @@ -7,7 +7,6 @@ from collections import OrderedDict import simplejson - def _get_pretty_format(contents, indent, sort_keys=True, top_keys=[]): def pairs_first(pairs): before = [pair for pair in pairs if pair[0] in top_keys] @@ -50,10 +49,12 @@ def parse_indent(s): 'Negative integer supplied to construct JSON indentation delimiter. ', ) + def parse_topkeys(s): # type: (str) -> array return s.split(',') + def pretty_format_json(argv=None): parser = argparse.ArgumentParser() parser.add_argument( diff --git a/tests/pretty_format_json_test.py b/tests/pretty_format_json_test.py index df2152a..72c2b86 100644 --- a/tests/pretty_format_json_test.py +++ b/tests/pretty_format_json_test.py @@ -64,14 +64,17 @@ def test_autofix_pretty_format_json(tmpdir): ret = pretty_format_json([srcfile.strpath]) assert ret == 0 + def test_orderfile_get_pretty_format(): ret = pretty_format_json(['--top-keys=alist', get_resource_path('pretty_formatted_json.json')]) assert ret == 0 + def test_not_orderfile_get_pretty_format(): ret = pretty_format_json(['--top-keys=blah', get_resource_path('pretty_formatted_json.json')]) assert ret == 1 + def test_badfile_pretty_format_json(): ret = pretty_format_json([get_resource_path('ok_yaml.yaml')]) assert ret == 1 From e9e9c3d57719551bbad7b8176f2830d0cf9a5597 Mon Sep 17 00:00:00 2001 From: David Brown Date: Thu, 3 Nov 2016 18:05:43 -0700 Subject: [PATCH 6/6] add test to show how it works a bit more --- testing/resources/top_sorted_json.json | 16 ++++++++++++++++ tests/pretty_format_json_test.py | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 testing/resources/top_sorted_json.json diff --git a/testing/resources/top_sorted_json.json b/testing/resources/top_sorted_json.json new file mode 100644 index 0000000..62083e7 --- /dev/null +++ b/testing/resources/top_sorted_json.json @@ -0,0 +1,16 @@ +{ + "01-alist": [ + 2, + 34, + 234 + ], + "alist": [ + 2, + 34, + 234 + ], + "00-foo": "bar", + "02-blah": null, + "blah": null, + "foo": "bar" +} diff --git a/tests/pretty_format_json_test.py b/tests/pretty_format_json_test.py index 72c2b86..7bfc31f 100644 --- a/tests/pretty_format_json_test.py +++ b/tests/pretty_format_json_test.py @@ -75,6 +75,11 @@ def test_not_orderfile_get_pretty_format(): assert ret == 1 +def test_top_sorted_get_pretty_format(): + ret = pretty_format_json(['--top-keys=01-alist,alist', get_resource_path('top_sorted_json.json')]) + assert ret == 0 + + def test_badfile_pretty_format_json(): ret = pretty_format_json([get_resource_path('ok_yaml.yaml')]) assert ret == 1