From f593057acc487b32cb463aa2de1e0a35fb893436 Mon Sep 17 00:00:00 2001 From: HongWeipeng Date: Tue, 23 Oct 2018 15:03:02 +0800 Subject: [PATCH 1/4] add jsonlines option to json.tool --- Lib/json/tool.py | 13 +++++++-- Lib/test/test_json/test_tool.py | 29 +++++++++++++++++++ .../2018-10-23-14-46-47.bpo-31553.JxRkAW.rst | 1 + 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 5932f4ecded7eb..1b4a1a10954ccb 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -26,19 +26,26 @@ def main(): help='write the output of infile to outfile') parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') + parser.add_argument('--jsonlines', action='store_true', default=False, + help='parse input using the jsonlines format') options = parser.parse_args() infile = options.infile or sys.stdin outfile = options.outfile or sys.stdout sort_keys = options.sort_keys + jsonlines = options.jsonlines with infile: try: - obj = json.load(infile) + if jsonlines: + objs = [json.loads(line) for line in infile] + else: + objs = [json.load(infile)] except ValueError as e: raise SystemExit(e) with outfile: - json.dump(obj, outfile, sort_keys=sort_keys, indent=4) - outfile.write('\n') + for obj in objs: + json.dump(obj, outfile, sort_keys=sort_keys, indent=4) + outfile.write('\n') if __name__ == '__main__': diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 9d93f931ca3958..c0ba66a321e15e 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -60,6 +60,28 @@ class TestTool(unittest.TestCase): ] """) + jsonlines_raw = textwrap.dedent("""\ + {"ingredients":["frog", "water", "chocolate", "glucose"]} + {"ingredients":["chocolate","steel bolts"]} + """) + + jsonlines_expect = textwrap.dedent("""\ + { + "ingredients": [ + "frog", + "water", + "chocolate", + "glucose" + ] + } + { + "ingredients": [ + "chocolate", + "steel bolts" + ] + } + """) + def test_stdin_stdout(self): args = sys.executable, '-m', 'json.tool' with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: @@ -92,6 +114,13 @@ def test_infile_outfile(self): self.assertEqual(out, b'') self.assertEqual(err, b'') + def test_jsonlines(self): + args = sys.executable, '-m', 'json.tool', '--jsonlines' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: + out, err = proc.communicate(self.jsonlines_raw.encode()) + self.assertEqual(out.splitlines(), self.jsonlines_expect.encode().splitlines()) + self.assertEqual(err, b'') + def test_help_flag(self): rc, out, err = assert_python_ok('-m', 'json.tool', '-h') self.assertEqual(rc, 0) diff --git a/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst b/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst new file mode 100644 index 00000000000000..fed2dadd06e54e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst @@ -0,0 +1 @@ +Add the --jsonlines option to json.tool. Patch by hongweipeng. \ No newline at end of file From 8d31c4fec0da09611210c1aa35cdf70e56c5453f Mon Sep 17 00:00:00 2001 From: HongWeipeng Date: Thu, 25 Oct 2018 17:13:47 +0800 Subject: [PATCH 2/4] code review --- Doc/library/json.rst | 6 ++++++ Doc/whatsnew/3.8.rst | 5 +++++ Lib/json/tool.py | 10 +++++----- .../Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst | 2 +- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8ce493d63995d4..3a3f97266d9042 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -714,6 +714,12 @@ Command line options .. versionadded:: 3.5 +.. cmdoption:: --json-lines + + Parse every input line as separate JSON object. + + .. versionadded:: 3.8 + .. cmdoption:: -h, --help Show the help message. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index bd3283caadb8b3..2122bb20623b9d 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -135,6 +135,11 @@ by right-clicking the button. (Contributed by Tal Einat in :issue:`1529353`.) The changes above have been backported to 3.7 maintenance releases. +json.tool +------- +Add option ``--json-lines`` to parse every input line as separate JSON object. + + os.path ------- diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 1b4a1a10954ccb..8c35f9a4bd2457 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -26,20 +26,20 @@ def main(): help='write the output of infile to outfile') parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') - parser.add_argument('--jsonlines', action='store_true', default=False, + parser.add_argument('--json-lines', action='store_true', default=False, help='parse input using the jsonlines format') options = parser.parse_args() infile = options.infile or sys.stdin outfile = options.outfile or sys.stdout sort_keys = options.sort_keys - jsonlines = options.jsonlines + json_lines = options.json_lines with infile: try: - if jsonlines: - objs = [json.loads(line) for line in infile] + if json_lines: + objs = (json.loads(line) for line in infile) else: - objs = [json.load(infile)] + objs = (json.load(infile), ) except ValueError as e: raise SystemExit(e) with outfile: diff --git a/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst b/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst index fed2dadd06e54e..de80e7cd7d118b 100644 --- a/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst +++ b/Misc/NEWS.d/next/Library/2018-10-23-14-46-47.bpo-31553.JxRkAW.rst @@ -1 +1 @@ -Add the --jsonlines option to json.tool. Patch by hongweipeng. \ No newline at end of file +Add the --json-lines option to json.tool. Patch by hongweipeng. \ No newline at end of file From 546b095700969a5ef72d184f9de983fea1357cd3 Mon Sep 17 00:00:00 2001 From: HongWeipeng Date: Fri, 26 Oct 2018 07:05:14 +0800 Subject: [PATCH 3/4] fix:avoid read infile after it close --- Lib/json/tool.py | 9 ++++----- Lib/test/test_json/test_tool.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 8c35f9a4bd2457..1d82bc82423670 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -34,18 +34,17 @@ def main(): outfile = options.outfile or sys.stdout sort_keys = options.sort_keys json_lines = options.json_lines - with infile: + with infile, outfile: try: if json_lines: objs = (json.loads(line) for line in infile) else: objs = (json.load(infile), ) + for obj in objs: + json.dump(obj, outfile, sort_keys=sort_keys, indent=4) + outfile.write('\n') except ValueError as e: raise SystemExit(e) - with outfile: - for obj in objs: - json.dump(obj, outfile, sort_keys=sort_keys, indent=4) - outfile.write('\n') if __name__ == '__main__': diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index c0ba66a321e15e..1e95bc79e5d189 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -115,7 +115,7 @@ def test_infile_outfile(self): self.assertEqual(err, b'') def test_jsonlines(self): - args = sys.executable, '-m', 'json.tool', '--jsonlines' + args = sys.executable, '-m', 'json.tool', '--json-lines' with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.jsonlines_raw.encode()) self.assertEqual(out.splitlines(), self.jsonlines_expect.encode().splitlines()) From 95933e26267f2f8aef7148f0ec82c90f05ce8cdb Mon Sep 17 00:00:00 2001 From: HongWeipeng Date: Fri, 26 Oct 2018 16:47:20 +0800 Subject: [PATCH 4/4] improve doc in whatsnew 3.8 --- Doc/whatsnew/3.8.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 2122bb20623b9d..d9a19d99427e64 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -136,9 +136,10 @@ The changes above have been backported to 3.7 maintenance releases. json.tool -------- -Add option ``--json-lines`` to parse every input line as separate JSON object. +--------- +Add option ``--json-lines`` to parse every input line as separate JSON object. +(Contributed by Weipeng Hong in :issue:`31553`.) os.path -------