From 6f7be26f539cbb86f35452652502d8eed14216da Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 12:56:45 +0900 Subject: [PATCH 01/18] Fix test_asyncio --- Lib/test/test_asyncio/test_base_events.py | 2 +- Lib/test/test_asyncio/test_events.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index e40e7999b64d15..5691d4250aca9e 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -2096,7 +2096,7 @@ def test_blocking_socket(self): def test_nonbinary_file(self): sock = self.make_socket() - with open(os_helper.TESTFN, 'r') as f: + with open(os_helper.TESTFN, encoding="utf-8") as f: with self.assertRaisesRegex(ValueError, "binary mode"): self.run_loop(self.loop.sock_sendfile(sock, f)) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 6523a79b4a6344..5511407ba53951 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1349,7 +1349,7 @@ def test_unclosed_pipe_transport(self): rpipe, wpipe = os.pipe() rpipeobj = io.open(rpipe, 'rb', 1024) - wpipeobj = io.open(wpipe, 'w', 1024) + wpipeobj = io.open(wpipe, 'w', 1024, encoding="utf-8") async def connect(): read_transport, _ = await loop.connect_read_pipe( From d0b154b4a8829ed111da5fdc3eabc64deaf4b0d0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 12:57:02 +0900 Subject: [PATCH 02/18] Fix test_bz2 --- Lib/test/test_bz2.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index ccc5e4df83a7a6..efed3a859ba217 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -922,14 +922,14 @@ def test_text_modes(self): for mode in ("wt", "xt"): if mode == "xt": unlink(self.filename) - with self.open(self.filename, mode) as f: + with self.open(self.filename, mode, encoding="ascii") as f: f.write(text) with open(self.filename, "rb") as f: file_data = ext_decompress(f.read()).decode("ascii") self.assertEqual(file_data, text_native_eol) - with self.open(self.filename, "rt") as f: + with self.open(self.filename, "rt", encoding="ascii") as f: self.assertEqual(f.read(), text) - with self.open(self.filename, "at") as f: + with self.open(self.filename, "at", encoding="ascii") as f: f.write(text) with open(self.filename, "rb") as f: file_data = ext_decompress(f.read()).decode("ascii") @@ -938,7 +938,8 @@ def test_text_modes(self): def test_x_mode(self): for mode in ("x", "xb", "xt"): unlink(self.filename) - with self.open(self.filename, mode) as f: + encoding = "utf-8" if "t" in mode else None + with self.open(self.filename, mode, encoding=encoding) as f: pass with self.assertRaises(FileExistsError): with self.open(self.filename, mode) as f: @@ -950,7 +951,7 @@ def test_fileobj(self): with self.open(BytesIO(self.DATA), "rb") as f: self.assertEqual(f.read(), self.TEXT) text = self.TEXT.decode("ascii") - with self.open(BytesIO(self.DATA), "rt") as f: + with self.open(BytesIO(self.DATA), "rt", encoding="utf-8") as f: self.assertEqual(f.read(), text) def test_bad_params(self): @@ -989,9 +990,9 @@ def test_encoding_error_handler(self): def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = self.TEXT.decode("ascii") - with self.open(self.filename, "wt", newline="\n") as f: + with self.open(self.filename, "wt", encoding="utf-8", newline="\n") as f: f.write(text) - with self.open(self.filename, "rt", newline="\r") as f: + with self.open(self.filename, "rt", encoding="utf-8", newline="\r") as f: self.assertEqual(f.readlines(), [text]) From ff78771d21ac6c7c8cbb3ca46f2a66bb02131d2a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:03:50 +0900 Subject: [PATCH 03/18] Fix test_math, test_cmath --- Lib/test/test_cmath.py | 2 +- Lib/test/test_math.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 668f27c8a08272..9bb939356144e6 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -60,7 +60,7 @@ class CMathTests(unittest.TestCase): test_functions.append(lambda x : cmath.log(14.-27j, x)) def setUp(self): - self.test_values = open(test_file) + self.test_values = open(test_file, encoding="utf-8") def tearDown(self): self.test_values.close() diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a6f6483f55d897..3d128749bec40f 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -130,7 +130,7 @@ def parse_mtestfile(fname): id fn arg -> expected [flag]* """ - with open(fname) as fp: + with open(fname, encoding="utf-8") as fp: for line in fp: # strip comments, and skip blank lines if '--' in line: @@ -153,7 +153,7 @@ def parse_testfile(fname): Empty lines or lines starting with -- are ignored yields id, fn, arg_real, arg_imag, exp_real, exp_imag """ - with open(fname) as fp: + with open(fname, encoding="utf-8") as fp: for line in fp: # skip comment lines and blank lines if line.startswith('--') or not line.strip(): From b920eee6c2467026ae05c4f5adb81ddf86f8fd12 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:04:58 +0900 Subject: [PATCH 04/18] Fix test_cmd_line --- Lib/test/test_cmd_line.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 25d3eec40c13de..375876768b29f5 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -512,7 +512,7 @@ def test_del___main__(self): # the dict whereas the module was destroyed filename = os_helper.TESTFN self.addCleanup(os_helper.unlink, filename) - with open(filename, "w") as script: + with open(filename, "w", encoding="utf-8") as script: print("import sys", file=script) print("del sys.modules['__main__']", file=script) assert_python_ok(filename) @@ -549,9 +549,9 @@ def test_isolatedmode(self): with os_helper.temp_cwd() as tmpdir: fake = os.path.join(tmpdir, "uuid.py") main = os.path.join(tmpdir, "main.py") - with open(fake, "w") as f: + with open(fake, "w", encoding="utf-8") as f: f.write("raise RuntimeError('isolated mode test')\n") - with open(main, "w") as f: + with open(main, "w", encoding="utf-8") as f: f.write("import uuid\n") f.write("print('ok')\n") self.assertRaises(subprocess.CalledProcessError, From 21cd4bb745a9d075bdedce2dea85ff31c33fda75 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:06:32 +0900 Subject: [PATCH 05/18] Fix test_cmd_line_script --- Lib/test/test_cmd_line_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f10ab40017a338..8f12b1e19bcfc6 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -400,7 +400,7 @@ def test_issue8202_dash_c_file_ignored(self): # does not alter the value of sys.path[0] with os_helper.temp_dir() as script_dir: with os_helper.change_cwd(path=script_dir): - with open("-c", "w") as f: + with open("-c", "w", encoding="utf-8") as f: f.write("data") rc, out, err = assert_python_ok('-c', 'import sys; print("sys.path[0]==%r" % sys.path[0])', @@ -416,7 +416,7 @@ def test_issue8202_dash_m_file_ignored(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'other') with os_helper.change_cwd(path=script_dir): - with open("-m", "w") as f: + with open("-m", "w", encoding="utf-8") as f: f.write("data") rc, out, err = assert_python_ok('-m', 'other', *example_args, __isolated=False) @@ -429,7 +429,7 @@ def test_issue20884(self): # will be failed. with os_helper.temp_dir() as script_dir: script_name = os.path.join(script_dir, "issue20884.py") - with open(script_name, "w", newline='\n') as f: + with open(script_name, "w", encoding="latin1", newline='\n') as f: f.write("#coding: iso-8859-1\n") f.write('"""\n') for _ in range(30): From a1113399a3c784447aeaf19dce33740776b1fb55 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:13:48 +0900 Subject: [PATCH 06/18] Fix test_compile --- Lib/test/test_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 591d5bb00b264e..c0de3bef5b6c9e 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -427,7 +427,7 @@ def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): fname = fname[:-1] - with open(fname, 'r') as f: + with open(fname, encoding='utf-8') as f: fcontents = f.read() sample_code = [ ['', 'x = 5'], From 55847ccd6f4142628d2082476467bb734c1566da Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:30:18 +0900 Subject: [PATCH 07/18] Fix test_contextlib --- Lib/test/test_contextlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 50943c1a17e9cb..453ef6c9f0832f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -316,13 +316,13 @@ def testWithOpen(self): tfn = tempfile.mktemp() try: f = None - with open(tfn, "w") as f: + with open(tfn, "w", encoding="utf-8") as f: self.assertFalse(f.closed) f.write("Booh\n") self.assertTrue(f.closed) f = None with self.assertRaises(ZeroDivisionError): - with open(tfn, "r") as f: + with open(tfn, "r", encoding="utf-8") as f: self.assertFalse(f.closed) self.assertEqual(f.read(), "Booh\n") 1 / 0 From 5129323eb55044d35ac67ab5a16f4742342fe5f9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:30:46 +0900 Subject: [PATCH 08/18] Fix test_profile --- Lib/test/test_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index 1bdf30acbb54b2..7de7d52ff83650 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -115,7 +115,7 @@ def test_run_profile_as_module(self): def test_output_file_when_changing_directory(self): with temp_dir() as tmpdir, change_cwd(tmpdir): os.mkdir('dest') - with open('demo.py', 'w') as f: + with open('demo.py', 'w', encoding="utf-8") as f: f.write('import os; os.chdir("dest")') assert_python_ok( From 1f53f7e5cdbb281a442156f125f917e4595d2d39 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 13:32:03 +0900 Subject: [PATCH 09/18] Fix ctypes/test/test_find --- Lib/ctypes/test/test_find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index 4a8a3820f3fe1c..1ff9d019b138a4 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -90,7 +90,7 @@ def test_find_on_libpath(self): srcname = os.path.join(d, 'dummy.c') libname = 'py_ctypes_test_dummy' dstname = os.path.join(d, 'lib%s.so' % libname) - with open(srcname, 'w') as f: + with open(srcname, 'wb') as f: pass self.assertTrue(os.path.exists(srcname)) # compile the file to a shared library From ac79763c50ea053fb47910d1c0322f7d1035ca8c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 17:07:48 +0900 Subject: [PATCH 10/18] Fix multibytecodec_support --- Lib/test/multibytecodec_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py index f76c0153f5ecf7..dbf0cc428e3ff6 100644 --- a/Lib/test/multibytecodec_support.py +++ b/Lib/test/multibytecodec_support.py @@ -296,7 +296,7 @@ def setUp(self): self.skipTest("Could not retrieve "+self.mapfileurl) def open_mapping_file(self): - return support.open_urlresource(self.mapfileurl) + return support.open_urlresource(self.mapfileurl, encoding="utf-8") def test_mapping_file(self): if self.mapfileurl.endswith('.xml'): From cc712018456c83102eee4ac5aeac686725557388 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Apr 2021 17:54:11 +0900 Subject: [PATCH 11/18] Fix test_multiprocessing --- Lib/test/_test_multiprocessing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index fd3b4303f034c1..ead92cfa2abfea 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -826,7 +826,7 @@ def test_stderr_flush(self): proc = self.Process(target=self._test_stderr_flush, args=(testfn,)) proc.start() proc.join() - with open(testfn, 'r') as f: + with open(testfn, encoding="utf-8") as f: err = f.read() # The whole traceback was printed self.assertIn("ZeroDivisionError", err) @@ -836,14 +836,14 @@ def test_stderr_flush(self): @classmethod def _test_stderr_flush(cls, testfn): fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) - sys.stderr = open(fd, 'w', closefd=False) + sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False) 1/0 # MARKER @classmethod def _test_sys_exit(cls, reason, testfn): fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) - sys.stderr = open(fd, 'w', closefd=False) + sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False) sys.exit(reason) def test_sys_exit(self): @@ -864,7 +864,7 @@ def test_sys_exit(self): join_process(p) self.assertEqual(p.exitcode, 1) - with open(testfn, 'r') as f: + with open(testfn, encoding="utf-8") as f: content = f.read() self.assertEqual(content.rstrip(), str(reason)) @@ -1118,7 +1118,7 @@ def test_task_done(self): def test_no_import_lock_contention(self): with os_helper.temp_cwd(): module_name = 'imported_by_an_imported_module' - with open(module_name + '.py', 'w') as f: + with open(module_name + '.py', 'w', encoding="utf-8") as f: f.write("""if 1: import multiprocessing From 8a6119bcb4e9c0c3f45ecec24d7e85e56fdfb1c5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 08:49:30 +0900 Subject: [PATCH 12/18] Fix test_configparser --- Lib/test/test_configparser.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 80a9f179ee2bbc..9373a62072ef47 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -714,31 +714,31 @@ def test_read_returns_file_list(self): file1 = support.findfile("cfgparser.1") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() - parsed_files = cf.read([file1, "nonexistent-file"]) + parsed_files = cf.read([file1, "nonexistent-file"], encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a filename: cf = self.newconfig() - parsed_files = cf.read(file1) + parsed_files = cf.read(file1, encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1)) + parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1]) + parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() - parsed_files = cf.read(["nonexistent-file"]) + parsed_files = cf.read(["nonexistent-file"], encoding="utf-8") self.assertEqual(parsed_files, []) # check when we pass no files: cf = self.newconfig() - parsed_files = cf.read([]) + parsed_files = cf.read([], encoding="utf-8") self.assertEqual(parsed_files, []) def test_read_returns_file_list_with_bytestring_path(self): @@ -747,15 +747,15 @@ def test_read_returns_file_list_with_bytestring_path(self): file1_bytestring = support.findfile("cfgparser.1").encode() # check when passing an existing bytestring path cf = self.newconfig() - parsed_files = cf.read(file1_bytestring) + parsed_files = cf.read(file1_bytestring, encoding="utf-8") self.assertEqual(parsed_files, [file1_bytestring]) # check when passing an non-existing bytestring path cf = self.newconfig() - parsed_files = cf.read(b'nonexistent-file') + parsed_files = cf.read(b'nonexistent-file', encoding="utf-8") self.assertEqual(parsed_files, []) # check when passing both an existing and non-existing bytestring path cf = self.newconfig() - parsed_files = cf.read([file1_bytestring, b'nonexistent-file']) + parsed_files = cf.read([file1_bytestring, b'nonexistent-file'], encoding="utf-8") self.assertEqual(parsed_files, [file1_bytestring]) # shared by subclasses @@ -1064,7 +1064,7 @@ def setUp(self): cf.add_section(s) for j in range(10): cf.set(s, 'lovely_spam{}'.format(j), self.wonderful_spam) - with open(os_helper.TESTFN, 'w') as f: + with open(os_helper.TESTFN, 'w', encoding="utf-8") as f: cf.write(f) def tearDown(self): @@ -1074,7 +1074,7 @@ def test_dominating_multiline_values(self): # We're reading from file because this is where the code changed # during performance updates in Python 3.2 cf_from_file = self.newconfig() - with open(os_helper.TESTFN) as f: + with open(os_helper.TESTFN, encoding="utf-8") as f: cf_from_file.read_file(f) self.assertEqual(cf_from_file.get('section8', 'lovely_spam4'), self.wonderful_spam.replace('\t\n', '\n')) @@ -1473,7 +1473,7 @@ def fromstring(self, string, defaults=None): class FakeFile: def __init__(self): file_path = support.findfile("cfgparser.1") - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: self.lines = f.readlines() self.lines.reverse() @@ -1500,7 +1500,7 @@ def test_file(self): pass # unfortunately we can't test bytes on this path for file_path in file_paths: parser = configparser.ConfigParser() - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: parser.read_file(f) self.assertIn("Foo Bar", parser) self.assertIn("foo", parser["Foo Bar"]) From 2616431fbcdac792b3ddcc9390db6ab365c15d6f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 08:56:19 +0900 Subject: [PATCH 13/18] Fix test_csv --- Lib/test/test_csv.py | 56 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index a98707ce3caed7..2411a91110dd43 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -133,7 +133,7 @@ def test_writer_dialect_attrs(self): def _write_test(self, fields, expect, **kwargs): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, **kwargs) writer.writerow(fields) fileobj.seek(0) @@ -141,7 +141,7 @@ def _write_test(self, fields, expect, **kwargs): expect + writer.dialect.lineterminator) def _write_error_test(self, exc, fields, **kwargs): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, **kwargs) with self.assertRaises(exc): writer.writerow(fields) @@ -232,7 +232,7 @@ def write(self, buf): writer = csv.writer(BrokenFile()) self.assertRaises(OSError, writer.writerows, [['a']]) - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) self.assertRaises(TypeError, writer.writerows, None) writer.writerows([['a', 'b'], ['c', 'd']]) @@ -240,26 +240,26 @@ def write(self, buf): self.assertEqual(fileobj.read(), "a,b\r\nc,d\r\n") def test_writerows_with_none(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) writer.writerows([['a', None], [None, 'd']]) fileobj.seek(0) self.assertEqual(fileobj.read(), "a,\r\n,d\r\n") - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) writer.writerows([[None], ['a']]) fileobj.seek(0) self.assertEqual(fileobj.read(), '""\r\na\r\n') - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) writer.writerows([['a'], [None]]) fileobj.seek(0) self.assertEqual(fileobj.read(), 'a\r\n""\r\n') def test_writerows_errors(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) self.assertRaises(TypeError, writer.writerows, None) self.assertRaises(OSError, writer.writerows, BadIterable()) @@ -270,7 +270,7 @@ def test_writerows_errors(self): def test_writerows_legacy_strings(self): import _testcapi c = _testcapi.unicode_legacy_string('a') - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) writer.writerows([[c]]) fileobj.seek(0) @@ -367,7 +367,7 @@ def test_read_linenum(self): self.assertEqual(r.line_num, 3) def test_roundtrip_quoteed_newlines(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) rows = [['a\nb','b'],['c','x\r\nd']] writer.writerows(rows) @@ -376,7 +376,7 @@ def test_roundtrip_quoteed_newlines(self): self.assertEqual(row, rows[i]) def test_roundtrip_escaped_unquoted_newlines(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") rows = [['a\nb','b'],['c','x\r\nd']] writer.writerows(rows) @@ -432,7 +432,7 @@ class space(csv.excel): quoting = csv.QUOTE_NONE escapechar = "\\" - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("abc def\nc1ccccc1 benzene\n") fileobj.seek(0) reader = csv.reader(fileobj, dialect=space()) @@ -493,7 +493,7 @@ def test_pickle(self): class TestCsvBase(unittest.TestCase): def readerAssertEqual(self, input, expected_result): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: fileobj.write(input) fileobj.seek(0) reader = csv.reader(fileobj, dialect = self.dialect) @@ -501,7 +501,7 @@ def readerAssertEqual(self, input, expected_result): self.assertEqual(fields, expected_result) def writerAssertEqual(self, input, expected_result): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, dialect = self.dialect) writer.writerows(input) fileobj.seek(0) @@ -643,13 +643,13 @@ class TestDictFields(unittest.TestCase): ### "long" means the row is longer than the number of fieldnames ### "short" means there are fewer elements in the row than fieldnames def test_writeheader_return_value(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.DictWriter(fileobj, fieldnames = ["f1", "f2", "f3"]) writeheader_return_value = writer.writeheader() self.assertEqual(writeheader_return_value, 10) def test_write_simple_dict(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.DictWriter(fileobj, fieldnames = ["f1", "f2", "f3"]) writer.writeheader() fileobj.seek(0) @@ -674,7 +674,7 @@ def test_write_no_fields(self): self.assertRaises(TypeError, csv.DictWriter, fileobj) def test_write_fields_not_in_fieldnames(self): - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.DictWriter(fileobj, fieldnames = ["f1", "f2", "f3"]) # Of special note is the non-string key (issue 19449) with self.assertRaises(ValueError) as cx: @@ -704,7 +704,7 @@ def test_write_field_not_in_field_names_ignore(self): self.assertEqual(fileobj.getvalue(), "1,2\r\n") def test_read_dict_fields(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("1,2,abc\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj, @@ -712,7 +712,7 @@ def test_read_dict_fields(self): self.assertEqual(next(reader), {"f1": '1', "f2": '2', "f3": 'abc'}) def test_read_dict_no_fieldnames(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("f1,f2,f3\r\n1,2,abc\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj) @@ -722,7 +722,7 @@ def test_read_dict_no_fieldnames(self): # Two test cases to make sure existing ways of implicitly setting # fieldnames continue to work. Both arise from discussion in issue3436. def test_read_dict_fieldnames_from_file(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("f1,f2,f3\r\n1,2,abc\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj, @@ -732,7 +732,7 @@ def test_read_dict_fieldnames_from_file(self): def test_read_dict_fieldnames_chain(self): import itertools - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("f1,f2,f3\r\n1,2,abc\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj) @@ -742,7 +742,7 @@ def test_read_dict_fieldnames_chain(self): self.assertEqual(row, {"f1": '1', "f2": '2', "f3": 'abc'}) def test_read_long(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("1,2,abc,4,5,6\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj, @@ -751,7 +751,7 @@ def test_read_long(self): None: ["abc", "4", "5", "6"]}) def test_read_long_with_rest(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("1,2,abc,4,5,6\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj, @@ -760,7 +760,7 @@ def test_read_long_with_rest(self): "_rest": ["abc", "4", "5", "6"]}) def test_read_long_with_rest_no_fieldnames(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("f1,f2\r\n1,2,abc,4,5,6\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj, restkey="_rest") @@ -769,7 +769,7 @@ def test_read_long_with_rest_no_fieldnames(self): "_rest": ["abc", "4", "5", "6"]}) def test_read_short(self): - with TemporaryFile("w+") as fileobj: + with TemporaryFile("w+", encoding="utf-8") as fileobj: fileobj.write("1,2,abc,4,5,6\r\n1,2,abc\r\n") fileobj.seek(0) reader = csv.DictReader(fileobj, @@ -818,7 +818,7 @@ def test_int_write(self): contents = [(20-i) for i in range(20)] a = array.array('i', contents) - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, dialect="excel") writer.writerow(a) expected = ",".join([str(i) for i in a])+"\r\n" @@ -829,7 +829,7 @@ def test_double_write(self): import array contents = [(20-i)*0.1 for i in range(20)] a = array.array('d', contents) - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, dialect="excel") writer.writerow(a) expected = ",".join([str(i) for i in a])+"\r\n" @@ -840,7 +840,7 @@ def test_float_write(self): import array contents = [(20-i)*0.1 for i in range(20)] a = array.array('f', contents) - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, dialect="excel") writer.writerow(a) expected = ",".join([str(i) for i in a])+"\r\n" @@ -851,7 +851,7 @@ def test_char_write(self): import array, string a = array.array('u', string.ascii_letters) - with TemporaryFile("w+", newline='') as fileobj: + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, dialect="excel") writer.writerow(a) expected = ",".join(a)+"\r\n" From 26a2b13ad8ca6141cfe489996027b4510901d32b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 08:57:08 +0900 Subject: [PATCH 14/18] Fix test_dbm_dumb --- Lib/test/test_dbm_dumb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index 071cb845013b53..ddaffb4757bc49 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -232,7 +232,7 @@ def test_create_new(self): self.assertEqual(f.keys(), []) def test_eval(self): - with open(_fname + '.dir', 'w') as stream: + with open(_fname + '.dir', 'w', encoding="utf-8") as stream: stream.write("str(print('Hacked!')), 0\n") with support.captured_stdout() as stdout: with self.assertRaises(ValueError): From 19dc2e44a5ee5d39198ddb60c9dc6b7076e57697 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 08:58:22 +0900 Subject: [PATCH 15/18] Fix test_decimal --- Lib/test/test_decimal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 6aa4a9b31a5f24..179a9ea7042fbf 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -289,7 +289,7 @@ def eval_file(self, file): global skip_expected if skip_expected: raise unittest.SkipTest - with open(file) as f: + with open(file, encoding="utf-8") as f: for line in f: line = line.replace('\r\n', '').replace('\n', '') #print line From 27738c84f08c7adad7e64a0628be50c873f72756 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 08:59:00 +0900 Subject: [PATCH 16/18] Fix test_difflib --- Lib/test/test_difflib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 42ac1fdcd81cdc..9a24e00f64b4a4 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -241,7 +241,7 @@ def test_html_diff(self): #with open('test_difflib_expect.html','w') as fp: # fp.write(actual) - with open(findfile('test_difflib_expect.html')) as fp: + with open(findfile('test_difflib_expect.html'), encoding="utf-8") as fp: self.assertEqual(actual, fp.read()) def test_recursion_limit(self): From d12c06912a95776a9efe6cbb67da1ba90b0c65a9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 09:03:11 +0900 Subject: [PATCH 17/18] os.fdopen() calls io.text_encoding() --- Lib/os.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 05e9c32c5a7117..06f421a650014d 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1020,11 +1020,12 @@ def __iter__(self): __all__.append("popen") # Supply os.fdopen() -def fdopen(fd, *args, **kwargs): +def fdopen(fd, mode="r", buffering=-1, encoding=None, *args, **kwargs): if not isinstance(fd, int): raise TypeError("invalid fd type (%s, expected integer)" % type(fd)) import io - return io.open(fd, *args, **kwargs) + encoding = io.text_encoding(encoding) + return io.open(fd, mode, buffering, encoding, *args, **kwargs) # For testing purposes, make sure the function is available when the C From 4ecbd81be8d7023c16b075609f3e7bdeed19902c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 3 Apr 2021 13:04:31 +0900 Subject: [PATCH 18/18] Fix os.fdopen() --- Lib/os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 06f421a650014d..ea09e8c7e10489 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1024,7 +1024,8 @@ def fdopen(fd, mode="r", buffering=-1, encoding=None, *args, **kwargs): if not isinstance(fd, int): raise TypeError("invalid fd type (%s, expected integer)" % type(fd)) import io - encoding = io.text_encoding(encoding) + if "b" not in mode: + encoding = io.text_encoding(encoding) return io.open(fd, mode, buffering, encoding, *args, **kwargs)