From 7ee43e11d0eb62d47a0870ff117c9c4c6f152aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yusufhan=20K=C4=B1r=C3=A7ova?= <59140093+kircova@users.noreply.github.com> Date: Sat, 21 Nov 2026 02:40:06 +0000 Subject: [PATCH] gh-160308: Buffer punycode incremental codec input until the final call --- Doc/library/codecs.rst | 9 +- Lib/encodings/punycode.py | 23 +- Lib/test/test_codecs.py | 226 +++++++++++------- ...-11-20-20-05-05.gh-issue-160308.5SR27q.rst | 6 + 4 files changed, 166 insertions(+), 98 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-20-20-05-05.gh-issue-160308.5SR27q.rst diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index f19407818528e2..620657d1b31261 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1409,8 +1409,13 @@ encodings. | palmos | | Encoding of PalmOS 3.5. | +--------------------+---------+---------------------------+ | punycode | | Implement :rfc:`3492`. | -| | | Stateful codecs are not | -| | | supported. | +| | | The incremental encoder | +| | | and decoder buffer the | +| | | input until the final | +| | | call. Stream readers and | +| | | writers, and text files | +| | | opened for writing, are | +| | | not supported. | | | | | | | | .. warning:: | | | | | diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1da012229837f0..d15ca2e0496d41 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -226,15 +226,22 @@ def decode(self, input, errors='strict'): res = punycode_decode(input, errors) return res, len(input) -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return punycode_encode(input) +class IncrementalEncoder(codecs.BufferedIncrementalEncoder): + def _buffer_encode(self, input, errors, final): + if not final: + # The encoding of each character depends on the whole string, + # so keep the input until the last call + return (b'', 0) + return (punycode_encode(input), len(input)) -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError(f"Unsupported error handling: {self.errors}") - return punycode_decode(input, self.errors) +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + if errors not in ('strict', 'replace', 'ignore'): + raise UnicodeError(f"Unsupported error handling: {errors}") + if not final: + # Keep the input until the last call, as in IncrementalEncoder + return ('', 0) + return (punycode_decode(input, errors), len(input)) class StreamWriter(Codec,codecs.StreamWriter): pass diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 4492e4d874189f..34cd559f2e9a7b 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1358,7 +1358,7 @@ def test_errors(self): print(repr(i)) -class PunycodeTest(unittest.TestCase): +class PunycodeTest(unittest.TestCase, MixInCheckStateHandling): def test_encode(self): for uni, puny in punycode_testcases: # Need to convert both strings to lower case, since @@ -1377,26 +1377,27 @@ def test_decode(self): puny = puny.decode("ascii").encode("ascii") self.assertEqual(uni, puny.decode("punycode")) + invalid_decode_testcases = [ + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), + # A delimiter not preceded by any basic code point (RFC 3492, 6.2) + (b"-", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"-80a1acny", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"xn--w&", "ignore", "xn-"), + (b"-80a1acny", "ignore", ""), + ] + def test_decode_invalid(self): - testcases = [ - (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), - (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), - (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), - (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), - (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), - (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), - (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), - (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), - (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), - (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), - (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), - # A delimiter not preceded by any basic code point (RFC 3492, 6.2) - (b"-", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), - (b"-80a1acny", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), - (b"xn--w&", "ignore", "xn-"), - (b"-80a1acny", "ignore", ""), - ] - for puny, errors, expected in testcases: + for puny, errors, expected in self.invalid_decode_testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): with self.assertRaises(UnicodeDecodeError) as cm: @@ -1409,6 +1410,58 @@ def test_decode_invalid(self): else: self.assertEqual(puny.decode("punycode", errors), expected) + def test_incremental_encode(self): + for uni, puny in punycode_testcases: + with self.subTest(uni=uni): + expected = uni.encode("punycode") + encoder = codecs.getincrementalencoder("punycode")() + result = b"".join(encoder.encode(c) for c in uni) + result += encoder.encode("", True) + self.assertEqual(encoder.getstate(), 0) + self.assertEqual(result, expected) + self.assertEqual( + b"".join(codecs.iterencode(uni, "punycode")), expected) + + def test_incremental_decode(self): + for uni, puny in punycode_testcases: + with self.subTest(puny=puny): + chunks = [bytes([c]) for c in puny] + decoder = codecs.getincrementaldecoder("punycode")() + result = "".join(decoder.decode(c) for c in chunks) + result += decoder.decode(b"", True) + self.assertEqual(decoder.getstate(), (b"", 0)) + self.assertEqual(result, uni) + self.assertEqual( + "".join(codecs.iterdecode(chunks, "punycode")), uni) + + def test_incremental_decode_invalid(self): + for puny, errors, expected in self.invalid_decode_testcases: + with self.subTest(puny=puny, errors=errors): + chunks = (bytes([c]) for c in puny) + if isinstance(expected, Exception): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode(chunks, "punycode", errors)) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + else: + self.assertEqual( + "".join(codecs.iterdecode(chunks, "punycode", errors)), + expected) + + decoder = codecs.getincrementaldecoder("punycode")("surrogateescape") + with self.assertRaises(UnicodeError): + decoder.decode(b"abc") + + def test_incremental_state(self): + for uni, puny in punycode_testcases: + with self.subTest(uni=uni): + self.check_state_handling_decode("punycode", uni, puny) + self.check_state_handling_encode( + "punycode", uni, uni.encode("punycode")) + utf_7_imap_testcases = [ # (unicode, modified UTF-7) @@ -2292,7 +2345,7 @@ def test_basic(self): # to work: # "undefined" -# The following encodings don't work in stateful mode +# The following encodings don't work with stream readers and writers broken_unicode_with_stateful = [ "punycode", ] @@ -2335,89 +2388,87 @@ def test_basics(self): decodedresult += reader.read() self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - if encoding not in broken_unicode_with_stateful: - # check incremental decoder/encoder and iterencode()/iterdecode() + # check incremental decoder/encoder and iterencode()/iterdecode() + try: + encoder = codecs.getincrementalencoder(encoding)() + except LookupError: # no IncrementalEncoder + pass + else: + # check incremental decoder/encoder + encodedresult = b"" + for c in s: + encodedresult += encoder.encode(c) + encodedresult += encoder.encode("", True) + decoder = codecs.getincrementaldecoder(encoding)() + decodedresult = "" + for c in encodedresult: + decodedresult += decoder.decode(bytes([c])) + decodedresult += decoder.decode(b"", True) + self.assertEqual(decodedresult, s, + "encoding=%r" % encoding) + + # check iterencode()/iterdecode() + result = "".join(codecs.iterdecode( + codecs.iterencode(s, encoding), encoding)) + self.assertEqual(result, s, "encoding=%r" % encoding) + + # check iterencode()/iterdecode() with empty string + result = "".join(codecs.iterdecode( + codecs.iterencode("", encoding), encoding)) + self.assertEqual(result, "") + + if encoding not in ("idna", "mbcs", "punycode"): + # check incremental decoder/encoder with errors argument try: - encoder = codecs.getincrementalencoder(encoding)() + encoder = codecs.getincrementalencoder(encoding)("ignore") except LookupError: # no IncrementalEncoder pass else: - # check incremental decoder/encoder - encodedresult = b"" - for c in s: - encodedresult += encoder.encode(c) - encodedresult += encoder.encode("", True) - decoder = codecs.getincrementaldecoder(encoding)() - decodedresult = "" - for c in encodedresult: - decodedresult += decoder.decode(bytes([c])) - decodedresult += decoder.decode(b"", True) + encodedresult = b"".join(encoder.encode(c) for c in s) + decoder = codecs.getincrementaldecoder(encoding)("ignore") + decodedresult = "".join(decoder.decode(bytes([c])) + for c in encodedresult) self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - # check iterencode()/iterdecode() - result = "".join(codecs.iterdecode( - codecs.iterencode(s, encoding), encoding)) - self.assertEqual(result, s, "encoding=%r" % encoding) - - # check iterencode()/iterdecode() with empty string - result = "".join(codecs.iterdecode( - codecs.iterencode("", encoding), encoding)) - self.assertEqual(result, "") - - if encoding not in ("idna", "mbcs"): - # check incremental decoder/encoder with errors argument - try: - encoder = codecs.getincrementalencoder(encoding)("ignore") - except LookupError: # no IncrementalEncoder - pass - else: - encodedresult = b"".join(encoder.encode(c) for c in s) - decoder = codecs.getincrementaldecoder(encoding)("ignore") - decodedresult = "".join(decoder.decode(bytes([c])) - for c in encodedresult) - self.assertEqual(decodedresult, s, - "encoding=%r" % encoding) - @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: - if encoding not in broken_unicode_with_stateful: - # check incremental decoder/encoder (fetched via the C API) + # check incremental decoder/encoder (fetched via the C API) + try: + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) + except LookupError: # no IncrementalEncoder + pass + else: + # check C API + encodedresult = b"" + for c in s: + encodedresult += cencoder.encode(c) + encodedresult += cencoder.encode("", True) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) + decodedresult = "" + for c in encodedresult: + decodedresult += cdecoder.decode(bytes([c])) + decodedresult += cdecoder.decode(b"", True) + self.assertEqual(decodedresult, s, + "encoding=%r" % encoding) + + if encoding not in ("idna", "mbcs", "punycode"): + # check incremental decoder/encoder with errors argument try: - cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: - # check C API - encodedresult = b"" - for c in s: - encodedresult += cencoder.encode(c) - encodedresult += cencoder.encode("", True) - cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) - decodedresult = "" - for c in encodedresult: - decodedresult += cdecoder.decode(bytes([c])) - decodedresult += cdecoder.decode(b"", True) + encodedresult = b"".join(cencoder.encode(c) for c in s) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") + decodedresult = "".join(cdecoder.decode(bytes([c])) + for c in encodedresult) self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - if encoding not in ("idna", "mbcs"): - # check incremental decoder/encoder with errors argument - try: - cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") - except LookupError: # no IncrementalEncoder - pass - else: - encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") - decodedresult = "".join(cdecoder.decode(bytes([c])) - for c in encodedresult) - self.assertEqual(decodedresult, s, - "encoding=%r" % encoding) - def test_seek(self): # all codecs should be able to encode these s = "%s\n%s\n" % (100*"abc123", 100*"def456") @@ -2455,9 +2506,8 @@ def test_decoder_state(self): # Check that getstate() and setstate() handle the state properly u = "abc123" for encoding in all_unicode_encodings: - if encoding not in broken_unicode_with_stateful: - self.check_state_handling_decode(encoding, u, u.encode(encoding)) - self.check_state_handling_encode(encoding, u, u.encode(encoding)) + self.check_state_handling_decode(encoding, u, u.encode(encoding)) + self.check_state_handling_encode(encoding, u, u.encode(encoding)) class CharmapTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-11-20-20-05-05.gh-issue-160308.5SR27q.rst b/Misc/NEWS.d/next/Library/2026-11-20-20-05-05.gh-issue-160308.5SR27q.rst new file mode 100644 index 00000000000000..bfebe4b17fb418 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-20-20-05-05.gh-issue-160308.5SR27q.rst @@ -0,0 +1,6 @@ +Fix the ``punycode`` incremental encoder and decoder, which gave wrong +results when the input was passed in more than one chunk, for example +through :func:`codecs.iterencode` or :func:`codecs.iterdecode`. They now +buffer the input until the final call, so text files opened for writing +with this encoding no longer write any output. The incremental decoder no +longer accepts :class:`str`.