Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions Doc/library/codecs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1409,8 +1409,13 @@ encodings.
| palmos | | Encoding of PalmOS 3.5. |
+--------------------+---------+---------------------------+
| punycode | | Implement :rfc:`3492`. |
| | | Stateful codecs are not |
| | | supported. |
| | | The incremental encoder |
| | | and decoder buffer the |
| | | input until the final |
| | | call. Stream readers and |
| | | writers, and text files |
| | | opened for writing, are |
| | | not supported. |
| | | |
| | | .. warning:: |
| | | |
Expand Down
23 changes: 15 additions & 8 deletions Lib/encodings/punycode.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,15 +226,22 @@ def decode(self, input, errors='strict'):
res = punycode_decode(input, errors)
return res, len(input)

class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return punycode_encode(input)
class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
def _buffer_encode(self, input, errors, final):
if not final:
# The encoding of each character depends on the whole string,
# so keep the input until the last call
return (b'', 0)
return (punycode_encode(input), len(input))

class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
if self.errors not in ('strict', 'replace', 'ignore'):
raise UnicodeError(f"Unsupported error handling: {self.errors}")
return punycode_decode(input, self.errors)
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
def _buffer_decode(self, input, errors, final):
if errors not in ('strict', 'replace', 'ignore'):
raise UnicodeError(f"Unsupported error handling: {errors}")
if not final:
# Keep the input until the last call, as in IncrementalEncoder
return ('', 0)
return (punycode_decode(input, errors), len(input))

class StreamWriter(Codec,codecs.StreamWriter):
pass
Expand Down
226 changes: 138 additions & 88 deletions Lib/test/test_codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -1358,7 +1358,7 @@ def test_errors(self):
print(repr(i))


class PunycodeTest(unittest.TestCase):
class PunycodeTest(unittest.TestCase, MixInCheckStateHandling):
def test_encode(self):
for uni, puny in punycode_testcases:
# Need to convert both strings to lower case, since
Expand All @@ -1377,26 +1377,27 @@ def test_decode(self):
puny = puny.decode("ascii").encode("ascii")
self.assertEqual(uni, puny.decode("punycode"))

invalid_decode_testcases = [
(b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")),
(b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
(b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")),
(b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")),
(b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")),
(b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")),
(b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")),
(b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")),
(b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")),
(b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")),
(b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")),
# A delimiter not preceded by any basic code point (RFC 3492, 6.2)
(b"-", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
(b"-80a1acny", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
(b"xn--w&", "ignore", "xn-"),
(b"-80a1acny", "ignore", ""),
]

def test_decode_invalid(self):
testcases = [
(b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")),
(b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
(b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")),
(b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")),
(b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")),
(b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")),
(b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")),
(b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")),
(b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")),
(b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")),
(b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")),
# A delimiter not preceded by any basic code point (RFC 3492, 6.2)
(b"-", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
(b"-80a1acny", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
(b"xn--w&", "ignore", "xn-"),
(b"-80a1acny", "ignore", ""),
]
for puny, errors, expected in testcases:
for puny, errors, expected in self.invalid_decode_testcases:
with self.subTest(puny=puny, errors=errors):
if isinstance(expected, Exception):
with self.assertRaises(UnicodeDecodeError) as cm:
Expand All @@ -1409,6 +1410,58 @@ def test_decode_invalid(self):
else:
self.assertEqual(puny.decode("punycode", errors), expected)

def test_incremental_encode(self):
for uni, puny in punycode_testcases:
with self.subTest(uni=uni):
expected = uni.encode("punycode")
encoder = codecs.getincrementalencoder("punycode")()
result = b"".join(encoder.encode(c) for c in uni)
result += encoder.encode("", True)
self.assertEqual(encoder.getstate(), 0)
self.assertEqual(result, expected)
self.assertEqual(
b"".join(codecs.iterencode(uni, "punycode")), expected)

def test_incremental_decode(self):
for uni, puny in punycode_testcases:
with self.subTest(puny=puny):
chunks = [bytes([c]) for c in puny]
decoder = codecs.getincrementaldecoder("punycode")()
result = "".join(decoder.decode(c) for c in chunks)
result += decoder.decode(b"", True)
self.assertEqual(decoder.getstate(), (b"", 0))
self.assertEqual(result, uni)
self.assertEqual(
"".join(codecs.iterdecode(chunks, "punycode")), uni)

def test_incremental_decode_invalid(self):
for puny, errors, expected in self.invalid_decode_testcases:
with self.subTest(puny=puny, errors=errors):
chunks = (bytes([c]) for c in puny)
if isinstance(expected, Exception):
with self.assertRaises(UnicodeDecodeError) as cm:
list(codecs.iterdecode(chunks, "punycode", errors))
exc = cm.exception
self.assertEqual(exc.encoding, expected.encoding)
self.assertEqual(exc.object, puny)
self.assertEqual(exc.start, expected.start)
self.assertEqual(exc.end, expected.end)
else:
self.assertEqual(
"".join(codecs.iterdecode(chunks, "punycode", errors)),
expected)

decoder = codecs.getincrementaldecoder("punycode")("surrogateescape")
with self.assertRaises(UnicodeError):
decoder.decode(b"abc")

def test_incremental_state(self):
for uni, puny in punycode_testcases:
with self.subTest(uni=uni):
self.check_state_handling_decode("punycode", uni, puny)
self.check_state_handling_encode(
"punycode", uni, uni.encode("punycode"))


utf_7_imap_testcases = [
# (unicode, modified UTF-7)
Expand Down Expand Up @@ -2292,7 +2345,7 @@ def test_basic(self):
# to work:
# "undefined"

# The following encodings don't work in stateful mode
# The following encodings don't work with stream readers and writers
broken_unicode_with_stateful = [
"punycode",
]
Expand Down Expand Up @@ -2335,89 +2388,87 @@ def test_basics(self):
decodedresult += reader.read()
self.assertEqual(decodedresult, s, "encoding=%r" % encoding)

if encoding not in broken_unicode_with_stateful:
# check incremental decoder/encoder and iterencode()/iterdecode()
# check incremental decoder/encoder and iterencode()/iterdecode()
try:
encoder = codecs.getincrementalencoder(encoding)()
except LookupError: # no IncrementalEncoder
pass
else:
# check incremental decoder/encoder
encodedresult = b""
for c in s:
encodedresult += encoder.encode(c)
encodedresult += encoder.encode("", True)
decoder = codecs.getincrementaldecoder(encoding)()
decodedresult = ""
for c in encodedresult:
decodedresult += decoder.decode(bytes([c]))
decodedresult += decoder.decode(b"", True)
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)

# check iterencode()/iterdecode()
result = "".join(codecs.iterdecode(
codecs.iterencode(s, encoding), encoding))
self.assertEqual(result, s, "encoding=%r" % encoding)

# check iterencode()/iterdecode() with empty string
result = "".join(codecs.iterdecode(
codecs.iterencode("", encoding), encoding))
self.assertEqual(result, "")

if encoding not in ("idna", "mbcs", "punycode"):
# check incremental decoder/encoder with errors argument
try:
encoder = codecs.getincrementalencoder(encoding)()
encoder = codecs.getincrementalencoder(encoding)("ignore")
except LookupError: # no IncrementalEncoder
pass
else:
# check incremental decoder/encoder
encodedresult = b""
for c in s:
encodedresult += encoder.encode(c)
encodedresult += encoder.encode("", True)
decoder = codecs.getincrementaldecoder(encoding)()
decodedresult = ""
for c in encodedresult:
decodedresult += decoder.decode(bytes([c]))
decodedresult += decoder.decode(b"", True)
encodedresult = b"".join(encoder.encode(c) for c in s)
decoder = codecs.getincrementaldecoder(encoding)("ignore")
decodedresult = "".join(decoder.decode(bytes([c]))
for c in encodedresult)
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)

# check iterencode()/iterdecode()
result = "".join(codecs.iterdecode(
codecs.iterencode(s, encoding), encoding))
self.assertEqual(result, s, "encoding=%r" % encoding)

# check iterencode()/iterdecode() with empty string
result = "".join(codecs.iterdecode(
codecs.iterencode("", encoding), encoding))
self.assertEqual(result, "")

if encoding not in ("idna", "mbcs"):
# check incremental decoder/encoder with errors argument
try:
encoder = codecs.getincrementalencoder(encoding)("ignore")
except LookupError: # no IncrementalEncoder
pass
else:
encodedresult = b"".join(encoder.encode(c) for c in s)
decoder = codecs.getincrementaldecoder(encoding)("ignore")
decodedresult = "".join(decoder.decode(bytes([c]))
for c in encodedresult)
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)

@support.cpython_only
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
def test_basics_capi(self):
s = "abc123" # all codecs should be able to encode these
for encoding in all_unicode_encodings:
if encoding not in broken_unicode_with_stateful:
# check incremental decoder/encoder (fetched via the C API)
# check incremental decoder/encoder (fetched via the C API)
try:
cencoder = _testlimitedcapi.codec_incrementalencoder(encoding)
except LookupError: # no IncrementalEncoder
pass
else:
# check C API
encodedresult = b""
for c in s:
encodedresult += cencoder.encode(c)
encodedresult += cencoder.encode("", True)
cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding)
decodedresult = ""
for c in encodedresult:
decodedresult += cdecoder.decode(bytes([c]))
decodedresult += cdecoder.decode(b"", True)
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)

if encoding not in ("idna", "mbcs", "punycode"):
# check incremental decoder/encoder with errors argument
try:
cencoder = _testlimitedcapi.codec_incrementalencoder(encoding)
cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore")
except LookupError: # no IncrementalEncoder
pass
else:
# check C API
encodedresult = b""
for c in s:
encodedresult += cencoder.encode(c)
encodedresult += cencoder.encode("", True)
cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding)
decodedresult = ""
for c in encodedresult:
decodedresult += cdecoder.decode(bytes([c]))
decodedresult += cdecoder.decode(b"", True)
encodedresult = b"".join(cencoder.encode(c) for c in s)
cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore")
decodedresult = "".join(cdecoder.decode(bytes([c]))
for c in encodedresult)
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)

if encoding not in ("idna", "mbcs"):
# check incremental decoder/encoder with errors argument
try:
cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore")
except LookupError: # no IncrementalEncoder
pass
else:
encodedresult = b"".join(cencoder.encode(c) for c in s)
cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore")
decodedresult = "".join(cdecoder.decode(bytes([c]))
for c in encodedresult)
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)

def test_seek(self):
# all codecs should be able to encode these
s = "%s\n%s\n" % (100*"abc123", 100*"def456")
Expand Down Expand Up @@ -2455,9 +2506,8 @@ def test_decoder_state(self):
# Check that getstate() and setstate() handle the state properly
u = "abc123"
for encoding in all_unicode_encodings:
if encoding not in broken_unicode_with_stateful:
self.check_state_handling_decode(encoding, u, u.encode(encoding))
self.check_state_handling_encode(encoding, u, u.encode(encoding))
self.check_state_handling_decode(encoding, u, u.encode(encoding))
self.check_state_handling_encode(encoding, u, u.encode(encoding))


class CharmapTest(unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Fix the ``punycode`` incremental encoder and decoder, which gave wrong
results when the input was passed in more than one chunk, for example
through :func:`codecs.iterencode` or :func:`codecs.iterdecode`. They now
buffer the input until the final call, so text files opened for writing
with this encoding no longer write any output. The incremental decoder no
longer accepts :class:`str`.
Loading