From 7babf38a74d3e1797447754b36803cef1f64b22b Mon Sep 17 00:00:00 2001 From: "Benjamin A. Beasley" Date: Tue, 2 Jun 2026 13:40:49 +0100 Subject: [PATCH 1/2] Test with chardet 6.0.0.post1 --- pyproject.toml | 2 +- tests/httpx2/client/test_client.py | 10 ++++++++-- tests/httpx2/models/test_responses.py | 8 ++++---- uv.lock | 8 ++++---- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 58464b08..470ea8dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dev = [ "httpx2[brotli,cli,http2,socks,zstd]", "httpcore2[asyncio,trio,http2,socks]", # Tests - "chardet==5.2.0", + "chardet==6.0.0.post1", "coverage[toml]==7.10.6", "cryptography==46.0.7", "pytest>=9.0.3", diff --git a/tests/httpx2/client/test_client.py b/tests/httpx2/client/test_client.py index 7ad44a3f..4981857d 100644 --- a/tests/httpx2/client/test_client.py +++ b/tests/httpx2/client/test_client.py @@ -429,7 +429,10 @@ def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response: assert response.status_code == 200 assert response.reason_phrase == "OK" - assert response.encoding == "ISO-8859-1" + # The text is short enough to be ambiguous: chardet may validly detect + # a different encoding than the one we actually used, and the following + # expectation may need to change when we upgrade chardet. + assert response.encoding == "WINDOWS-1252" assert response.text == text @@ -456,5 +459,8 @@ def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response: assert response.status_code == 200 assert response.reason_phrase == "OK" - assert response.encoding == "ISO-8859-1" + # The text is short enough to be ambiguous: chardet may validly detect + # a different encoding than the one we actually used, and the following + # expectation may need to change when we upgrade chardet. + assert response.encoding == "WINDOWS-1252" assert response.text == text diff --git a/tests/httpx2/models/test_responses.py b/tests/httpx2/models/test_responses.py index 5a9fbbbd..0ae58243 100644 --- a/tests/httpx2/models/test_responses.py +++ b/tests/httpx2/models/test_responses.py @@ -1012,10 +1012,10 @@ def test_response_decode_text_using_autodetect() -> None: assert response.status_code == 200 assert response.reason_phrase == "OK" - # The encoded byte string is consistent with either ISO-8859-1 or - # WINDOWS-1252. Versions <6.0 of chardet claim the former, while chardet - # 6.0 detects the latter. - assert response.encoding in ("ISO-8859-1", "WINDOWS-1252") + # The text is short enough to be ambiguous: chardet may validly detect a + # different encoding than the one we actually used, and the following + # expectation may need to change when we upgrade chardet. + assert response.encoding == "WINDOWS-1252" assert response.text == text diff --git a/uv.lock b/uv.lock index 92449446..7b0c8f57 100644 --- a/uv.lock +++ b/uv.lock @@ -27,7 +27,7 @@ bench = [ ] dev = [ { name = "build", specifier = "==1.3.0" }, - { name = "chardet", specifier = "==5.2.0" }, + { name = "chardet", specifier = "==6.0.0.post1" }, { name = "coverage", extras = ["toml"], specifier = "==7.10.6" }, { name = "cryptography", specifier = "==46.0.7" }, { name = "httpcore2", extras = ["asyncio", "trio", "http2", "socks"], editable = "src/httpcore2" }, @@ -443,11 +443,11 @@ wheels = [ [[package]] name = "chardet" -version = "5.2.0" +version = "6.0.0.post1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/42/fb9436c103a881a377e34b9f58d77b5f503461c702ff654ebe86151bcfe9/chardet-6.0.0.post1.tar.gz", hash = "sha256:6b78048c3c97c7b2ed1fbad7a18f76f5a6547f7d34dbab536cc13887c9a92fa4", size = 12521798, upload-time = "2026-02-22T15:09:17.925Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/5de54f632c2de53cd3415b3703383d5fff43a94cbc0567ef362515261a21/chardet-6.0.0.post1-py3-none-any.whl", hash = "sha256:c894a36800549adf7bb5f2af47033281b75fdfcd2aa0f0243be0ad22a52e2dcb", size = 627245, upload-time = "2026-02-22T15:09:15.876Z" }, ] [[package]] From 2591068de596bb4a453170bf2a06d288f76c7ef0 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Tue, 2 Jun 2026 15:49:03 +0200 Subject: [PATCH 2/2] Use unambiguous WINDOWS-1252 sample text in autodetect tests The previous French sample used only accented letters in bytes 0xA0-0xFF, where ISO-8859-1 and WINDOWS-1252 are byte-identical, so chardet was free to report either encoding. Adding curly quotes and an em dash (bytes 0x80-0x9F, control characters in ISO-8859-1) makes the byte string unambiguously WINDOWS-1252 across chardet versions, so the expectation no longer needs to change on upgrade. --- tests/httpx2/client/test_client.py | 30 +++++++++++++-------------- tests/httpx2/models/test_responses.py | 15 +++++++------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/tests/httpx2/client/test_client.py b/tests/httpx2/client/test_client.py index 4981857d..83de5c3d 100644 --- a/tests/httpx2/client/test_client.py +++ b/tests/httpx2/client/test_client.py @@ -410,17 +410,19 @@ def test_client_decode_text_using_autodetect() -> None: # encoding autodetection to be used when no "Content-Type: text/plain; charset=..." # info is present. # - # Here we have some french text encoded with ISO-8859-1, rather than UTF-8. + # Here we have some french text encoded with WINDOWS-1252, rather than UTF-8. + # The curly quotes and em dash occupy bytes 0x80-0x9F, which are control + # characters in ISO-8859-1, so the encoding is unambiguously WINDOWS-1252. text = ( - "Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains " + "Non-seulement Despréaux ne se trompait pas — mais de tous les écrivains " "que la France a produits, sans excepter Voltaire lui-même, imprégné de " - "l'esprit anglais par son séjour à Londres, c'est incontestablement " - "Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la " + "l’esprit anglais par son séjour à Londres, c’est incontestablement " + "“Molière” ou Poquelin qui reproduit avec l’exactitude la plus vive et la " "plus complète le fond du génie français." ) def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response: - content = text.encode("ISO-8859-1") + content = text.encode("WINDOWS-1252") return httpx2.Response(200, content=content) transport = httpx2.MockTransport(cp1252_but_no_content_type) @@ -429,9 +431,6 @@ def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response: assert response.status_code == 200 assert response.reason_phrase == "OK" - # The text is short enough to be ambiguous: chardet may validly detect - # a different encoding than the one we actually used, and the following - # expectation may need to change when we upgrade chardet. assert response.encoding == "WINDOWS-1252" assert response.text == text @@ -440,17 +439,19 @@ def test_client_decode_text_using_explicit_encoding() -> None: # Ensure that a 'default_encoding="..."' on the response is used for text decoding # when no "Content-Type: text/plain; charset=..."" info is present. # - # Here we have some french text encoded with ISO-8859-1, rather than UTF-8. + # Here we have some french text encoded with WINDOWS-1252, rather than UTF-8. + # The curly quotes and em dash occupy bytes 0x80-0x9F, which are control + # characters in ISO-8859-1, so the encoding is unambiguously WINDOWS-1252. text = ( - "Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains " + "Non-seulement Despréaux ne se trompait pas — mais de tous les écrivains " "que la France a produits, sans excepter Voltaire lui-même, imprégné de " - "l'esprit anglais par son séjour à Londres, c'est incontestablement " - "Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la " + "l’esprit anglais par son séjour à Londres, c’est incontestablement " + "“Molière” ou Poquelin qui reproduit avec l’exactitude la plus vive et la " "plus complète le fond du génie français." ) def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response: - content = text.encode("ISO-8859-1") + content = text.encode("WINDOWS-1252") return httpx2.Response(200, content=content) transport = httpx2.MockTransport(cp1252_but_no_content_type) @@ -459,8 +460,5 @@ def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response: assert response.status_code == 200 assert response.reason_phrase == "OK" - # The text is short enough to be ambiguous: chardet may validly detect - # a different encoding than the one we actually used, and the following - # expectation may need to change when we upgrade chardet. assert response.encoding == "WINDOWS-1252" assert response.text == text diff --git a/tests/httpx2/models/test_responses.py b/tests/httpx2/models/test_responses.py index 0ae58243..e4958d59 100644 --- a/tests/httpx2/models/test_responses.py +++ b/tests/httpx2/models/test_responses.py @@ -999,22 +999,21 @@ def test_response_decode_text_using_autodetect() -> None: # encoding autodetection to be used when no "Content-Type: text/plain; charset=..." # info is present. # - # Here we have some french text encoded with ISO-8859-1, rather than UTF-8. + # Here we have some french text encoded with WINDOWS-1252, rather than UTF-8. + # The curly quotes and em dash occupy bytes 0x80-0x9F, which are control + # characters in ISO-8859-1, so the encoding is unambiguously WINDOWS-1252. text = ( - "Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains " + "Non-seulement Despréaux ne se trompait pas — mais de tous les écrivains " "que la France a produits, sans excepter Voltaire lui-même, imprégné de " - "l'esprit anglais par son séjour à Londres, c'est incontestablement " - "Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la " + "l’esprit anglais par son séjour à Londres, c’est incontestablement " + "“Molière” ou Poquelin qui reproduit avec l’exactitude la plus vive et la " "plus complète le fond du génie français." ) - content = text.encode("ISO-8859-1") + content = text.encode("WINDOWS-1252") response = httpx2.Response(200, content=content, default_encoding=autodetect) assert response.status_code == 200 assert response.reason_phrase == "OK" - # The text is short enough to be ambiguous: chardet may validly detect a - # different encoding than the one we actually used, and the following - # expectation may need to change when we upgrade chardet. assert response.encoding == "WINDOWS-1252" assert response.text == text