From e05e0f86fd377468e2958d34af27400df9ef22d0 Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Sat, 14 Nov 2026 02:15:39 +0900 Subject: [PATCH 1/2] fix: drop weighted candidates when the detector resets With language weights set, UniversalDetector.close() after reset() with no new data returned freed memory as the encoding, usually raising UnicodeDecodeError. uchardet's Reset() freed the candidate strings while its weighed_candidates still borrowed them. Bump the uchardet submodule to PyYoshi/uchardet@1702c11, which clears them on reset and adds a lifecycle test that the Native ASan / UBSan job runs, and add a Python regression test. --- CHANGES.md | 7 +++++++ src/ext/uchardet | 2 +- tests/test_1.py | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e4c3bba..1eedfa9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ # CHANGES +## Unreleased + +- Fix `UniversalDetector` returning freed memory as its result, which could + raise `UnicodeDecodeError`, when `close()` follows `reset()` on a detector + created with `language_weights`. The bundled uchardet now drops its weighted + candidates on reset + ## 2.4.0 (2026-10-11) - Support Python 3.15, including the free-threaded CPython 3.15 build, with diff --git a/src/ext/uchardet b/src/ext/uchardet index 7993e0a..1702c11 160000 --- a/src/ext/uchardet +++ b/src/ext/uchardet @@ -1 +1 @@ -Subproject commit 7993e0afe3b6c64f0ea3b43c7d4987118f6c5c12 +Subproject commit 1702c116f4f53dd40864551a75a2d4131057a545 diff --git a/tests/test_1.py b/tests/test_1.py index a223350..f4bd177 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -174,6 +174,18 @@ def test_language_weights_are_opt_in_and_persist_across_reset(self): assert detector.result == weighted detector.reset() + def test_language_weights_survive_reset_without_new_data(self): + # uchardet's reset freed the candidates that its weighted view still + # pointed to, so closing again returned freed memory as the encoding. + detector = cchardet.UniversalDetector(language_weights={"en": 0.5}) + detector.feed(b"plain ASCII text") + detector.close() + assert detector.result["encoding"] == "ASCII" + + detector.reset() + detector.close() + assert detector.result == {"encoding": None, "confidence": None, "language": None} + def test_language_weights_are_validated(self): for weights in ({"english": 1.0}, {"en": -0.1}, {"en": 1.1}, {"en": float("nan")}): try: From 194bab4d4dbdbf16f3e4d595b144247a3ca278ec Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Sat, 14 Nov 2026 02:15:46 +0900 Subject: [PATCH 2/2] fix: serialize UniversalDetector calls with a per-instance lock UniversalDetector runs uchardet without the GIL, so concurrent calls on one detector could enter the same native state at once. That corrupted memory and crashed the process on both regular and free-threaded builds, for example with free(): invalid pointer in uchardet_reset() or a segfault in close(). Hold a cython.pymutex in every method and property. It stays held across the nogil sections, unlike cython.critical_section, and waiting for it releases the GIL or detaches the thread state. Because it is not reentrant, feed() completes early detection through a final _close_locked(), and Python objects are converted or built outside the lock so that finalizers or user mappings cannot re-enter it. Add a test that drives one shared detector from eight threads, document the threading behavior, and bound hangs in CI with a job timeout and pytest's faulthandler_timeout. --- .github/workflows/test.yml | 2 + CHANGES.md | 4 ++ README.md | 7 ++ pyproject.toml | 2 + src/cchardet/_cchardet.pyx | 138 ++++++++++++++++++++++++------------- tests/test_1.py | 43 ++++++++++++ 6 files changed, 147 insertions(+), 49 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 77075c1..b38bfab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,6 +37,8 @@ jobs: test: name: Python ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} + # The thread-safety tests would hang rather than fail on a lock regression. + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/CHANGES.md b/CHANGES.md index 1eedfa9..6c4bd90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ## Unreleased +- Fix crashes when one `UniversalDetector` is used from several threads at + once. uchardet runs without the GIL, so concurrent calls could corrupt its + state on both regular and free-threaded builds; the detector's methods now + hold a per-instance lock - Fix `UniversalDetector` returning freed memory as its result, which could raise `UnicodeDecodeError`, when `close()` follows `reset()` on a detector created with `language_weights`. The bundled uchardet now drops its weighted diff --git a/README.md b/README.md index 3b26fae..ce70cb2 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,13 @@ cChardet supports CPython 3.11 through 3.15, including the free-threaded CPython 3.14 and 3.15 builds. Each version is tested on Linux and Windows on both x86-64 and arm64, and on macOS on Apple silicon. +`detect()` and `detect_all()` can be called from many threads at once. A +`UniversalDetector` serializes its methods with a per-instance lock, so sharing +one between threads cannot crash the interpreter, but the calls interleave: +every feed goes into one stream, any thread's `reset()` or `close()` affects all +callers, and `done` and `result` wait for a feed in progress. Use one detector +per input. + ## Development Install [uv](https://docs.astral.sh/uv/), clone this repository with its submodules, diff --git a/pyproject.toml b/pyproject.toml index f196730..bd38e47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,8 @@ files = ["src/cchardet", "tests/test_typing.py"] [tool.pytest.ini_options] pythonpath = ["."] +# Dump every thread's stack if a test hangs, for example on a lock regression. +faulthandler_timeout = 300 [tool.setuptools.package-data] cchardet = ["*.pyi", "py.typed"] diff --git a/src/cchardet/_cchardet.pyx b/src/cchardet/_cchardet.pyx index b03fdc6..49356a1 100644 --- a/src/cchardet/_cchardet.pyx +++ b/src/cchardet/_cchardet.pyx @@ -2,6 +2,7 @@ #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 #cython: freethreading_compatible=True +cimport cython from libc.stddef cimport size_t cdef extern from *: @@ -44,16 +45,30 @@ cdef int _handle_data(uchardet_t ud, const_char_ptr data, size_t length) except return 0 -cdef void _apply_language_weights(uchardet_t ud, object language_weights): - cdef bytes language +cdef list _language_weight_pairs(object language_weights): + """Convert the weights to (bytes, float) pairs before touching a detector.""" + cdef list pairs = [] cdef float weight if language_weights is None: - return + return pairs for language_name, language_weight in language_weights.items(): - language = language_name.encode("ascii") weight = language_weight + pairs.append((language_name.encode("ascii"), weight)) + return pairs + + +cdef int _weigh_languages(uchardet_t ud, list pairs) except -1: + cdef bytes language + cdef float weight + + for language, weight in pairs: uchardet_weigh_language(ud, language, weight) + return 0 + + +cdef int _apply_language_weights(uchardet_t ud, object language_weights) except -1: + return _weigh_languages(ud, _language_weight_pairs(language_weights)) def detect_with_details(bytes msg, max_bytes=None, language_weights=None): cdef size_t length = len(msg) @@ -151,6 +166,14 @@ def detect_all(bytes msg, max_bytes=None, language_weights=None): uchardet_delete(ud) cdef class UniversalDetector: + # uchardet runs without the GIL, so concurrent calls on one detector could + # otherwise corrupt its native state even on regular builds. The lock + # serializes every method; acquiring it releases the GIL while waiting. + # It is not reentrant: methods must not call each other while holding it, + # and no Python code that could reach this detector (user mappings, + # finalizers run by a collection) may run under it. So Python objects are + # converted or built outside the lock wherever the normal path needs them. + cdef cython.pymutex _lock cdef uchardet_t _ud cdef int _done cdef int _closed @@ -162,56 +185,68 @@ cdef class UniversalDetector: self._ud = uchardet_new() def __init__(self, language_weights=None): - self._done = 0 - self._closed = 0 - self._detected_charset = b"" - self._detected_language = b"" - self._detected_confidence = 0.0 - _apply_language_weights(self._ud, language_weights) + cdef list weights = _language_weight_pairs(language_weights) + with self._lock: + self._done = 0 + self._closed = 0 + self._detected_charset = b"" + self._detected_language = b"" + self._detected_confidence = 0.0 + _weigh_languages(self._ud, weights) def reset(self): - self._done = 0 - self._closed = 0 - self._detected_charset = b"" - self._detected_language = b"" - self._detected_confidence = 0.0 - with nogil: - uchardet_reset(self._ud) + with self._lock: + self._done = 0 + self._closed = 0 + self._detected_charset = b"" + self._detected_language = b"" + self._detected_confidence = 0.0 + with nogil: + uchardet_reset(self._ud) def feed(self, bytes msg): cdef size_t length - cdef int result + cdef int result = 0 cdef const_char_ptr data - if self._closed: - return + with self._lock: + if self._closed: + return - length = len(msg) - if length > 0: - data = msg - with nogil: - result = _handle_data(self._ud, data, length) + length = len(msg) + if length > 0: + data = msg + with nogil: + result = _handle_data(self._ud, data, length) - if result != 0: - self._closed = 1 - raise RuntimeError("uchardet failed to handle data") - if uchardet_is_done(self._ud): - self.close() + if result != 0: + self._closed = 1 + elif uchardet_is_done(self._ud): + self._close_locked() + if result != 0: + raise RuntimeError("uchardet failed to handle data") def close(self): + with self._lock: + self._close_locked() + + @cython.final + cdef int _close_locked(self) except -1: cdef const_char_ptr language_ptr - if not self._closed: - with nogil: - uchardet_data_end(self._ud) + if self._closed: + return 0 + with nogil: + uchardet_data_end(self._ud) - self._detected_charset = uchardet_get_encoding(self._ud, 0) - language_ptr = uchardet_get_language(self._ud, 0) - if language_ptr != NULL: - self._detected_language = language_ptr - self._detected_confidence = uchardet_get_confidence(self._ud, 0) + self._detected_charset = uchardet_get_encoding(self._ud, 0) + language_ptr = uchardet_get_language(self._ud, 0) + if language_ptr != NULL: + self._detected_language = language_ptr + self._detected_confidence = uchardet_get_confidence(self._ud, 0) - self._done = 1 - self._closed = 1 + self._done = 1 + self._closed = 1 + return 0 def __dealloc__(self): if self._ud != NULL: @@ -219,15 +254,20 @@ cdef class UniversalDetector: @property def done(self): - return bool(self._done) + cdef int done + with self._lock: + done = self._done + return bool(done) @property def result(self): - if len(self._detected_charset): - return ( - self._detected_charset, - self._detected_language or None, - self._detected_confidence, - ) - else: - return None, None, None + cdef bytes charset + cdef bytes language + cdef float confidence + with self._lock: + charset = self._detected_charset + language = self._detected_language + confidence = self._detected_confidence + if len(charset): + return charset, language or None, confidence + return None, None, None diff --git a/tests/test_1.py b/tests/test_1.py index f4bd177..49ed384 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -2,6 +2,7 @@ import os import sys import sysconfig +import threading from concurrent.futures import ThreadPoolExecutor import cchardet @@ -31,6 +32,33 @@ SKIP_LIST_DEC.extend(SKIP_LIST_DETECT) +def hammer_from_threads(detector, sample, workers=8, steps=200): + """Call feed/reset/close/result on one detector from several threads at once.""" + + def close_and_read(): + detector.close() + detector.done + result = detector.result + assert (result["encoding"] is None) == (result["confidence"] is None) + + # Every worker cycles through all operations so they keep overlapping. + operations = [ + lambda: detector.feed(sample), + detector.reset, + lambda: detector.feed(sample), + close_and_read, + ] + barrier = threading.Barrier(workers) + + def hammer(index): + barrier.wait(timeout=30) + for step in range(steps): + operations[(index + step) % len(operations)]() + + with ThreadPoolExecutor(max_workers=workers) as executor: + list(executor.map(hammer, range(workers))) + + class TestCChardet: def test_free_threaded_build_keeps_gil_disabled(self): if sysconfig.get_config_var("Py_GIL_DISABLED"): @@ -134,6 +162,21 @@ def detect_results(sample): results = list(executor.map(detect_results, repeated_samples)) assert results == expected * 100 + def test_shared_detector_survives_concurrent_calls(self): + # The native calls run without the GIL. Before UniversalDetector held a + # per-instance lock, sharing one detector across threads corrupted + # uchardet's state and crashed the process, even on GIL builds. + with open(os.path.join(TESTDATA_DIR, "ja/euc-jp.txt"), "rb") as file: + sample = file.read() * 16 + for weights in (None, {"ja": 0.5}): + detector = cchardet.UniversalDetector(language_weights=weights) + hammer_from_threads(detector, sample) + + detector.reset() + detector.feed(sample) + detector.close() + assert detector.result == cchardet.detect(sample, language_weights=weights) + def test_detect_max_bytes(self): data = b"plain ASCII" + "日本語".encode() assert cchardet.detect(data, max_bytes=11)["encoding"] == "ASCII"