diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 77075c1..b38bfab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,6 +37,8 @@ jobs: test: name: Python ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} + # The thread-safety tests would hang rather than fail on a lock regression. + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/CHANGES.md b/CHANGES.md index e4c3bba..6c4bd90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,16 @@ # CHANGES +## Unreleased + +- Fix crashes when one `UniversalDetector` is used from several threads at + once. uchardet runs without the GIL, so concurrent calls could corrupt its + state on both regular and free-threaded builds; the detector's methods now + hold a per-instance lock +- Fix `UniversalDetector` returning freed memory as its result, which could + raise `UnicodeDecodeError`, when `close()` follows `reset()` on a detector + created with `language_weights`. The bundled uchardet now drops its weighted + candidates on reset + ## 2.4.0 (2026-10-11) - Support Python 3.15, including the free-threaded CPython 3.15 build, with diff --git a/README.md b/README.md index 3b26fae..ce70cb2 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,13 @@ cChardet supports CPython 3.11 through 3.15, including the free-threaded CPython 3.14 and 3.15 builds. Each version is tested on Linux and Windows on both x86-64 and arm64, and on macOS on Apple silicon. +`detect()` and `detect_all()` can be called from many threads at once. A +`UniversalDetector` serializes its methods with a per-instance lock, so sharing +one between threads cannot crash the interpreter, but the calls interleave: +every feed goes into one stream, any thread's `reset()` or `close()` affects all +callers, and `done` and `result` wait for a feed in progress. Use one detector +per input. + ## Development Install [uv](https://docs.astral.sh/uv/), clone this repository with its submodules, diff --git a/pyproject.toml b/pyproject.toml index f196730..bd38e47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,8 @@ files = ["src/cchardet", "tests/test_typing.py"] [tool.pytest.ini_options] pythonpath = ["."] +# Dump every thread's stack if a test hangs, for example on a lock regression. +faulthandler_timeout = 300 [tool.setuptools.package-data] cchardet = ["*.pyi", "py.typed"] diff --git a/src/cchardet/_cchardet.pyx b/src/cchardet/_cchardet.pyx index b03fdc6..49356a1 100644 --- a/src/cchardet/_cchardet.pyx +++ b/src/cchardet/_cchardet.pyx @@ -2,6 +2,7 @@ #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 #cython: freethreading_compatible=True +cimport cython from libc.stddef cimport size_t cdef extern from *: @@ -44,16 +45,30 @@ cdef int _handle_data(uchardet_t ud, const_char_ptr data, size_t length) except return 0 -cdef void _apply_language_weights(uchardet_t ud, object language_weights): - cdef bytes language +cdef list _language_weight_pairs(object language_weights): + """Convert the weights to (bytes, float) pairs before touching a detector.""" + cdef list pairs = [] cdef float weight if language_weights is None: - return + return pairs for language_name, language_weight in language_weights.items(): - language = language_name.encode("ascii") weight = language_weight + pairs.append((language_name.encode("ascii"), weight)) + return pairs + + +cdef int _weigh_languages(uchardet_t ud, list pairs) except -1: + cdef bytes language + cdef float weight + + for language, weight in pairs: uchardet_weigh_language(ud, language, weight) + return 0 + + +cdef int _apply_language_weights(uchardet_t ud, object language_weights) except -1: + return _weigh_languages(ud, _language_weight_pairs(language_weights)) def detect_with_details(bytes msg, max_bytes=None, language_weights=None): cdef size_t length = len(msg) @@ -151,6 +166,14 @@ def detect_all(bytes msg, max_bytes=None, language_weights=None): uchardet_delete(ud) cdef class UniversalDetector: + # uchardet runs without the GIL, so concurrent calls on one detector could + # otherwise corrupt its native state even on regular builds. The lock + # serializes every method; acquiring it releases the GIL while waiting. + # It is not reentrant: methods must not call each other while holding it, + # and no Python code that could reach this detector (user mappings, + # finalizers run by a collection) may run under it. So Python objects are + # converted or built outside the lock wherever the normal path needs them. + cdef cython.pymutex _lock cdef uchardet_t _ud cdef int _done cdef int _closed @@ -162,56 +185,68 @@ cdef class UniversalDetector: self._ud = uchardet_new() def __init__(self, language_weights=None): - self._done = 0 - self._closed = 0 - self._detected_charset = b"" - self._detected_language = b"" - self._detected_confidence = 0.0 - _apply_language_weights(self._ud, language_weights) + cdef list weights = _language_weight_pairs(language_weights) + with self._lock: + self._done = 0 + self._closed = 0 + self._detected_charset = b"" + self._detected_language = b"" + self._detected_confidence = 0.0 + _weigh_languages(self._ud, weights) def reset(self): - self._done = 0 - self._closed = 0 - self._detected_charset = b"" - self._detected_language = b"" - self._detected_confidence = 0.0 - with nogil: - uchardet_reset(self._ud) + with self._lock: + self._done = 0 + self._closed = 0 + self._detected_charset = b"" + self._detected_language = b"" + self._detected_confidence = 0.0 + with nogil: + uchardet_reset(self._ud) def feed(self, bytes msg): cdef size_t length - cdef int result + cdef int result = 0 cdef const_char_ptr data - if self._closed: - return + with self._lock: + if self._closed: + return - length = len(msg) - if length > 0: - data = msg - with nogil: - result = _handle_data(self._ud, data, length) + length = len(msg) + if length > 0: + data = msg + with nogil: + result = _handle_data(self._ud, data, length) - if result != 0: - self._closed = 1 - raise RuntimeError("uchardet failed to handle data") - if uchardet_is_done(self._ud): - self.close() + if result != 0: + self._closed = 1 + elif uchardet_is_done(self._ud): + self._close_locked() + if result != 0: + raise RuntimeError("uchardet failed to handle data") def close(self): + with self._lock: + self._close_locked() + + @cython.final + cdef int _close_locked(self) except -1: cdef const_char_ptr language_ptr - if not self._closed: - with nogil: - uchardet_data_end(self._ud) + if self._closed: + return 0 + with nogil: + uchardet_data_end(self._ud) - self._detected_charset = uchardet_get_encoding(self._ud, 0) - language_ptr = uchardet_get_language(self._ud, 0) - if language_ptr != NULL: - self._detected_language = language_ptr - self._detected_confidence = uchardet_get_confidence(self._ud, 0) + self._detected_charset = uchardet_get_encoding(self._ud, 0) + language_ptr = uchardet_get_language(self._ud, 0) + if language_ptr != NULL: + self._detected_language = language_ptr + self._detected_confidence = uchardet_get_confidence(self._ud, 0) - self._done = 1 - self._closed = 1 + self._done = 1 + self._closed = 1 + return 0 def __dealloc__(self): if self._ud != NULL: @@ -219,15 +254,20 @@ cdef class UniversalDetector: @property def done(self): - return bool(self._done) + cdef int done + with self._lock: + done = self._done + return bool(done) @property def result(self): - if len(self._detected_charset): - return ( - self._detected_charset, - self._detected_language or None, - self._detected_confidence, - ) - else: - return None, None, None + cdef bytes charset + cdef bytes language + cdef float confidence + with self._lock: + charset = self._detected_charset + language = self._detected_language + confidence = self._detected_confidence + if len(charset): + return charset, language or None, confidence + return None, None, None diff --git a/src/ext/uchardet b/src/ext/uchardet index 7993e0a..1702c11 160000 --- a/src/ext/uchardet +++ b/src/ext/uchardet @@ -1 +1 @@ -Subproject commit 7993e0afe3b6c64f0ea3b43c7d4987118f6c5c12 +Subproject commit 1702c116f4f53dd40864551a75a2d4131057a545 diff --git a/tests/test_1.py b/tests/test_1.py index a223350..49ed384 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -2,6 +2,7 @@ import os import sys import sysconfig +import threading from concurrent.futures import ThreadPoolExecutor import cchardet @@ -31,6 +32,33 @@ SKIP_LIST_DEC.extend(SKIP_LIST_DETECT) +def hammer_from_threads(detector, sample, workers=8, steps=200): + """Call feed/reset/close/result on one detector from several threads at once.""" + + def close_and_read(): + detector.close() + detector.done + result = detector.result + assert (result["encoding"] is None) == (result["confidence"] is None) + + # Every worker cycles through all operations so they keep overlapping. + operations = [ + lambda: detector.feed(sample), + detector.reset, + lambda: detector.feed(sample), + close_and_read, + ] + barrier = threading.Barrier(workers) + + def hammer(index): + barrier.wait(timeout=30) + for step in range(steps): + operations[(index + step) % len(operations)]() + + with ThreadPoolExecutor(max_workers=workers) as executor: + list(executor.map(hammer, range(workers))) + + class TestCChardet: def test_free_threaded_build_keeps_gil_disabled(self): if sysconfig.get_config_var("Py_GIL_DISABLED"): @@ -134,6 +162,21 @@ def detect_results(sample): results = list(executor.map(detect_results, repeated_samples)) assert results == expected * 100 + def test_shared_detector_survives_concurrent_calls(self): + # The native calls run without the GIL. Before UniversalDetector held a + # per-instance lock, sharing one detector across threads corrupted + # uchardet's state and crashed the process, even on GIL builds. + with open(os.path.join(TESTDATA_DIR, "ja/euc-jp.txt"), "rb") as file: + sample = file.read() * 16 + for weights in (None, {"ja": 0.5}): + detector = cchardet.UniversalDetector(language_weights=weights) + hammer_from_threads(detector, sample) + + detector.reset() + detector.feed(sample) + detector.close() + assert detector.result == cchardet.detect(sample, language_weights=weights) + def test_detect_max_bytes(self): data = b"plain ASCII" + "日本語".encode() assert cchardet.detect(data, max_bytes=11)["encoding"] == "ASCII" @@ -174,6 +217,18 @@ def test_language_weights_are_opt_in_and_persist_across_reset(self): assert detector.result == weighted detector.reset() + def test_language_weights_survive_reset_without_new_data(self): + # uchardet's reset freed the candidates that its weighted view still + # pointed to, so closing again returned freed memory as the encoding. + detector = cchardet.UniversalDetector(language_weights={"en": 0.5}) + detector.feed(b"plain ASCII text") + detector.close() + assert detector.result["encoding"] == "ASCII" + + detector.reset() + detector.close() + assert detector.result == {"encoding": None, "confidence": None, "language": None} + def test_language_weights_are_validated(self): for weights in ({"english": 1.0}, {"en": -0.1}, {"en": 1.1}, {"en": float("nan")}): try: