Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ jobs:
test:
name: Python ${{ matrix.python-version }} / ${{ matrix.os }}
runs-on: ${{ matrix.os }}
# The thread-safety tests would hang rather than fail on a lock regression.
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
Expand Down
11 changes: 11 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
# CHANGES

## Unreleased

- Fix crashes when one `UniversalDetector` is used from several threads at
once. uchardet runs without the GIL, so concurrent calls could corrupt its
state on both regular and free-threaded builds; the detector's methods now
hold a per-instance lock
- Fix `UniversalDetector` returning freed memory as its result, which could
raise `UnicodeDecodeError`, when `close()` follows `reset()` on a detector
created with `language_weights`. The bundled uchardet now drops its weighted
candidates on reset

## 2.4.0 (2026-10-11)

- Support Python 3.15, including the free-threaded CPython 3.15 build, with
Expand Down
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ cChardet supports CPython 3.11 through 3.15, including the free-threaded
CPython 3.14 and 3.15 builds. Each version is tested on Linux and Windows on
both x86-64 and arm64, and on macOS on Apple silicon.

`detect()` and `detect_all()` can be called from many threads at once. A
`UniversalDetector` serializes its methods with a per-instance lock, so sharing
one between threads cannot crash the interpreter, but the calls interleave:
every feed goes into one stream, any thread's `reset()` or `close()` affects all
callers, and `done` and `result` wait for a feed in progress. Use one detector
per input.

## Development

Install [uv](https://docs.astral.sh/uv/), clone this repository with its submodules,
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ files = ["src/cchardet", "tests/test_typing.py"]

[tool.pytest.ini_options]
pythonpath = ["."]
# Dump every thread's stack if a test hangs, for example on a lock regression.
faulthandler_timeout = 300

[tool.setuptools.package-data]
cchardet = ["*.pyi", "py.typed"]
Expand Down
138 changes: 89 additions & 49 deletions src/cchardet/_cchardet.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#cython: embedsignature=True, c_string_encoding=ascii, language_level=3
#cython: freethreading_compatible=True

cimport cython
from libc.stddef cimport size_t

cdef extern from *:
Expand Down Expand Up @@ -44,16 +45,30 @@ cdef int _handle_data(uchardet_t ud, const_char_ptr data, size_t length) except
return 0


cdef void _apply_language_weights(uchardet_t ud, object language_weights):
cdef bytes language
cdef list _language_weight_pairs(object language_weights):
"""Convert the weights to (bytes, float) pairs before touching a detector."""
cdef list pairs = []
cdef float weight

if language_weights is None:
return
return pairs
for language_name, language_weight in language_weights.items():
language = language_name.encode("ascii")
weight = language_weight
pairs.append((language_name.encode("ascii"), weight))
return pairs


cdef int _weigh_languages(uchardet_t ud, list pairs) except -1:
cdef bytes language
cdef float weight

for language, weight in pairs:
uchardet_weigh_language(ud, language, weight)
return 0


cdef int _apply_language_weights(uchardet_t ud, object language_weights) except -1:
return _weigh_languages(ud, _language_weight_pairs(language_weights))

def detect_with_details(bytes msg, max_bytes=None, language_weights=None):
cdef size_t length = len(msg)
Expand Down Expand Up @@ -151,6 +166,14 @@ def detect_all(bytes msg, max_bytes=None, language_weights=None):
uchardet_delete(ud)

cdef class UniversalDetector:
# uchardet runs without the GIL, so concurrent calls on one detector could
# otherwise corrupt its native state even on regular builds. The lock
# serializes every method; acquiring it releases the GIL while waiting.
# It is not reentrant: methods must not call each other while holding it,
# and no Python code that could reach this detector (user mappings,
# finalizers run by a collection) may run under it. So Python objects are
# converted or built outside the lock wherever the normal path needs them.
cdef cython.pymutex _lock
cdef uchardet_t _ud
cdef int _done
cdef int _closed
Expand All @@ -162,72 +185,89 @@ cdef class UniversalDetector:
self._ud = uchardet_new()

def __init__(self, language_weights=None):
self._done = 0
self._closed = 0
self._detected_charset = b""
self._detected_language = b""
self._detected_confidence = 0.0
_apply_language_weights(self._ud, language_weights)
cdef list weights = _language_weight_pairs(language_weights)
with self._lock:
self._done = 0
self._closed = 0
self._detected_charset = b""
self._detected_language = b""
self._detected_confidence = 0.0
_weigh_languages(self._ud, weights)

def reset(self):
self._done = 0
self._closed = 0
self._detected_charset = b""
self._detected_language = b""
self._detected_confidence = 0.0
with nogil:
uchardet_reset(self._ud)
with self._lock:
self._done = 0
self._closed = 0
self._detected_charset = b""
self._detected_language = b""
self._detected_confidence = 0.0
with nogil:
uchardet_reset(self._ud)

def feed(self, bytes msg):
cdef size_t length
cdef int result
cdef int result = 0
cdef const_char_ptr data

if self._closed:
return
with self._lock:
if self._closed:
return

length = len(msg)
if length > 0:
data = msg
with nogil:
result = _handle_data(self._ud, data, length)
length = len(msg)
if length > 0:
data = msg
with nogil:
result = _handle_data(self._ud, data, length)

if result != 0:
self._closed = 1
raise RuntimeError("uchardet failed to handle data")
if uchardet_is_done(self._ud):
self.close()
if result != 0:
self._closed = 1
elif uchardet_is_done(self._ud):
self._close_locked()
if result != 0:
raise RuntimeError("uchardet failed to handle data")

def close(self):
with self._lock:
self._close_locked()

@cython.final
cdef int _close_locked(self) except -1:
cdef const_char_ptr language_ptr
if not self._closed:
with nogil:
uchardet_data_end(self._ud)
if self._closed:
return 0
with nogil:
uchardet_data_end(self._ud)

self._detected_charset = uchardet_get_encoding(self._ud, 0)
language_ptr = uchardet_get_language(self._ud, 0)
if language_ptr != NULL:
self._detected_language = language_ptr
self._detected_confidence = uchardet_get_confidence(self._ud, 0)
self._detected_charset = uchardet_get_encoding(self._ud, 0)
language_ptr = uchardet_get_language(self._ud, 0)
if language_ptr != NULL:
self._detected_language = language_ptr
self._detected_confidence = uchardet_get_confidence(self._ud, 0)

self._done = 1
self._closed = 1
self._done = 1
self._closed = 1
return 0

def __dealloc__(self):
if self._ud != NULL:
uchardet_delete(self._ud)

@property
def done(self):
return bool(self._done)
cdef int done
with self._lock:
done = self._done
return bool(done)

@property
def result(self):
if len(self._detected_charset):
return (
self._detected_charset,
self._detected_language or None,
self._detected_confidence,
)
else:
return None, None, None
cdef bytes charset
cdef bytes language
cdef float confidence
with self._lock:
charset = self._detected_charset
language = self._detected_language
confidence = self._detected_confidence
if len(charset):
return charset, language or None, confidence
return None, None, None
2 changes: 1 addition & 1 deletion src/ext/uchardet
55 changes: 55 additions & 0 deletions tests/test_1.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os
import sys
import sysconfig
import threading
from concurrent.futures import ThreadPoolExecutor

import cchardet
Expand Down Expand Up @@ -31,6 +32,33 @@
SKIP_LIST_DEC.extend(SKIP_LIST_DETECT)


def hammer_from_threads(detector, sample, workers=8, steps=200):
"""Call feed/reset/close/result on one detector from several threads at once."""

def close_and_read():
detector.close()
detector.done
result = detector.result
assert (result["encoding"] is None) == (result["confidence"] is None)

# Every worker cycles through all operations so they keep overlapping.
operations = [
lambda: detector.feed(sample),
detector.reset,
lambda: detector.feed(sample),
close_and_read,
]
barrier = threading.Barrier(workers)

def hammer(index):
barrier.wait(timeout=30)
for step in range(steps):
operations[(index + step) % len(operations)]()

with ThreadPoolExecutor(max_workers=workers) as executor:
list(executor.map(hammer, range(workers)))


class TestCChardet:
def test_free_threaded_build_keeps_gil_disabled(self):
if sysconfig.get_config_var("Py_GIL_DISABLED"):
Expand Down Expand Up @@ -134,6 +162,21 @@ def detect_results(sample):
results = list(executor.map(detect_results, repeated_samples))
assert results == expected * 100

def test_shared_detector_survives_concurrent_calls(self):
# The native calls run without the GIL. Before UniversalDetector held a
# per-instance lock, sharing one detector across threads corrupted
# uchardet's state and crashed the process, even on GIL builds.
with open(os.path.join(TESTDATA_DIR, "ja/euc-jp.txt"), "rb") as file:
sample = file.read() * 16
for weights in (None, {"ja": 0.5}):
detector = cchardet.UniversalDetector(language_weights=weights)
hammer_from_threads(detector, sample)

detector.reset()
detector.feed(sample)
detector.close()
assert detector.result == cchardet.detect(sample, language_weights=weights)

def test_detect_max_bytes(self):
data = b"plain ASCII" + "日本語".encode()
assert cchardet.detect(data, max_bytes=11)["encoding"] == "ASCII"
Expand Down Expand Up @@ -174,6 +217,18 @@ def test_language_weights_are_opt_in_and_persist_across_reset(self):
assert detector.result == weighted
detector.reset()

def test_language_weights_survive_reset_without_new_data(self):
# uchardet's reset freed the candidates that its weighted view still
# pointed to, so closing again returned freed memory as the encoding.
detector = cchardet.UniversalDetector(language_weights={"en": 0.5})
detector.feed(b"plain ASCII text")
detector.close()
assert detector.result["encoding"] == "ASCII"

detector.reset()
detector.close()
assert detector.result == {"encoding": None, "confidence": None, "language": None}

def test_language_weights_are_validated(self):
for weights in ({"english": 1.0}, {"en": -0.1}, {"en": 1.1}, {"en": float("nan")}):
try:
Expand Down
Loading