From dc62d02d95615de3b52db86e414fae31b586bc65 Mon Sep 17 00:00:00 2001 From: SunnyRad Date: Tue, 10 Nov 2026 22:17:23 +0800 Subject: [PATCH 1/2] fix(util): decode stdin as UTF-8, not the ANSI code page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit force_utf8_stdio() reconfigured stdout and stderr but left stdin at cp936, so the `api` op -- which reads JSON-lines requests from stdin -- decoded its client's UTF-8 bytes through GBK. On a Chinese Windows install `{"query": "会话记录"}` arrived as `"浼氳瘽璁板綍"` and the search returned `count: 0`. A miss and a corrupted query are the same answer to the caller, which is why this survived: the index was never wrong. The bug also hides from a casual echo check. For this specific pair the two decodings share bytes -- `gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d` -- so the echoed query can render correctly while the match is already corrupted. The regression test therefore asserts `count`, not the echo. This mirrors the `_force_utf8` helper the Claude Code SessionStart hook already carries (integrations/claude_session_start.py), which includes stdin for exactly this reason. Verified on Windows/cp936 against the plugin's real path (`python -m voyager.cli api`): before: clean -> count 0, PYTHONIOENCODING=gbk -> count 0, utf-8 -> count 1 after: clean -> count 1, PYTHONIOENCODING=gbk -> count 1, utf-8 -> count 1 Mutation-checked: removing `sys.stdin` from the loop turns both new tests red and leaves the other five green. --- tests/test_utf8_stdio.py | 109 +++++++++++++++++++++++++++++++++++++-- voyager/util.py | 35 +++++++++---- 2 files changed, 129 insertions(+), 15 deletions(-) diff --git a/tests/test_utf8_stdio.py b/tests/test_utf8_stdio.py index 3a38b22..623a765 100644 --- a/tests/test_utf8_stdio.py +++ b/tests/test_utf8_stdio.py @@ -1,15 +1,23 @@ """Regression: the CLI must speak UTF-8 whatever the Windows code page is. -The DSH bridge decodes our stdout with ``setEncoding('utf8')``. On a Chinese -Windows install Python defaults stdout to cp936 (GBK), so every non-ASCII title -arrived as U+FFFD while the index itself was perfectly fine -- the data was -never wrong, only the transport. Reported with byte-level evidence against -voyager 1.3.0 (dsh-sessionflow#1). +The DSH bridge decodes our stdout with ``setEncoding('utf8')`` and writes our +stdin as UTF-8. On a Chinese Windows install Python defaults *both* streams to +cp936 (GBK), so: + +* stdout: every non-ASCII title arrived as U+FFFD while the index itself was + perfectly fine -- the data was never wrong, only the transport. +* stdin: the ``api`` op's JSON-lines requests were silently decoded through + GBK, so ``{"query": "会话记录"}`` became ``"浼氳瘽璁板綍"`` and the search + answered ``count: 0``. "No such session" and "your query was corrupted in + transit" are the same answer to the caller, which is what made it survive. + +Reported with byte-level evidence against voyager 1.3.0 (dsh-sessionflow#1). """ from __future__ import annotations import io +import json import os import subprocess import sys @@ -51,6 +59,33 @@ def write(self, s): # pragma: no cover - never reached force_utf8_stdio() # must not raise +def test_force_utf8_covers_stdin(monkeypatch): + """The `api` op reads requests from stdin; it must not stay GBK. + + Regression for the encode/decode asymmetry: stdout was fixed at the process + boundary but stdin was left at the ANSI code page, so a client writing UTF-8 + request lines had them silently decoded through GBK. + """ + stdin = io.TextIOWrapper(io.BytesIO(), encoding="gbk", errors="strict") + monkeypatch.setattr(sys, "stdin", stdin) + monkeypatch.setattr(sys, "stdout", _gbk_stream()) + monkeypatch.setattr(sys, "stderr", _gbk_stream()) + + force_utf8_stdio() + + assert sys.stdin.encoding.lower().replace("-", "") == "utf8", ( + "stdin stayed %r: an api client's UTF-8 request would be decoded as GBK" + % (sys.stdin.encoding,)) + + +def test_force_utf8_tolerates_none_stdin(monkeypatch): + """fd 0 closed (sys.stdin is None) must not break every command.""" + monkeypatch.setattr(sys, "stdin", None) + monkeypatch.setattr(sys, "stdout", _gbk_stream()) + monkeypatch.setattr(sys, "stderr", _gbk_stream()) + force_utf8_stdio() # must not raise + + def test_cli_main_forces_utf8_before_parsing(monkeypatch): """The fix lives at the single entry point, so every subcommand inherits it.""" called = [] @@ -107,3 +142,67 @@ def test_cli_json_bytes_are_utf8_under_gbk(tmp_path): text = proc.stdout.decode("utf-8") # raises UnicodeDecodeError if broken assert "テスト" in text assert "🚀" in text + + +def test_api_op_reads_utf8_stdin_under_gbk(tmp_path): + """End to end over the bridge's real path: a CJK query must still match. + + ``search`` is an FTS5 search over *event bodies*, so the query word goes in + the event body; a title-only match never happens and would make this test + pass for the wrong reason. + + ``count`` is the assertion, not the echoed ``query``: for this pair a GBK + echo of the UTF-8 bytes renders as plausible CJK + (``gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d``), so an assertion on the + echo can pass while the match is already corrupted. + """ + from voyager.model import new_event, new_session + from voyager.store import Store + + query = "会话记录" + db = tmp_path / "index.db" + src = tmp_path / "s.jsonl" + src.write_text("{}\n", encoding="utf-8") + store = Store(db) + try: + sess = new_session( + id="codex:zh", provider="codex", native_session_id="zh", + title="会话 中文标题", started_at=1.0, updated_at=1.0, + repo_root="D:/插件开发", cwd="D:/插件开发") + store.replace_session( + sess, + [new_event(sid="codex:zh", seq=1, kind="user", ts=1.0, + content="这里是一段会话记录，用于测试中文检索。", + origin="human")], + "codex", src) + finally: + store.close() + + home = tmp_path / "home" + home.mkdir() + env = os.environ.copy() + env.update({ + "HOME": str(home), "USERPROFILE": str(home), + "APPDATA": str(home / "AppData/Roaming"), + "LOCALAPPDATA": str(home / "AppData/Local"), + "PYTHONPATH": str(ROOT), "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "gbk", # the condition that used to corrupt it + }) + env.pop("PYTHONUTF8", None) + + request = json.dumps({"id": 1, "op": "search", + "params": {"query": query, "db": str(db)}}, + ensure_ascii=False) + "\n" + proc = subprocess.run( + [sys.executable, "-m", "voyager.cli", "api"], + input=request.encode("utf-8"), # exactly what the Node bridge writes + capture_output=True, env=env, cwd=str(tmp_path), timeout=180, + ) + assert proc.returncode == 0, proc.stderr.decode("utf-8", "replace") + line = proc.stdout.decode("utf-8").strip().splitlines()[-1] + result = json.loads(line)["result"] + + assert result["count"] == 1, ( + "a UTF-8 query read through a GBK stdin found nothing; echoed as %r" + % (result.get("query"),)) + assert result["results"][0]["id"] == "codex:zh" diff --git a/voyager/util.py b/voyager/util.py index 08af250..3bfa338 100644 --- a/voyager/util.py +++ b/voyager/util.py @@ -31,22 +31,37 @@ def force_utf8_stdio() -> None: """Best-effort UTF-8 on the standard streams. Never raises. - On a Chinese Windows install Python writes stdout in the ANSI code page - (GBK / cp936) unless ``PYTHONUTF8=1`` is set. Everything that reads our - output as UTF-8 -- the DSH bridge (``child.stdout.setEncoding('utf8')``), - the VS Code client, MCP clients -- then decodes every non-ASCII character - as U+FFFD. The index is fine; only the transport is wrong, so the fix - belongs at the process boundary, not at each print(). + On a Chinese Windows install Python decodes stdin *and* encodes stdout in + the ANSI code page (GBK / cp936) unless ``PYTHONUTF8=1`` is set. + + **stdout** — everything that reads our output as UTF-8 -- the DSH bridge + (``child.stdout.setEncoding('utf8')``), the VS Code client, MCP clients -- + otherwise decodes every non-ASCII character as U+FFFD. The index is fine; + only the transport is wrong, so the fix belongs at the process boundary, + not at each print(). + + **stdin** — the ``api`` op reads JSON-lines requests from stdin + (:func:`voyager.api.serve`). With a GBK stdin the UTF-8 bytes a client + writes are not rejected: they are silently decoded through the wrong codec, + so ``{"query": "会话记录"}`` becomes ``"浼氳瘽璁板綍"`` and the search returns + ``count: 0``. That is worse than an error -- "no such session" and "your + query was corrupted in transit" are the same answer to the caller. + + The confusion is easy to miss because a GBK *echo* of UTF-8 bytes often + renders as plausible CJK: ``gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d`` + for exactly this pair, so the echoed query can look correct while the match + is already wrong. Only the result count and the stored bytes are evidence. ``errors="replace"`` keeps a hostile/locale-specific character from turning a successful command into a traceback: the JSON or the table is still parseable, one glyph is lost. - Streams that are detached (``sys.stdout is None``) or already wrapped by a - test harness may not support ``reconfigure``; both cases are tolerated - rather than allowed to abort the command. + Streams that are detached (``sys.stdin`` is ``None`` when fd 0 is closed) + or already wrapped by a test harness may not support ``reconfigure``; both + cases are tolerated rather than allowed to abort the command. This mirrors + the ``_force_utf8`` helper the SessionStart hook already carries. """ - for stream in (sys.stdout, sys.stderr): + for stream in (sys.stdin, sys.stdout, sys.stderr): reconfigure = getattr(stream, "reconfigure", None) if reconfigure is None: continue From 87cd92a32febf2f62f2de3313ceccbe46c9bcd7d Mon Sep 17 00:00:00 2001 From: SunnyRad Date: Tue, 10 Nov 2026 23:30:39 +0800 Subject: [PATCH 2/2] fix(api): the bridge's stdin was GBK; pi serve was never implemented Two defects, both in the stdio bridge the VS Code client drives. 1. stdin was decoded in the ANSI code page. 1.3.1 forced UTF-8 on stdout/stderr only, so on a Chinese Windows install the UTF-8 request lines a client writes were silently decoded through GBK: {"query": "????"} searched for "??????" and answered count: 0. "No such session" and "your query was corrupted in transit" are the same answer to the caller, which is why it survived. The setup now lives in api.serve() -- the one place that reads stdin -- instead of only in cli.main(), so oyager api, python -m voyager.api and python -m voyager.api serve are all covered. force_utf8_stdio() is idempotent, so the double call is free. 2. serve was documented but never parsed. docs/API.md, the CHANGELOG and vscode-extension/extension.js (which spawns ["-m","voyager.api","serve"]) all use the verb, but neither parser accepted it: the child exited 2 on "unrecognized arguments: serve" before reading a single request, so the shipped sidebar could never have worked. Both parsers now accept serve as a no-op verb; an unknown verb is still a usage error. Tests assert count, never the echoed query: for this pair a GBK echo of the UTF-8 bytes renders as plausible CJK (gbk("???") == utf8("??") == e4bc9ae8af9d), so an assertion on the echo can pass while the match is already corrupted. Mutation-verified: dropping sys.stdin from force_utf8_stdio fails exactly the 4 stdin-reading paths and leaves the other 6 green; removing the serve verb fails exactly test_documented_serve_verb_is_accepted. Regression: 1372 passed, 8 skipped, 2 failed -- both failures are PackageNotFoundError from voyager not being pip-installed here, and both reproduce identically on pristine main. --- CHANGELOG.md | 23 ++++++++++++++++++- tests/test_utf8_stdio.py | 48 ++++++++++++++++++++++++++++++++++++++-- voyager/api.py | 30 +++++++++++++++++++++++-- voyager/cli.py | 6 +++++ 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffadf46..eb7eb43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,28 @@ All notable changes to Voyager are documented here. Format loosely follows ## [Unreleased] -(nothing yet) +### Fixed + +- **The `api` bridge decoded stdin in the ANSI code page, so a CJK query + answered `count: 0`.** 1.3.1 forced UTF-8 on stdout/stderr only; on a Chinese + Windows install Python also *decodes* stdin as cp936, so the UTF-8 request + lines a client writes were not rejected — they were silently decoded through + GBK, and `{"query": "会话记录"}` searched for `"浼氳瘽璁板綍"` and found nothing. + "No such session" and "your query was corrupted in transit" are the same + answer to the caller, which is why it survived. The setup now lives in + `api.serve()` (the one place that reads stdin) rather than only in + `cli.main()`, so `voyager api`, `python -m voyager.api` and + `python -m voyager.api serve` are all covered. The confusion is easy to miss + because a GBK *echo* of the UTF-8 bytes renders as plausible CJK + (`gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d`), so only `count` and the stored + bytes are evidence — the regression test asserts `count`, never the echo. +- **`voyager api serve` — the documented command — exited 2 without serving.** + `docs/API.md`, the CHANGELOG and `vscode-extension/extension.js` (which spawns + `["-m", "voyager.api", "serve"]`) all use the `serve` verb, but neither parser + ever accepted it: the child died on `unrecognized arguments: serve` before + reading a single request, so the shipped VS Code sidebar could never have + worked. Both parsers now accept `serve` as a no-op verb; an unknown verb is + still a usage error. ## [1.3.1] — 2026-11-09 diff --git a/tests/test_utf8_stdio.py b/tests/test_utf8_stdio.py index 623a765..8b81ba2 100644 --- a/tests/test_utf8_stdio.py +++ b/tests/test_utf8_stdio.py @@ -156,6 +156,47 @@ def test_api_op_reads_utf8_stdin_under_gbk(tmp_path): (``gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d``), so an assertion on the echo can pass while the match is already corrupted. """ + _assert_cjk_query_matches([sys.executable, "-m", "voyager.cli", "api"], + tmp_path) + + +def test_documented_serve_verb_is_accepted(tmp_path): + """`python -m voyager.api serve` is what the shipped client spawns. + + ``vscode-extension/extension.js`` builds exactly + ``spawn(python, ["-m", "voyager.api", "serve"])`` and docs/API.md documents + ``voyager api serve``, but neither parser ever accepted the verb: the child + exited 2 with "unrecognized arguments: serve" before reading a request, so + the sidebar could never have worked. A documented invocation is a contract; + this asserts it answers rather than exits. + """ + _assert_cjk_query_matches([sys.executable, "-m", "voyager.api", "serve"], + tmp_path) + + +def test_bare_api_module_still_serves(tmp_path): + """The un-verbed form must keep working -- it is the other documented path.""" + _assert_cjk_query_matches([sys.executable, "-m", "voyager.api"], tmp_path) + + +def test_unknown_api_verb_is_still_rejected(): + """Accepting `serve` must not turn the parser into "anything goes". + + A typo has to fail loudly; the fix is one accepted verb, not a catch-all. + """ + proc = subprocess.run([sys.executable, "-m", "voyager.api", "bogus"], + capture_output=True, cwd=str(ROOT), timeout=180) + assert proc.returncode == 2, ( + "a bogus verb exited %d, not argparse's usage error 2" + % proc.returncode) + + +def _assert_cjk_query_matches(argv, tmp_path): + """Seed one CJK session, then assert ``argv`` answers a CJK query. + + Shared by the invocation-form tests so they differ only in the command a + client would type -- the assertion (``count``, never the echo) stays fixed. + """ from voyager.model import new_event, new_session from voyager.store import Store @@ -194,11 +235,14 @@ def test_api_op_reads_utf8_stdin_under_gbk(tmp_path): "params": {"query": query, "db": str(db)}}, ensure_ascii=False) + "\n" proc = subprocess.run( - [sys.executable, "-m", "voyager.cli", "api"], + argv, input=request.encode("utf-8"), # exactly what the Node bridge writes capture_output=True, env=env, cwd=str(tmp_path), timeout=180, ) - assert proc.returncode == 0, proc.stderr.decode("utf-8", "replace") + assert proc.returncode == 0, ( + "%s exited %d: %s" + % (" ".join(argv[1:]), proc.returncode, + proc.stderr.decode("utf-8", "replace").strip()[:300])) line = proc.stdout.decode("utf-8").strip().splitlines()[-1] result = json.loads(line)["result"] diff --git a/voyager/api.py b/voyager/api.py index ed24655..89c9ab5 100644 --- a/voyager/api.py +++ b/voyager/api.py @@ -6,6 +6,13 @@ any other UI) talks to this surface, over stdio JSON-lines via `voyager api serve` or by importing these functions directly. +All three stdio entry points — `voyager api`, `voyager api serve` and +`python -m voyager.api [serve]` — converge on :func:`serve`, which is therefore +where the stream setup belongs. ``serve`` accepts the ``serve`` verb as a +no-op: it is the form this docstring, ``docs/API.md``, the CHANGELOG and the +shipped ``vscode-extension/extension.js`` all use, and it is what a caller +following the docs will type. + Read-only by design: the only "write-ish" op is bundle_preview, which writes nothing (it renders a bundle in memory). @@ -28,7 +35,7 @@ from .continuity import build_continuation_bundle from .ranker import extract_candidate_facts, rank_candidates from .store import default_db_path, open_read, Store -from .util import parse_when, same_repo_loose +from .util import force_utf8_stdio, parse_when, same_repo_loose def _row(r) -> Dict[str, Any]: @@ -542,7 +549,19 @@ def handle_request(req: Dict[str, Any], db: Optional[Path] = None) -> Dict[str, def serve(db: Optional[Path] = None) -> None: - """JSON-lines stdio loop: one request per line, one response per line.""" + """JSON-lines stdio loop: one request per line, one response per line. + + The stream setup lives *here*, not only in ``cli.main()``, because this is + the one place that reads stdin: ``voyager api``, ``python -m voyager.api`` + and the VS Code extension's ``python -m voyager.api serve`` all converge on + it. A client writes UTF-8; on a Chinese Windows install stdin defaults to + the ANSI code page (cp936), so the bytes would be decoded through GBK + without an error and ``{"query": "会话记录"}`` would answer ``count: 0`` -- + indistinguishable from "no such session". ``force_utf8_stdio`` is + idempotent, so calling it again from an entry point that already did is + free. + """ + force_utf8_stdio() for line in sys.stdin: line = line.strip() if not line: @@ -561,6 +580,13 @@ def main() -> None: p = argparse.ArgumentParser(prog="voyager-api", description="Voyager local API (stdio JSON-lines)") p.add_argument("--db", help="index db path (default ~/.voyager/index.db)") + # `serve` is the documented verb -- docs/API.md and CHANGELOG both say + # `voyager api serve`, and the shipped VS Code extension spawns exactly + # `python -m voyager.api serve` (extension.js). This parser never accepted + # it, so that client exited 2 before reading a single request: "unrecognized + # arguments: serve". Accepted here as a no-op verb so the documented form + # works instead of breaking callers that were written to the docs. + p.add_argument("verb", nargs="?", choices=["serve"], help=argparse.SUPPRESS) args = p.parse_args() serve(Path(args.db) if args.db else None) diff --git a/voyager/cli.py b/voyager/cli.py index bfc9f1d..2a28d27 100644 --- a/voyager/cli.py +++ b/voyager/cli.py @@ -2472,6 +2472,12 @@ def _cmd_demo(args) -> int: sp = sub.add_parser("api", help="local stdio JSON-lines API (VS Code client)", parents=[common]) + # `voyager api serve` is what docs/API.md documents and what every caller + # of this command was told to type; the subparser only ever accepted a bare + # `voyager api`, so the documented invocation died with "unrecognized + # arguments: serve" before reading a request. The verb is a no-op -- `api` + # already means serve -- accepted so the documented form works. + sp.add_argument("verb", nargs="?", choices=["serve"], help=argparse.SUPPRESS) sp.set_defaults(func=cmd_api) sp = sub.add_parser("status", parents=[common],