fix(util): decode stdin as UTF-8, not the ANSI code page

CHANGELOG.md

@@ -5,7 +5,28 @@ All notable changes to Voyager are documented here. Format loosely follows
 
 ## [Unreleased]
 
-(nothing yet)
+### Fixed
+
+- **The `api` bridge decoded stdin in the ANSI code page, so a CJK query
+  answered `count: 0`.** 1.3.1 forced UTF-8 on stdout/stderr only; on a Chinese
+  Windows install Python also *decodes* stdin as cp936, so the UTF-8 request
+  lines a client writes were not rejected — they were silently decoded through
+  GBK, and `{"query": "会话记录"}` searched for `"浼氳瘽璁板綍"` and found nothing.
+  "No such session" and "your query was corrupted in transit" are the same
+  answer to the caller, which is why it survived. The setup now lives in
+  `api.serve()` (the one place that reads stdin) rather than only in
+  `cli.main()`, so `voyager api`, `python -m voyager.api` and
+  `python -m voyager.api serve` are all covered. The confusion is easy to miss
+  because a GBK *echo* of the UTF-8 bytes renders as plausible CJK
+  (`gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d`), so only `count` and the stored
+  bytes are evidence — the regression test asserts `count`, never the echo.
+- **`voyager api serve` — the documented command — exited 2 without serving.**
+  `docs/API.md`, the CHANGELOG and `vscode-extension/extension.js` (which spawns
+  `["-m", "voyager.api", "serve"]`) all use the `serve` verb, but neither parser
+  ever accepted it: the child died on `unrecognized arguments: serve` before
+  reading a single request, so the shipped VS Code sidebar could never have
+  worked. Both parsers now accept `serve` as a no-op verb; an unknown verb is
+  still a usage error.
 
 ## [1.3.1] — 2026-11-09
 

tests/test_utf8_stdio.py

@@ -1,15 +1,23 @@
 """Regression: the CLI must speak UTF-8 whatever the Windows code page is.
 
-The DSH bridge decodes our stdout with ``setEncoding('utf8')``.  On a Chinese
-Windows install Python defaults stdout to cp936 (GBK), so every non-ASCII title
-arrived as U+FFFD while the index itself was perfectly fine -- the data was
-never wrong, only the transport.  Reported with byte-level evidence against
-voyager 1.3.0 (dsh-sessionflow#1).
+The DSH bridge decodes our stdout with ``setEncoding('utf8')`` and writes our
+stdin as UTF-8.  On a Chinese Windows install Python defaults *both* streams to
+cp936 (GBK), so:
+
+* stdout: every non-ASCII title arrived as U+FFFD while the index itself was
+  perfectly fine -- the data was never wrong, only the transport.
+* stdin: the ``api`` op's JSON-lines requests were silently decoded through
+  GBK, so ``{"query": "会话记录"}`` became ``"浼氳瘽璁板綍"`` and the search
+  answered ``count: 0``.  "No such session" and "your query was corrupted in
+  transit" are the same answer to the caller, which is what made it survive.
+
+Reported with byte-level evidence against voyager 1.3.0 (dsh-sessionflow#1).
 """
 
 from __future__ import annotations
 
 import io
+import json
 import os
 import subprocess
 import sys
@@ -51,6 +59,33 @@ def write(self, s):          # pragma: no cover - never reached
     force_utf8_stdio()               # must not raise
 
 
+def test_force_utf8_covers_stdin(monkeypatch):
+    """The `api` op reads requests from stdin; it must not stay GBK.
+
+    Regression for the encode/decode asymmetry: stdout was fixed at the process
+    boundary but stdin was left at the ANSI code page, so a client writing UTF-8
+    request lines had them silently decoded through GBK.
+    """
+    stdin = io.TextIOWrapper(io.BytesIO(), encoding="gbk", errors="strict")
+    monkeypatch.setattr(sys, "stdin", stdin)
+    monkeypatch.setattr(sys, "stdout", _gbk_stream())
+    monkeypatch.setattr(sys, "stderr", _gbk_stream())
+
+    force_utf8_stdio()
+
+    assert sys.stdin.encoding.lower().replace("-", "") == "utf8", (
+        "stdin stayed %r: an api client's UTF-8 request would be decoded as GBK"
+        % (sys.stdin.encoding,))
+
+
+def test_force_utf8_tolerates_none_stdin(monkeypatch):
+    """fd 0 closed (sys.stdin is None) must not break every command."""
+    monkeypatch.setattr(sys, "stdin", None)
+    monkeypatch.setattr(sys, "stdout", _gbk_stream())
+    monkeypatch.setattr(sys, "stderr", _gbk_stream())
+    force_utf8_stdio()               # must not raise
+
+
 def test_cli_main_forces_utf8_before_parsing(monkeypatch):
     """The fix lives at the single entry point, so every subcommand inherits it."""
     called = []
@@ -107,3 +142,111 @@ def test_cli_json_bytes_are_utf8_under_gbk(tmp_path):
     text = proc.stdout.decode("utf-8")     # raises UnicodeDecodeError if broken
     assert "テスト" in text
     assert "🚀" in text
+
+
+def test_api_op_reads_utf8_stdin_under_gbk(tmp_path):
+    """End to end over the bridge's real path: a CJK query must still match.
+
+    ``search`` is an FTS5 search over *event bodies*, so the query word goes in
+    the event body; a title-only match never happens and would make this test
+    pass for the wrong reason.
+
+    ``count`` is the assertion, not the echoed ``query``: for this pair a GBK
+    echo of the UTF-8 bytes renders as plausible CJK
+    (``gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d``), so an assertion on the
+    echo can pass while the match is already corrupted.
+    """
+    _assert_cjk_query_matches([sys.executable, "-m", "voyager.cli", "api"],
+                              tmp_path)
+
+
+def test_documented_serve_verb_is_accepted(tmp_path):
+    """`python -m voyager.api serve` is what the shipped client spawns.
+
+    ``vscode-extension/extension.js`` builds exactly
+    ``spawn(python, ["-m", "voyager.api", "serve"])`` and docs/API.md documents
+    ``voyager api serve``, but neither parser ever accepted the verb: the child
+    exited 2 with "unrecognized arguments: serve" before reading a request, so
+    the sidebar could never have worked.  A documented invocation is a contract;
+    this asserts it answers rather than exits.
+    """
+    _assert_cjk_query_matches([sys.executable, "-m", "voyager.api", "serve"],
+                              tmp_path)
+
+
+def test_bare_api_module_still_serves(tmp_path):
+    """The un-verbed form must keep working -- it is the other documented path."""
+    _assert_cjk_query_matches([sys.executable, "-m", "voyager.api"], tmp_path)
+
+
+def test_unknown_api_verb_is_still_rejected():
+    """Accepting `serve` must not turn the parser into "anything goes".
+
+    A typo has to fail loudly; the fix is one accepted verb, not a catch-all.
+    """
+    proc = subprocess.run([sys.executable, "-m", "voyager.api", "bogus"],
+                          capture_output=True, cwd=str(ROOT), timeout=180)
+    assert proc.returncode == 2, (
+        "a bogus verb exited %d, not argparse's usage error 2"
+        % proc.returncode)
+
+
+def _assert_cjk_query_matches(argv, tmp_path):
+    """Seed one CJK session, then assert ``argv`` answers a CJK query.
+
+    Shared by the invocation-form tests so they differ only in the command a
+    client would type -- the assertion (``count``, never the echo) stays fixed.
+    """
+    from voyager.model import new_event, new_session
+    from voyager.store import Store
+
+    query = "会话记录"
+    db = tmp_path / "index.db"
+    src = tmp_path / "s.jsonl"
+    src.write_text("{}\n", encoding="utf-8")
+    store = Store(db)
+    try:
+        sess = new_session(
+            id="codex:zh", provider="codex", native_session_id="zh",
+            title="会话 中文标题", started_at=1.0, updated_at=1.0,
+            repo_root="D:/插件开发", cwd="D:/插件开发")
+        store.replace_session(
+            sess,
+            [new_event(sid="codex:zh", seq=1, kind="user", ts=1.0,
+                       content="这里是一段会话记录，用于测试中文检索。",
+                       origin="human")],
+            "codex", src)
+    finally:
+        store.close()
+
+    home = tmp_path / "home"
+    home.mkdir()
+    env = os.environ.copy()
+    env.update({
+        "HOME": str(home), "USERPROFILE": str(home),
+        "APPDATA": str(home / "AppData/Roaming"),
+        "LOCALAPPDATA": str(home / "AppData/Local"),
+        "PYTHONPATH": str(ROOT), "PYTHONDONTWRITEBYTECODE": "1",
+        "PYTHONIOENCODING": "gbk",     # the condition that used to corrupt it
+    })
+    env.pop("PYTHONUTF8", None)
+
+    request = json.dumps({"id": 1, "op": "search",
+                          "params": {"query": query, "db": str(db)}},
+                         ensure_ascii=False) + "\n"
+    proc = subprocess.run(
+        argv,
+        input=request.encode("utf-8"),      # exactly what the Node bridge writes
+        capture_output=True, env=env, cwd=str(tmp_path), timeout=180,
+    )
+    assert proc.returncode == 0, (
+        "%s exited %d: %s"
+        % (" ".join(argv[1:]), proc.returncode,
+           proc.stderr.decode("utf-8", "replace").strip()[:300]))
+    line = proc.stdout.decode("utf-8").strip().splitlines()[-1]
+    result = json.loads(line)["result"]
+
+    assert result["count"] == 1, (
+        "a UTF-8 query read through a GBK stdin found nothing; echoed as %r"
+        % (result.get("query"),))
+    assert result["results"][0]["id"] == "codex:zh"

voyager/api.py

@@ -6,6 +6,13 @@
 any other UI) talks to this surface, over stdio JSON-lines via
 `voyager api serve` or by importing these functions directly.
 
+All three stdio entry points — `voyager api`, `voyager api serve` and
+`python -m voyager.api [serve]` — converge on :func:`serve`, which is therefore
+where the stream setup belongs.  ``serve`` accepts the ``serve`` verb as a
+no-op: it is the form this docstring, ``docs/API.md``, the CHANGELOG and the
+shipped ``vscode-extension/extension.js`` all use, and it is what a caller
+following the docs will type.
+
 Read-only by design: the only "write-ish" op is bundle_preview, which
 writes nothing (it renders a bundle in memory).
 
@@ -28,7 +35,7 @@
 from .continuity import build_continuation_bundle
 from .ranker import extract_candidate_facts, rank_candidates
 from .store import default_db_path, open_read, Store
-from .util import parse_when, same_repo_loose
+from .util import force_utf8_stdio, parse_when, same_repo_loose
 
 
 def _row(r) -> Dict[str, Any]:
@@ -542,7 +549,19 @@ def handle_request(req: Dict[str, Any], db: Optional[Path] = None) -> Dict[str,
 
 
 def serve(db: Optional[Path] = None) -> None:
-    """JSON-lines stdio loop: one request per line, one response per line."""
+    """JSON-lines stdio loop: one request per line, one response per line.
+
+    The stream setup lives *here*, not only in ``cli.main()``, because this is
+    the one place that reads stdin: ``voyager api``, ``python -m voyager.api``
+    and the VS Code extension's ``python -m voyager.api serve`` all converge on
+    it.  A client writes UTF-8; on a Chinese Windows install stdin defaults to
+    the ANSI code page (cp936), so the bytes would be decoded through GBK
+    without an error and ``{"query": "会话记录"}`` would answer ``count: 0`` --
+    indistinguishable from "no such session".  ``force_utf8_stdio`` is
+    idempotent, so calling it again from an entry point that already did is
+    free.
+    """
+    force_utf8_stdio()
     for line in sys.stdin:
         line = line.strip()
         if not line:
@@ -561,6 +580,13 @@ def main() -> None:
     p = argparse.ArgumentParser(prog="voyager-api",
                                 description="Voyager local API (stdio JSON-lines)")
     p.add_argument("--db", help="index db path (default ~/.voyager/index.db)")
+    # `serve` is the documented verb -- docs/API.md and CHANGELOG both say
+    # `voyager api serve`, and the shipped VS Code extension spawns exactly
+    # `python -m voyager.api serve` (extension.js).  This parser never accepted
+    # it, so that client exited 2 before reading a single request: "unrecognized
+    # arguments: serve".  Accepted here as a no-op verb so the documented form
+    # works instead of breaking callers that were written to the docs.
+    p.add_argument("verb", nargs="?", choices=["serve"], help=argparse.SUPPRESS)
     args = p.parse_args()
     serve(Path(args.db) if args.db else None)
 

voyager/cli.py

@@ -2472,6 +2472,12 @@ def _cmd_demo(args) -> int:
 
     sp = sub.add_parser("api", help="local stdio JSON-lines API (VS Code client)",
                         parents=[common])
+    # `voyager api serve` is what docs/API.md documents and what every caller
+    # of this command was told to type; the subparser only ever accepted a bare
+    # `voyager api`, so the documented invocation died with "unrecognized
+    # arguments: serve" before reading a request.  The verb is a no-op -- `api`
+    # already means serve -- accepted so the documented form works.
+    sp.add_argument("verb", nargs="?", choices=["serve"], help=argparse.SUPPRESS)
     sp.set_defaults(func=cmd_api)
 
     sp = sub.add_parser("status", parents=[common],

voyager/util.py

@@ -31,22 +31,37 @@
 def force_utf8_stdio() -> None:
     """Best-effort UTF-8 on the standard streams.  Never raises.
 
-    On a Chinese Windows install Python writes stdout in the ANSI code page
-    (GBK / cp936) unless ``PYTHONUTF8=1`` is set.  Everything that reads our
-    output as UTF-8 -- the DSH bridge (``child.stdout.setEncoding('utf8')``),
-    the VS Code client, MCP clients -- then decodes every non-ASCII character
-    as U+FFFD.  The index is fine; only the transport is wrong, so the fix
-    belongs at the process boundary, not at each print().
+    On a Chinese Windows install Python decodes stdin *and* encodes stdout in
+    the ANSI code page (GBK / cp936) unless ``PYTHONUTF8=1`` is set.
+
+    **stdout** — everything that reads our output as UTF-8 -- the DSH bridge
+    (``child.stdout.setEncoding('utf8')``), the VS Code client, MCP clients --
+    otherwise decodes every non-ASCII character as U+FFFD.  The index is fine;
+    only the transport is wrong, so the fix belongs at the process boundary,
+    not at each print().
+
+    **stdin** — the ``api`` op reads JSON-lines requests from stdin
+    (:func:`voyager.api.serve`).  With a GBK stdin the UTF-8 bytes a client
+    writes are not rejected: they are silently decoded through the wrong codec,
+    so ``{"query": "会话记录"}`` becomes ``"浼氳瘽璁板綍"`` and the search returns
+    ``count: 0``.  That is worse than an error -- "no such session" and "your
+    query was corrupted in transit" are the same answer to the caller.
+
+    The confusion is easy to miss because a GBK *echo* of UTF-8 bytes often
+    renders as plausible CJK: ``gbk("浼氳瘽") == utf8("会话") == e4bc9ae8af9d``
+    for exactly this pair, so the echoed query can look correct while the match
+    is already wrong.  Only the result count and the stored bytes are evidence.
 
     ``errors="replace"`` keeps a hostile/locale-specific character from
     turning a successful command into a traceback: the JSON or the table is
     still parseable, one glyph is lost.
 
-    Streams that are detached (``sys.stdout is None``) or already wrapped by a
-    test harness may not support ``reconfigure``; both cases are tolerated
-    rather than allowed to abort the command.
+    Streams that are detached (``sys.stdin`` is ``None`` when fd 0 is closed)
+    or already wrapped by a test harness may not support ``reconfigure``; both
+    cases are tolerated rather than allowed to abort the command.  This mirrors
+    the ``_force_utf8`` helper the SessionStart hook already carries.
     """
-    for stream in (sys.stdout, sys.stderr):
+    for stream in (sys.stdin, sys.stdout, sys.stderr):
         reconfigure = getattr(stream, "reconfigure", None)
         if reconfigure is None:
             continue