From 79dd52d8a95ceaa264111e865d657baf419f4a89 Mon Sep 17 00:00:00 2001
From: Rayan-and-beyond <263488867+Rayan-and-beyond@users.noreply.github.com>
Date: Tue, 15 Sep 2026 21:16:59 +0000
Subject: [PATCH] fix: account for CJK text in estimator

---
 src/index.mjs       |  4 +++-
 test/basic.test.mjs | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/index.mjs b/src/index.mjs
index ecb07e4..008727c 100644
--- a/src/index.mjs
+++ b/src/index.mjs
@@ -10,7 +10,9 @@ export function estimateTokens(text) {
   if (!s) return 0;
   const chars = s.length;
   const words = (s.match(/\S+/g) || []).length;
-  return Math.max(Math.ceil(chars / 4), Math.ceil(words * 1.3));
+  const cjkChars = (s.match(/[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]/g) || []).length;
+  const charEstimate = Math.ceil((chars - cjkChars) / 4) + cjkChars;
+  return Math.max(charEstimate, Math.ceil(words * 1.3));
 }
 
 // Flatten any supported payload into text units: {role, kind, text}.
diff --git a/test/basic.test.mjs b/test/basic.test.mjs
index 5c4158f..2222eaf 100644
--- a/test/basic.test.mjs
+++ b/test/basic.test.mjs
@@ -172,3 +172,14 @@ test("units rejects non-array payload.messages with TypeError", () => {
     { name: "TypeError", message: /payload\.messages must be an array/ },
   );
 });
+
+test("estimateTokens keeps CJK text within the documented reference tolerance", () => {
+  const text = "漢".repeat(200);
+  const referenceTokens = 200;
+  const tolerance = 0.15;
+  const estimated = estimateTokens(text);
+  assert.ok(
+    Math.abs(estimated - referenceTokens) <= referenceTokens * tolerance,
+    `estimated ${estimated}, expected within 15% of ${referenceTokens}`,
+  );
+});