diff --git a/AGENTS.md b/AGENTS.md
index 6b0560d..60d1e6b 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -4,7 +4,7 @@
Token Pilot is evolving from a Spring AI usage-tracking starter into a framework-independent Java LLM control and accounting core with optional framework and observability adapters.
-Current truth: post-call usage normalization, cost calculation, ledger events, Micrometer publishing, Clock-based monthly budget windows, pure budget decisions, typed missing-pricing policies, pricing snapshots, legacy provider-boundary BLOCK enforcement, Spring AI integration, and starter autoconfiguration are implemented. Candidate-aware preflight admission, context admission, atomic reservation, and estimate/actual cost reconciliation are 30-day MVP targets, not current capabilities.
+Current truth: post-call usage normalization, cost calculation, ledger events, Micrometer publishing, Clock-based monthly budget windows, pure budget decisions, typed missing-pricing policies, pricing snapshots, framework-independent token count results, a UTF-8 byte heuristic estimator, and a preflight cost-bound projection are implemented. Candidate-aware request production, context admission, atomic reservation, and estimate/actual cost reconciliation are 30-day MVP targets, not current capabilities.
Distribution direction: publish a framework-independent core and an optional Spring AI convenience starter from the same repository and release train. The existing starter artifact is `token-pilot-starter`; `token-pilot-spring-ai-starter` is only a target name until a compatibility ADR and module change land.
@@ -71,7 +71,7 @@ Token Pilot의 제품 포지션은 framework-independent Java LLM control and ac
| Module | Status | Notes |
| --- | --- | --- |
-| `token-pilot-core` | Basic implementation complete | Domain records, pricing, calculator, registry, ledger manager, pricing snapshots, and missing-pricing evaluator |
+| `token-pilot-core` | Basic implementation complete | Domain records, pricing, calculator, registry, ledger manager, pricing snapshots, token count results, UTF-8 byte heuristic estimation, and a preflight cost-bound projection; versioned model snapshots and context admission are still required before reservation |
| `token-pilot-spring-ai` | Basic implementation complete | Spring AI 2.0.0 `UsageExtractor`, `LedgerAdvisor`, pricing snapshot resolution, response usage recording, reconciliation decisions, and legacy provider-boundary BLOCK enforcement |
| `token-pilot-micrometer` | Basic implementation complete | `MetricsOptions`, tag whitelist, and metric metadata exist; metric ownership must be narrowed |
| `token-pilot-budget` | Basic non-atomic implementation | Typed monthly keys, Clock/ZoneId windows, and pure status/admission decisions implemented; needs candidate estimation, reservation, idempotency, and reconciliation |
@@ -335,6 +335,10 @@ The active checklist is in `docs/30_DAY_MVP_REPORT.md`; detailed long-term works
- Sample app E2E uses a fake Spring AI `ChatModel`; real provider API behavior is not yet verified.
- `token-pilot-spring-ai-starter` does not exist in the current build; never use it as an install instruction until implemented and published.
- Maven Central release consumption must be re-verified for both core and starter paths before announcing `0.1.0`.
+- Preflight cost bounds must use one immutable pricing snapshot from calculation through reservation and reconciliation; resolving a mutable registry again by model/policy identifiers can mix prices from different requests.
+- A preflight cost bound is not context admission evidence. The REQUEST token result and reserved output must be checked against the versioned model context window before a provider call or reservation is authorized.
+- `DefaultPreflightCostEstimator` currently relies on the existing exclusive `TokenType` pricing shape; richer pricing combinations and finite/unbounded policy capabilities must be owned by a validated pricing policy snapshot rather than caller-supplied flags.
+- The UTF-8 byte heuristic estimator returns `TEXT_ONLY` and uses `BYTE_LEVEL_BPE_UTF8` only as a byte-level safety-basis identifier; it is not an exact BPE implementation and must not be used as full-request admission evidence.
## Verification
@@ -396,6 +400,19 @@ Stage and deploy a Central release:
## Update History
+### 2026-08-10
+
+- Added the UTF-8 byte heuristic estimator and made preflight cost calculation consume the exact immutable pricing snapshot carried by its context instead of re-resolving a mutable registry.
+- Kept arithmetic failures inside the typed preflight unavailable-result contract for pricing inputs whose decimal scale cannot be represented by the calculation.
+
+### 2026-08-09
+
+- Clarified that the preflight cost-bound projection is not context admission or atomic reservation, and documented the immutable pricing-snapshot requirement for later integration.
+
+### 2026-08-08
+
+- Added framework-independent token count result contracts and conservative preflight cost bounds that preserve immutable pricing metadata, use exact decimal arithmetic, and return typed unavailable outcomes instead of zero-price fallbacks.
+
### 2026-08-04
- Added typed missing-pricing policies, immutable pricing snapshots, core rate validation/reconciliation decisions, and Spring AI pre-call pricing resolution; `FAIL_CLOSED` rejects missing plans/rates before provider invocation and `FAIL_OPEN` preserves `UNPRICED`.
diff --git a/build.gradle b/build.gradle
index 30428b2..8330de9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -657,11 +657,9 @@ package compatibility;
import io.tokenpilot.core.TokenEstimator;
import io.tokenpilot.core.domain.Cost;
-import io.tokenpilot.core.domain.TokenCountAccuracy;
import io.tokenpilot.core.domain.TokenCountResult;
import io.tokenpilot.core.domain.TokenCountScope;
-import io.tokenpilot.core.domain.TokenEstimatorDescriptor;
-import io.tokenpilot.core.domain.TokenizationBasis;
+import io.tokenpilot.core.internal.LedgerComponents;
import java.math.BigDecimal;
import java.util.Currency;
@@ -675,17 +673,13 @@ public final class CoreConsumer {
throw new IllegalStateException("Token Pilot core consumer verification failed");
}
- TokenCountResult expected = TokenCountResult.counted(
- 0L,
- 0L,
- TokenCountAccuracy.EXACT,
- TokenCountScope.TEXT_ONLY,
- new TokenEstimatorDescriptor("consumer-estimator", "1"),
- new TokenizationBasis("consumer-basis")
- );
- TokenEstimator estimator = text -> expected;
- TokenCountResult actual = estimator.estimate("");
- if (!actual.isCounted() || !actual.isExact() || actual.tokens().orElseThrow() != 0L) {
+ TokenEstimator estimator = LedgerComponents.utf8ByteHeuristicTokenEstimator();
+ TokenCountResult actual = estimator.estimate("hello");
+ if (!actual.isCounted()
+ || actual.isExact()
+ || actual.tokens().orElseThrow() != 2L
+ || actual.safeUpperBoundTokens().orElseThrow() != 5L
+ || actual.scope() != TokenCountScope.TEXT_ONLY) {
throw new IllegalStateException("Token Pilot estimator consumer verification failed");
}
System.out.println("token-pilot-core Java 25 consumer OK");
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/PreflightCostEstimator.java b/token-pilot-core/src/main/java/io/tokenpilot/core/PreflightCostEstimator.java
new file mode 100644
index 0000000..cd5d4c0
--- /dev/null
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/PreflightCostEstimator.java
@@ -0,0 +1,40 @@
+package io.tokenpilot.core;
+
+import io.tokenpilot.core.domain.PreflightCostResult;
+import io.tokenpilot.core.domain.PreflightPricingContext;
+import io.tokenpilot.core.domain.TokenCountResult;
+
+/**
+ * 호출 전에 계산한 REQUEST 범위의 토큰 상한을 금액 상한으로 변환하는 Core 계약입니다.
+ *
+ * 이 계약은 비용을 계산할 뿐 context window 적합성이나 provider 호출 허가를
+ * 판정하지 않습니다. 호출자는 요청 토큰 결과가 모델의 context admission을 통과했고,
+ * {@link PreflightPricingContext}가 계산에 사용할 하나의 불변 pricing snapshot을
+ * 보관하는지 먼저 보장해야 합니다. 계산기는 다른 snapshot을 registry에서 다시
+ * 조회하지 않습니다.
+ *
+ * 계산 가능한 경우에도 예약 근거로 사용할 값은
+ * {@link PreflightCostResult.Bounded#safeUpperBoundCost()}뿐입니다.
+ * {@code estimatedCost}는 관찰과 표시를 위한 값입니다.
+ */
+public interface PreflightCostEstimator {
+
+ /**
+ * atomic reservation에 사용할 호출 전 비용 상한을 계산합니다.
+ * 입력이 REQUEST 범위가 아니거나 tokenizer 기준이 맞지 않거나 가격 snapshot이
+ * 없거나 문맥에 snapshot이 없으면 숫자 비용 대신 제한된 unavailable 결과를 반환해야 합니다.
+ *
+ * @param pricingContext canonical model, pricing policy, catalog version과
+ * 검증된 pricing 조건을 담은 계산 문맥
+ * @param requestInput 실제 전송 요청 전체를 계산한 REQUEST 범위의 token 결과
+ * @param reservedOutputTokens 호출 전에 확보할 최대 출력 token 수
+ * @return 계산 가능한 비용 상한 또는 그 사유를 담은 unavailable 결과
+ * @throws NullPointerException 필수 인자가 {@code null}인 경우
+ * @throws IllegalArgumentException reservedOutputTokens가 음수인 경우
+ */
+ PreflightCostResult estimate(
+ PreflightPricingContext pricingContext,
+ TokenCountResult requestInput,
+ long reservedOutputTokens
+ );
+}
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightCostResult.java b/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightCostResult.java
new file mode 100644
index 0000000..d6fcf78
--- /dev/null
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightCostResult.java
@@ -0,0 +1,152 @@
+package io.tokenpilot.core.domain;
+
+import java.util.Currency;
+import java.util.Objects;
+
+/**
+ * 호출 전 비용 계산 결과입니다. 숫자 상한이 있는 결과와 계산 불가 결과를
+ * sealed hierarchy로 분리해 0원과 pricing 부재를 혼동하지 않습니다.
+ */
+public sealed interface PreflightCostResult
+ permits PreflightCostResult.Bounded, PreflightCostResult.Unavailable {
+
+ /**
+ * 하나의 불변 pricing snapshot으로 계산한 유한 비용 상한입니다.
+ * 내부 계산에서는 반올림하지 않으며, context admission을 통과했다는 의미는 아닙니다.
+ * 예약 계층은 반드시 {@code safeUpperBoundCost}를 사용해야 합니다.
+ *
+ * @param estimatedCost 관찰과 표시를 위한 예상 비용
+ * @param safeUpperBoundCost 예약에 사용할 보수적인 비용 상한
+ * @param inputEstimatedTokens 정보 제공용 입력 token 계산값
+ * @param inputSafeUpperBoundTokens 보수적인 입력 token 안전 상한
+ * @param reservedOutputTokens 예약한 최대 출력 token 수
+ * @param canonicalModelId canonical model 식별자
+ * @param pricingPolicyId 불변 pricing policy 식별자
+ * @param catalogVersion 계산에 사용한 catalog 버전
+ * @param pricingSnapshot 계산·예약·정산에 전달할 정확한 가격 snapshot
+ * @param estimatorDescriptor token estimator 식별 정보
+ * @param tokenizationBasis 검증된 tokenizer 호환성 기준
+ */
+ record Bounded(
+ Cost estimatedCost,
+ Cost safeUpperBoundCost,
+ long inputEstimatedTokens,
+ long inputSafeUpperBoundTokens,
+ long reservedOutputTokens,
+ String canonicalModelId,
+ String pricingPolicyId,
+ String catalogVersion,
+ PricingSnapshot pricingSnapshot,
+ TokenEstimatorDescriptor estimatorDescriptor,
+ TokenizationBasis tokenizationBasis
+ ) implements PreflightCostResult {
+
+ /**
+ * 비용, token 관계와 재현성 metadata를 검증합니다.
+ */
+ public Bounded {
+ estimatedCost = Objects.requireNonNull(estimatedCost, "estimatedCost must not be null");
+ safeUpperBoundCost = Objects.requireNonNull(
+ safeUpperBoundCost,
+ "safeUpperBoundCost must not be null"
+ );
+ validateTokens(inputEstimatedTokens, inputSafeUpperBoundTokens, reservedOutputTokens);
+ canonicalModelId = requireText(canonicalModelId, "canonicalModelId");
+ pricingPolicyId = requireText(pricingPolicyId, "pricingPolicyId");
+ catalogVersion = requireText(catalogVersion, "catalogVersion");
+ pricingSnapshot = Objects.requireNonNull(pricingSnapshot, "pricingSnapshot must not be null");
+ if (!pricingSnapshot.modelId().equals(canonicalModelId)
+ || !pricingSnapshot.pricingPolicyId().equals(pricingPolicyId)
+ || !pricingSnapshot.catalogVersion().equals(catalogVersion)) {
+ throw new IllegalArgumentException("pricingSnapshot identity must match result metadata");
+ }
+ estimatorDescriptor = Objects.requireNonNull(
+ estimatorDescriptor,
+ "estimatorDescriptor must not be null"
+ );
+ tokenizationBasis = Objects.requireNonNull(tokenizationBasis, "tokenizationBasis must not be null");
+
+ if (!estimatedCost.currency().equals(safeUpperBoundCost.currency())) {
+ throw new IllegalArgumentException("estimated and safe upper bound currencies must match");
+ }
+ if (!pricingSnapshot.currency().equals(safeUpperBoundCost.currency())) {
+ throw new IllegalArgumentException("pricingSnapshot currency must match cost currency");
+ }
+ if (safeUpperBoundCost.compareTo(estimatedCost) < 0) {
+ throw new IllegalArgumentException("safeUpperBoundCost must be greater than or equal to estimatedCost");
+ }
+ }
+
+ /**
+ * 계산된 비용의 통화를 반환합니다.
+ *
+ * @return estimated cost와 safe upper bound cost가 공유하는 통화
+ */
+ public Currency currency() {
+ return safeUpperBoundCost.currency();
+ }
+ }
+
+ /**
+ * fail-closed 경계에서 숫자 비용으로 취급할 수 없는 결과입니다.
+ * 이 결과는 0원이나 임의의 기본 단가를 의미하지 않으며, provider 호출과 예약을
+ * 계속할지 여부는 각 control 계층이 사유를 확인해 결정해야 합니다.
+ *
+ * @param reason 숫자 비용을 생성하지 못한 제한된 사유
+ * @param canonicalModelId canonical model 식별자
+ * @param pricingPolicyId 참조한 pricing policy 식별자
+ * @param catalogVersion 참조한 catalog 버전
+ * @param currency 모델이 요구한 비용 통화
+ * @param reservedOutputTokens 요청한 최대 출력 token 수
+ * @param estimatorDescriptor token estimator 식별 정보
+ * @param tokenizationBasis token 결과의 tokenizer 호환성 기준
+ */
+ record Unavailable(
+ PreflightCostUnavailableReason reason,
+ String canonicalModelId,
+ String pricingPolicyId,
+ String catalogVersion,
+ Currency currency,
+ long reservedOutputTokens,
+ TokenEstimatorDescriptor estimatorDescriptor,
+ TokenizationBasis tokenizationBasis
+ ) implements PreflightCostResult {
+
+ /**
+ * Unavailable 사유와 진단 metadata를 검증합니다.
+ */
+ public Unavailable {
+ reason = Objects.requireNonNull(reason, "reason must not be null");
+ canonicalModelId = requireText(canonicalModelId, "canonicalModelId");
+ pricingPolicyId = requireText(pricingPolicyId, "pricingPolicyId");
+ catalogVersion = requireText(catalogVersion, "catalogVersion");
+ currency = Objects.requireNonNull(currency, "currency must not be null");
+ if (reservedOutputTokens < 0) {
+ throw new IllegalArgumentException("reservedOutputTokens must be non-negative");
+ }
+ estimatorDescriptor = Objects.requireNonNull(
+ estimatorDescriptor,
+ "estimatorDescriptor must not be null"
+ );
+ tokenizationBasis = Objects.requireNonNull(tokenizationBasis, "tokenizationBasis must not be null");
+ }
+ }
+
+ private static void validateTokens(long estimated, long safeUpperBound, long reservedOutput) {
+ if (estimated < 0 || safeUpperBound < 0 || reservedOutput < 0) {
+ throw new IllegalArgumentException("token values must be non-negative");
+ }
+ if (safeUpperBound < estimated) {
+ throw new IllegalArgumentException(
+ "inputSafeUpperBoundTokens must be greater than or equal to inputEstimatedTokens"
+ );
+ }
+ }
+
+ private static String requireText(String value, String name) {
+ if (value == null || value.isBlank()) {
+ throw new IllegalArgumentException(name + " must not be blank");
+ }
+ return value;
+ }
+}
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightCostUnavailableReason.java b/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightCostUnavailableReason.java
new file mode 100644
index 0000000..7a08569
--- /dev/null
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightCostUnavailableReason.java
@@ -0,0 +1,25 @@
+package io.tokenpilot.core.domain;
+
+/**
+ * 호출 전 보수적 비용 상한을 숫자로 확정할 수 없는 제한된 사유입니다.
+ */
+public enum PreflightCostUnavailableReason {
+ /** token 계산 결과를 사용할 수 없는 상태입니다. */
+ COUNT_UNAVAILABLE,
+ /** 전체 REQUEST 범위를 포함하지 않은 token 결과입니다. */
+ INCOMPLETE_SCOPE,
+ /** estimator와 model의 tokenizer 기준이 호환되지 않습니다. */
+ INCOMPATIBLE_TOKENIZER,
+ /** 참조한 pricing policy snapshot을 찾지 못했습니다. */
+ PRICING_NOT_FOUND,
+ /** 비용 계산에 필요한 기본 입력/출력 단가가 누락됐습니다. */
+ INCOMPLETE_PRICING,
+ /** pricing policy가 호출 전에 유한한 상한을 제공할 수 없습니다. */
+ UNBOUNDED_PRICING,
+ /** model과 pricing snapshot의 통화가 다릅니다. */
+ CURRENCY_MISMATCH,
+ /** canonical model, policy 또는 catalog 식별자가 snapshot과 다릅니다. */
+ PRICING_SNAPSHOT_MISMATCH,
+ /** 가격 정밀도가 계산 범위를 벗어나 정형 비용 결과를 만들 수 없습니다. */
+ ARITHMETIC_FAILURE
+}
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightPricingContext.java b/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightPricingContext.java
new file mode 100644
index 0000000..42a6541
--- /dev/null
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/domain/PreflightPricingContext.java
@@ -0,0 +1,98 @@
+package io.tokenpilot.core.domain;
+
+import java.util.Currency;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * canonical model 해석 이후 preflight 비용 계산에 전달하는 불변 pricing projection입니다.
+ *
+ * alias는 ModelRegistry에서 먼저 canonical model로 해석해야 하며, 이 문맥을
+ * 임의의 문자열 조합으로 만들면 안 됩니다. {@code catalogVersion},
+ * {@code pricingPolicyId}, 통화와 tokenizer 기준은 같은 모델 정의와 pricing
+ * snapshot에서 함께 파생되어야 합니다.
+ *
+ * 계산 가능한 문맥은 registry에서 한 번 확정한 동일한 불변 snapshot을 함께
+ * 보관해야 합니다. 계산·예약·정산 과정에서 mutable registry를 다시 조회하면
+ * 서로 다른 가격을 섞을 수 있습니다. 가격을 찾지 못한 문맥은 빈 snapshot으로
+ * 만들 수 있으며, 계산기는 이를 {@code PRICING_NOT_FOUND}로 반환합니다.
+ * {@link UpperBoundCapability} 역시 호출자가 임의로 선언하는 값이 아니라 검증된
+ * pricing policy의 결과여야 합니다.
+ *
+ * @param canonicalModelId canonical model 식별자
+ * @param pricingPolicyId 불변 pricing policy 식별자
+ * @param catalogVersion model과 pricing catalog의 버전
+ * @param tokenizationBasis 모델이 허용하는 tokenizer 호환성 기준
+ * @param currency 모델이 요구하는 비용 통화
+ * @param upperBoundCapability pricing policy가 유한한 비용 상한을 제공할 수 있는지 나타내는 결과
+ * @param pricingSnapshot 이 계산·예약·정산에 사용할 확정 가격 snapshot
+ */
+public record PreflightPricingContext(
+ String canonicalModelId,
+ String pricingPolicyId,
+ String catalogVersion,
+ TokenizationBasis tokenizationBasis,
+ Currency currency,
+ UpperBoundCapability upperBoundCapability,
+ Optional pricingSnapshot
+) {
+
+ /** 모든 식별자와 호환성 metadata가 비어 있지 않은지 검증합니다. */
+ public PreflightPricingContext {
+ canonicalModelId = requireText(canonicalModelId, "canonicalModelId");
+ pricingPolicyId = requireText(pricingPolicyId, "pricingPolicyId");
+ catalogVersion = requireText(catalogVersion, "catalogVersion");
+ tokenizationBasis = Objects.requireNonNull(tokenizationBasis, "tokenizationBasis must not be null");
+ currency = Objects.requireNonNull(currency, "currency must not be null");
+ upperBoundCapability = Objects.requireNonNull(
+ upperBoundCapability,
+ "upperBoundCapability must not be null"
+ );
+ pricingSnapshot = Objects.requireNonNull(pricingSnapshot, "pricingSnapshot must not be null");
+ }
+
+ /**
+ * 아직 가격 snapshot을 찾지 못한 문맥을 생성합니다.
+ * 계산기는 이 문맥을 숫자 비용이 아닌 {@code PRICING_NOT_FOUND}로 처리합니다.
+ *
+ * @param canonicalModelId canonical model 식별자
+ * @param pricingPolicyId pricing policy 식별자
+ * @param catalogVersion model과 pricing catalog의 버전
+ * @param tokenizationBasis tokenizer 호환성 기준
+ * @param currency 비용 통화
+ * @param upperBoundCapability 가격 정책의 상한 제공 가능 여부
+ */
+ public PreflightPricingContext(
+ String canonicalModelId,
+ String pricingPolicyId,
+ String catalogVersion,
+ TokenizationBasis tokenizationBasis,
+ Currency currency,
+ UpperBoundCapability upperBoundCapability
+ ) {
+ this(
+ canonicalModelId,
+ pricingPolicyId,
+ catalogVersion,
+ tokenizationBasis,
+ currency,
+ upperBoundCapability,
+ Optional.empty()
+ );
+ }
+
+ private static String requireText(String value, String name) {
+ if (value == null || value.isBlank()) {
+ throw new IllegalArgumentException(name + " must not be blank");
+ }
+ return value;
+ }
+
+ /** pricing policy가 호출 전에 유한한 보수적 상한을 제공할 수 있는지 나타냅니다. */
+ public enum UpperBoundCapability {
+ /** 모든 적용 경로에 유한한 최대 단가가 있습니다. */
+ FINITE,
+ /** 하나 이상의 적용 경로에 유한한 최대 단가가 없습니다. */
+ UNBOUNDED
+ }
+}
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/internal/DefaultPreflightCostEstimator.java b/token-pilot-core/src/main/java/io/tokenpilot/core/internal/DefaultPreflightCostEstimator.java
new file mode 100644
index 0000000..67cd8c1
--- /dev/null
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/internal/DefaultPreflightCostEstimator.java
@@ -0,0 +1,194 @@
+package io.tokenpilot.core.internal;
+
+import io.tokenpilot.core.PreflightCostEstimator;
+import io.tokenpilot.core.domain.Cost;
+import io.tokenpilot.core.domain.PreflightCostResult;
+import io.tokenpilot.core.domain.PreflightCostUnavailableReason;
+import io.tokenpilot.core.domain.PreflightPricingContext;
+import io.tokenpilot.core.domain.PricingSnapshot;
+import io.tokenpilot.core.domain.TokenCountResult;
+import io.tokenpilot.core.domain.TokenCountScope;
+import io.tokenpilot.core.domain.TokenType;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * 불변 pricing snapshot의 배타적인 입력/출력 가격 경로에서 각각 최대 단가를 선택해
+ * 같은 token을 중복 합산하지 않고 보수적인 비용 상한을 계산하는 기본 구현입니다.
+ *
+ * 이 구현은 비용만 계산하며 context admission이나 provider 호출을 허가하지 않습니다.
+ * 입력 문맥이 보관한 동일한 snapshot만 사용하며, 계산 중 registry를 다시 조회하지
+ * 않습니다. 호출 계층은 그 snapshot을 예약·정산에도 그대로 전달해야 합니다.
+ */
+class DefaultPreflightCostEstimator implements PreflightCostEstimator {
+
+ private static final int TOKENS_PER_K_SHIFT = 3;
+
+ @Override
+ public PreflightCostResult estimate(
+ PreflightPricingContext pricingContext,
+ TokenCountResult requestInput,
+ long reservedOutputTokens
+ ) {
+ Objects.requireNonNull(pricingContext, "pricingContext must not be null");
+ Objects.requireNonNull(requestInput, "requestInput must not be null");
+ if (reservedOutputTokens < 0) {
+ throw new IllegalArgumentException("reservedOutputTokens must be non-negative");
+ }
+
+ if (!requestInput.isCounted()) {
+ return unavailable(
+ PreflightCostUnavailableReason.COUNT_UNAVAILABLE,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+ if (requestInput.scope() != TokenCountScope.REQUEST) {
+ return unavailable(
+ PreflightCostUnavailableReason.INCOMPLETE_SCOPE,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+ if (!pricingContext.tokenizationBasis().equals(requestInput.tokenizationBasis())) {
+ return unavailable(
+ PreflightCostUnavailableReason.INCOMPATIBLE_TOKENIZER,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+ if (pricingContext.upperBoundCapability() == PreflightPricingContext.UpperBoundCapability.UNBOUNDED) {
+ return unavailable(
+ PreflightCostUnavailableReason.UNBOUNDED_PRICING,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+
+ if (pricingContext.pricingSnapshot().isEmpty()) {
+ return unavailable(
+ PreflightCostUnavailableReason.PRICING_NOT_FOUND,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+
+ PricingSnapshot snapshot = pricingContext.pricingSnapshot().orElseThrow();
+ if (!snapshot.modelId().equals(pricingContext.canonicalModelId())
+ || !snapshot.pricingPolicyId().equals(pricingContext.pricingPolicyId())
+ || !snapshot.catalogVersion().equals(pricingContext.catalogVersion())) {
+ return unavailable(
+ PreflightCostUnavailableReason.PRICING_SNAPSHOT_MISMATCH,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+ if (!snapshot.currency().equals(pricingContext.currency())) {
+ return unavailable(
+ PreflightCostUnavailableReason.CURRENCY_MISMATCH,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+
+ Map rates = snapshot.rates();
+ BigDecimal promptRate = rates.get(TokenType.PROMPT);
+ BigDecimal completionRate = rates.get(TokenType.COMPLETION);
+ if (promptRate == null || completionRate == null) {
+ return unavailable(
+ PreflightCostUnavailableReason.INCOMPLETE_PRICING,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+
+ long inputEstimatedTokens = requestInput.tokens().orElseThrow();
+ long inputSafeUpperBoundTokens = requestInput.safeUpperBoundTokens().orElseThrow();
+
+ BigDecimal conservativeInputRate = maxRate(
+ promptRate,
+ rates.getOrDefault(TokenType.CACHE_READ_PROMPT, promptRate),
+ rates.getOrDefault(TokenType.CACHE_CREATION_PROMPT, promptRate)
+ );
+ BigDecimal conservativeOutputRate = maxRate(
+ completionRate,
+ rates.getOrDefault(TokenType.REASONING, completionRate)
+ );
+
+ Cost estimatedCost;
+ Cost safeUpperBoundCost;
+ try {
+ estimatedCost = new Cost(
+ costFor(inputEstimatedTokens, promptRate)
+ .add(costFor(reservedOutputTokens, completionRate)),
+ snapshot.currency()
+ );
+ safeUpperBoundCost = new Cost(
+ costFor(inputSafeUpperBoundTokens, conservativeInputRate)
+ .add(costFor(reservedOutputTokens, conservativeOutputRate)),
+ snapshot.currency()
+ );
+ } catch (ArithmeticException exception) {
+ return unavailable(
+ PreflightCostUnavailableReason.ARITHMETIC_FAILURE,
+ pricingContext,
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+
+ return new PreflightCostResult.Bounded(
+ estimatedCost,
+ safeUpperBoundCost,
+ inputEstimatedTokens,
+ inputSafeUpperBoundTokens,
+ reservedOutputTokens,
+ snapshot.modelId(),
+ snapshot.pricingPolicyId(),
+ snapshot.catalogVersion(),
+ snapshot,
+ requestInput.estimatorDescriptor(),
+ requestInput.tokenizationBasis()
+ );
+ }
+
+ private PreflightCostResult.Unavailable unavailable(
+ PreflightCostUnavailableReason reason,
+ PreflightPricingContext pricingContext,
+ TokenCountResult requestInput,
+ long reservedOutputTokens
+ ) {
+ return new PreflightCostResult.Unavailable(
+ reason,
+ pricingContext.canonicalModelId(),
+ pricingContext.pricingPolicyId(),
+ pricingContext.catalogVersion(),
+ pricingContext.currency(),
+ reservedOutputTokens,
+ requestInput.estimatorDescriptor(),
+ requestInput.tokenizationBasis()
+ );
+ }
+
+ private BigDecimal maxRate(BigDecimal first, BigDecimal... remaining) {
+ BigDecimal maximum = first;
+ for (BigDecimal candidate : remaining) {
+ maximum = maximum.max(candidate);
+ }
+ return maximum;
+ }
+
+ private BigDecimal costFor(long tokens, BigDecimal ratePerK) {
+ return ratePerK.multiply(BigDecimal.valueOf(tokens)).movePointLeft(TOKENS_PER_K_SHIFT);
+ }
+}
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/internal/HeuristicTokenEstimator.java b/token-pilot-core/src/main/java/io/tokenpilot/core/internal/HeuristicTokenEstimator.java
new file mode 100644
index 0000000..9f48af7
--- /dev/null
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/internal/HeuristicTokenEstimator.java
@@ -0,0 +1,80 @@
+package io.tokenpilot.core.internal;
+
+import io.tokenpilot.core.TokenEstimator;
+import io.tokenpilot.core.domain.TokenCountAccuracy;
+import io.tokenpilot.core.domain.TokenCountResult;
+import io.tokenpilot.core.domain.TokenCountScope;
+import io.tokenpilot.core.domain.TokenEstimatorDescriptor;
+import io.tokenpilot.core.domain.TokenizationBasis;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+/**
+ * UTF-8 byte 길이를 기반으로 문자열의 text-only token 수를 추정합니다.
+ *
+ * {@code BYTE_LEVEL_BPE_UTF8}는 byte-level tokenizer의 안전성 가정을
+ * 나타내는 compatibility basis일 뿐, 이 구현이 exact BPE merge를 수행한다는
+ * 뜻은 아닙니다. 결과는 전체 요청의 context admission 근거로 사용할 수 없습니다.
+ */
+final class HeuristicTokenEstimator implements TokenEstimator {
+
+ private static final int BYTES_PER_ESTIMATED_TOKEN = 4;
+ private static final String ESTIMATOR_ID = "tokenpilot-utf8-byte-heuristic";
+ private static final String ESTIMATOR_VERSION = "1";
+ private static final String TOKENIZATION_BASIS_ID = "BYTE_LEVEL_BPE_UTF8";
+
+ private static final TokenEstimatorDescriptor ESTIMATOR_DESCRIPTOR =
+ new TokenEstimatorDescriptor(
+ ESTIMATOR_ID,
+ ESTIMATOR_VERSION
+ );
+
+ private static final TokenizationBasis TOKENIZATION_BASIS =
+ new TokenizationBasis(TOKENIZATION_BASIS_ID);
+
+ /**
+ * 원문을 정규화하지 않고 UTF-8 byte 길이로 계산합니다.
+ *
+ * @param text 계산할 원문
+ * @return text-only 휴리스틱 token 계산 결과
+ * @throws NullPointerException text가 null인 경우
+ * @throws IllegalArgumentException text에 malformed UTF-16이 포함된 경우
+ */
+ @Override
+ public TokenCountResult estimate(String text) {
+ Objects.requireNonNull(text, "text must not be null");
+
+ long utf8Bytes = utf8ByteLength(text);
+ long estimatedTokens = Math.ceilDiv(utf8Bytes, BYTES_PER_ESTIMATED_TOKEN);
+
+ return TokenCountResult.counted(
+ estimatedTokens,
+ utf8Bytes,
+ TokenCountAccuracy.HEURISTIC,
+ TokenCountScope.TEXT_ONLY,
+ ESTIMATOR_DESCRIPTOR,
+ TOKENIZATION_BASIS
+ );
+ }
+
+ private static long utf8ByteLength(String text) {
+ CharsetEncoder encoder = StandardCharsets.UTF_8.newEncoder();
+ encoder.onMalformedInput(CodingErrorAction.REPORT);
+ return encodedByteLength(encoder, text);
+ }
+
+ private static long encodedByteLength(CharsetEncoder encoder, String text) {
+ try {
+ CharBuffer input = CharBuffer.wrap(text);
+ ByteBuffer encoded = encoder.encode(input);
+ return encoded.remaining();
+ } catch (CharacterCodingException exception) {
+ throw new IllegalArgumentException("text contains invalid UTF-16", exception);
+ }
+ }
+}
diff --git a/token-pilot-core/src/main/java/io/tokenpilot/core/internal/LedgerComponents.java b/token-pilot-core/src/main/java/io/tokenpilot/core/internal/LedgerComponents.java
index 8ced30b..b4d4882 100644
--- a/token-pilot-core/src/main/java/io/tokenpilot/core/internal/LedgerComponents.java
+++ b/token-pilot-core/src/main/java/io/tokenpilot/core/internal/LedgerComponents.java
@@ -6,6 +6,8 @@
import io.tokenpilot.core.PricingEvaluator;
import io.tokenpilot.core.PricingProvider;
import io.tokenpilot.core.PricingRegistry;
+import io.tokenpilot.core.PreflightCostEstimator;
+import io.tokenpilot.core.TokenEstimator;
import java.util.List;
@@ -25,6 +27,28 @@ public static PricingEvaluator defaultPricingEvaluator() {
return new DefaultPricingEvaluator();
}
+ /**
+ * 기본 preflight 비용 상한 계산기를 생성합니다.
+ *
+ * 계산기는 REQUEST 범위의 token 결과와 하나의 불변 pricing snapshot을
+ * 보관한 문맥을 요구합니다. 이 factory는 context admission이나 atomic reservation을
+ * 대신 수행하지 않습니다.
+ *
+ * @return 기본 preflight 비용 상한 계산기
+ */
+ public static PreflightCostEstimator defaultPreflightCostEstimator() {
+ return new DefaultPreflightCostEstimator();
+ }
+
+ /**
+ * UTF-8 byte 길이 기반의 text-only 휴리스틱 token estimator를 생성합니다.
+ *
+ * @return UTF-8 byte 휴리스틱 estimator
+ */
+ public static TokenEstimator utf8ByteHeuristicTokenEstimator() {
+ return new HeuristicTokenEstimator();
+ }
+
public static PricingRegistry inMemoryPricingRegistry(List providers) {
return new InMemoryPricingRegistry(providers);
}
diff --git a/token-pilot-core/src/test/java/io/tokenpilot/core/internal/DefaultPreflightCostEstimatorTest.java b/token-pilot-core/src/test/java/io/tokenpilot/core/internal/DefaultPreflightCostEstimatorTest.java
new file mode 100644
index 0000000..44670ea
--- /dev/null
+++ b/token-pilot-core/src/test/java/io/tokenpilot/core/internal/DefaultPreflightCostEstimatorTest.java
@@ -0,0 +1,394 @@
+package io.tokenpilot.core.internal;
+
+import io.tokenpilot.core.PreflightCostEstimator;
+import io.tokenpilot.core.PricingProvider;
+import io.tokenpilot.core.PricingRegistry;
+import io.tokenpilot.core.domain.PreflightCostResult;
+import io.tokenpilot.core.domain.PreflightCostUnavailableReason;
+import io.tokenpilot.core.domain.PreflightPricingContext;
+import io.tokenpilot.core.domain.PricingPlan;
+import io.tokenpilot.core.domain.PricingSnapshot;
+import io.tokenpilot.core.domain.TokenCountAccuracy;
+import io.tokenpilot.core.domain.TokenCountResult;
+import io.tokenpilot.core.domain.TokenCountScope;
+import io.tokenpilot.core.domain.TokenCountUnavailableReason;
+import io.tokenpilot.core.domain.TokenEstimatorDescriptor;
+import io.tokenpilot.core.domain.TokenType;
+import io.tokenpilot.core.domain.TokenizationBasis;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Currency;
+import java.util.Map;
+import java.util.Optional;
+
+import static io.tokenpilot.core.domain.PreflightPricingContext.UpperBoundCapability.FINITE;
+import static io.tokenpilot.core.domain.PreflightPricingContext.UpperBoundCapability.UNBOUNDED;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class DefaultPreflightCostEstimatorTest {
+
+ private static final Currency USD = Currency.getInstance("USD");
+ private static final TokenizationBasis BASIS = new TokenizationBasis("o200k_base");
+ private static final TokenEstimatorDescriptor ESTIMATOR =
+ new TokenEstimatorDescriptor("request-estimator", "1");
+
+ @Test
+ @DisplayName("estimated cost와 보수적 safe upper bound를 분리해 계산한다")
+ void calculateEstimatedAndSafeUpperBoundCost() {
+ PreflightCostEstimator estimator = estimator(plan(Map.of(
+ TokenType.PROMPT, decimal("0.010"),
+ TokenType.CACHE_READ_PROMPT, decimal("0.002"),
+ TokenType.CACHE_CREATION_PROMPT, decimal("0.030"),
+ TokenType.COMPLETION, decimal("0.020"),
+ TokenType.REASONING, decimal("0.050")
+ )));
+
+ PreflightCostResult result = estimator.estimate(
+ context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(1_000, 1_200),
+ 500
+ );
+
+ assertThat(result).isInstanceOf(PreflightCostResult.Bounded.class);
+ PreflightCostResult.Bounded bounded = (PreflightCostResult.Bounded) result;
+ assertThat(bounded.estimatedCost().value()).isEqualByComparingTo("0.020");
+ assertThat(bounded.safeUpperBoundCost().value()).isEqualByComparingTo("0.061");
+ assertThat(bounded.safeUpperBoundCost().compareTo(bounded.estimatedCost())).isGreaterThanOrEqualTo(0);
+ assertThat(bounded.inputEstimatedTokens()).isEqualTo(1_000);
+ assertThat(bounded.inputSafeUpperBoundTokens()).isEqualTo(1_200);
+ assertThat(bounded.reservedOutputTokens()).isEqualTo(500);
+ assertThat(bounded.currency()).isEqualTo(USD);
+ assertThat(bounded.canonicalModelId()).isEqualTo("model-v1");
+ assertThat(bounded.pricingPolicyId()).isEqualTo("standard");
+ assertThat(bounded.catalogVersion()).isEqualTo(PricingSnapshot.DEFAULT_CATALOG_VERSION);
+ assertThat(bounded.pricingSnapshot().rates()).containsEntry(TokenType.PROMPT, decimal("0.010"));
+ assertThat(bounded.estimatorDescriptor()).isEqualTo(ESTIMATOR);
+ assertThat(bounded.tokenizationBasis()).isEqualTo(BASIS);
+ }
+
+ @Test
+ @DisplayName("cache와 reasoning 경로는 최대 단가만 선택해 token을 이중 과금하지 않는다")
+ void chooseConservativeRatesWithoutDoubleCountingTokens() {
+ PreflightCostEstimator estimator = estimator(plan(Map.of(
+ TokenType.PROMPT, decimal("1"),
+ TokenType.CACHE_READ_PROMPT, decimal("2"),
+ TokenType.CACHE_CREATION_PROMPT, decimal("3"),
+ TokenType.COMPLETION, decimal("4"),
+ TokenType.REASONING, decimal("5")
+ )));
+
+ PreflightCostResult.Bounded result = bounded(estimator.estimate(
+ context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(1_000, 1_000),
+ 1_000
+ ));
+
+ assertThat(result.safeUpperBoundCost().value()).isEqualByComparingTo("8");
+ }
+
+ @Test
+ @DisplayName("input safe bound와 reserved output이 증가하면 비용 상한은 감소하지 않는다")
+ void keepSafeUpperBoundMonotonic() {
+ PreflightCostEstimator estimator = estimator(plan(Map.of(
+ TokenType.PROMPT, decimal("0.01"),
+ TokenType.COMPLETION, decimal("0.03")
+ )));
+ PreflightPricingContext context = context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION);
+
+ PreflightCostResult.Bounded base = bounded(estimator.estimate(context, requestCount(100, 120), 50));
+ PreflightCostResult.Bounded moreInput = bounded(estimator.estimate(context, requestCount(100, 121), 50));
+ PreflightCostResult.Bounded moreOutput = bounded(estimator.estimate(context, requestCount(100, 120), 51));
+
+ assertThat(moreInput.safeUpperBoundCost().compareTo(base.safeUpperBoundCost())).isGreaterThanOrEqualTo(0);
+ assertThat(moreOutput.safeUpperBoundCost().compareTo(base.safeUpperBoundCost())).isGreaterThanOrEqualTo(0);
+ }
+
+ @Test
+ @DisplayName("TEXT_ONLY, unavailable count와 tokenizer mismatch는 숫자 비용을 만들지 않는다")
+ void rejectInvalidTokenEvidence() {
+ PreflightCostEstimator estimator = estimator(standardPlan());
+ PreflightPricingContext context = context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION);
+
+ assertUnavailable(
+ estimator.estimate(context, counted(100, 120, TokenCountScope.TEXT_ONLY, BASIS), 10),
+ PreflightCostUnavailableReason.INCOMPLETE_SCOPE
+ );
+ assertUnavailable(
+ estimator.estimate(context, unavailableCount(), 10),
+ PreflightCostUnavailableReason.COUNT_UNAVAILABLE
+ );
+ assertUnavailable(
+ estimator.estimate(
+ context,
+ counted(100, 120, TokenCountScope.REQUEST, new TokenizationBasis("cl100k_base")),
+ 10
+ ),
+ PreflightCostUnavailableReason.INCOMPATIBLE_TOKENIZER
+ );
+ }
+
+ @Test
+ @DisplayName("가격 미등록, 필수 단가 누락과 unbounded pricing은 typed unavailable이다")
+ void distinguishUnavailablePricingFromZeroCost() {
+ PreflightPricingContext finiteContext = context(
+ FINITE,
+ USD,
+ PricingSnapshot.DEFAULT_CATALOG_VERSION
+ );
+
+ assertUnavailable(
+ estimator().estimate(finiteContext, requestCount(100, 120), 10),
+ PreflightCostUnavailableReason.PRICING_NOT_FOUND
+ );
+ assertUnavailable(
+ estimator(plan(Map.of(TokenType.PROMPT, decimal("0.01"))))
+ .estimate(finiteContext, requestCount(100, 120), 10),
+ PreflightCostUnavailableReason.INCOMPLETE_PRICING
+ );
+ assertUnavailable(
+ estimator(standardPlan()).estimate(
+ context(UNBOUNDED, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(100, 120),
+ 10
+ ),
+ PreflightCostUnavailableReason.UNBOUNDED_PRICING
+ );
+ }
+
+ @Test
+ @DisplayName("모든 필수 단가가 명시적인 0인 정책만 0원 bound를 만든다")
+ void allowExplicitFreePricingPolicy() {
+ PreflightCostEstimator estimator = estimator(plan(Map.of(
+ TokenType.PROMPT, BigDecimal.ZERO,
+ TokenType.COMPLETION, BigDecimal.ZERO
+ )));
+
+ PreflightCostResult.Bounded result = bounded(estimator.estimate(
+ context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(1_000, 2_000),
+ 1_000
+ ));
+
+ assertThat(result.estimatedCost().value()).isEqualByComparingTo(BigDecimal.ZERO);
+ assertThat(result.safeUpperBoundCost().value()).isEqualByComparingTo(BigDecimal.ZERO);
+ }
+
+ @Test
+ @DisplayName("currency와 snapshot identity mismatch를 합산하거나 변환하지 않는다")
+ void rejectCurrencyAndSnapshotMismatch() {
+ PreflightCostEstimator estimator = estimator(standardPlan());
+
+ assertUnavailable(
+ estimator.estimate(
+ context(FINITE, Currency.getInstance("EUR"), PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(100, 120),
+ 10
+ ),
+ PreflightCostUnavailableReason.CURRENCY_MISMATCH
+ );
+ assertUnavailable(
+ estimator.estimate(
+ context(FINITE, USD, "catalog-v2"),
+ requestCount(100, 120),
+ 10
+ ),
+ PreflightCostUnavailableReason.PRICING_SNAPSHOT_MISMATCH
+ );
+ }
+
+ @Test
+ @DisplayName("registry 가격이 바뀌어도 문맥에 담긴 snapshot으로 계산한다")
+ void usesCapturedPricingSnapshotAfterRegistryUpdate() {
+ PricingPlan initialPlan = plan(Map.of(
+ TokenType.PROMPT, decimal("10"),
+ TokenType.COMPLETION, decimal("10")
+ ));
+ PricingPlan updatedPlan = plan(Map.of(
+ TokenType.PROMPT, decimal("1"),
+ TokenType.COMPLETION, decimal("1")
+ ));
+ PricingProvider provider = () -> Arrays.asList(initialPlan);
+ PricingRegistry registry = LedgerComponents.inMemoryPricingRegistry(java.util.List.of(provider));
+ PricingSnapshot captured = registry.resolveSnapshot("model-v1", "standard").orElseThrow();
+ registry.registerPlan(updatedPlan);
+
+ PreflightPricingContext context = withSnapshot(
+ context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ captured
+ );
+ PreflightCostResult.Bounded result = bounded(
+ LedgerComponents.defaultPreflightCostEstimator().estimate(
+ context,
+ requestCount(1_000, 1_000),
+ 0
+ )
+ );
+
+ assertThat(result.safeUpperBoundCost().value()).isEqualByComparingTo("10");
+ }
+
+ @Test
+ @DisplayName("계산할 수 없는 BigDecimal 정밀도는 정형 unavailable 결과로 반환한다")
+ void returnsUnavailableForUnsupportedDecimalScale() {
+ BigDecimal extremeRate = new BigDecimal(BigInteger.ONE, Integer.MAX_VALUE);
+ PricingSnapshot snapshot = new PricingSnapshot(
+ "model-v1",
+ "standard",
+ PricingSnapshot.DEFAULT_CATALOG_VERSION,
+ Instant.EPOCH,
+ Map.of(
+ TokenType.PROMPT, extremeRate,
+ TokenType.COMPLETION, BigDecimal.ZERO
+ ),
+ USD
+ );
+
+ PreflightCostResult result = LedgerComponents.defaultPreflightCostEstimator().estimate(
+ withSnapshot(context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION), snapshot),
+ requestCount(1, 1),
+ 0
+ );
+
+ assertUnavailable(result, PreflightCostUnavailableReason.ARITHMETIC_FAILURE);
+ }
+
+ @Test
+ @DisplayName("Long.MAX_VALUE token에서도 overflow 없이 BigDecimal 정밀도를 보존한다")
+ void preservePrecisionForVeryLargeTokenCounts() {
+ BigDecimal promptRate = decimal("0.12345678901234567890123456789");
+ BigDecimal completionRate = decimal("0.98765432109876543210987654321");
+ PreflightCostEstimator estimator = estimator(plan(Map.of(
+ TokenType.PROMPT, promptRate,
+ TokenType.COMPLETION, completionRate
+ )));
+
+ PreflightCostResult.Bounded result = bounded(estimator.estimate(
+ context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(Long.MAX_VALUE, Long.MAX_VALUE),
+ Long.MAX_VALUE
+ ));
+ BigDecimal expected = promptRate.multiply(BigDecimal.valueOf(Long.MAX_VALUE)).movePointLeft(3)
+ .add(completionRate.multiply(BigDecimal.valueOf(Long.MAX_VALUE)).movePointLeft(3));
+
+ assertThat(result.estimatedCost().value()).isEqualByComparingTo(expected);
+ assertThat(result.safeUpperBoundCost().value()).isEqualByComparingTo(expected);
+ }
+
+ @Test
+ @DisplayName("reserved output token은 음수일 수 없다")
+ void rejectNegativeReservedOutputTokens() {
+ PreflightCostEstimator estimator = estimator(standardPlan());
+
+ assertThatThrownBy(() -> estimator.estimate(
+ context(FINITE, USD, PricingSnapshot.DEFAULT_CATALOG_VERSION),
+ requestCount(100, 120),
+ -1
+ )).isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("reservedOutputTokens");
+ }
+
+ private static PreflightCostEstimator estimator(PricingPlan... plans) {
+ PricingSnapshot snapshot = plans.length == 0
+ ? null
+ : PricingSnapshot.from(plans[0], PricingSnapshot.DEFAULT_CATALOG_VERSION, Instant.EPOCH);
+ PreflightCostEstimator delegate = LedgerComponents.defaultPreflightCostEstimator();
+ return (context, requestInput, reservedOutputTokens) -> delegate.estimate(
+ withSnapshot(context, snapshot),
+ requestInput,
+ reservedOutputTokens
+ );
+ }
+
+ private static PreflightPricingContext withSnapshot(
+ PreflightPricingContext context,
+ PricingSnapshot snapshot
+ ) {
+ return new PreflightPricingContext(
+ context.canonicalModelId(),
+ context.pricingPolicyId(),
+ context.catalogVersion(),
+ context.tokenizationBasis(),
+ context.currency(),
+ context.upperBoundCapability(),
+ Optional.ofNullable(snapshot)
+ );
+ }
+
+ private static PricingPlan standardPlan() {
+ return plan(Map.of(
+ TokenType.PROMPT, decimal("0.01"),
+ TokenType.COMPLETION, decimal("0.03")
+ ));
+ }
+
+ private static PricingPlan plan(Map rates) {
+ return new PricingPlan("model-v1", "standard", rates, USD);
+ }
+
+ private static PreflightPricingContext context(
+ PreflightPricingContext.UpperBoundCapability capability,
+ Currency currency,
+ String catalogVersion
+ ) {
+ return new PreflightPricingContext(
+ "model-v1",
+ "standard",
+ catalogVersion,
+ BASIS,
+ currency,
+ capability
+ );
+ }
+
+ private static TokenCountResult requestCount(long estimated, long safeUpperBound) {
+ return counted(estimated, safeUpperBound, TokenCountScope.REQUEST, BASIS);
+ }
+
+ private static TokenCountResult counted(
+ long estimated,
+ long safeUpperBound,
+ TokenCountScope scope,
+ TokenizationBasis basis
+ ) {
+ return TokenCountResult.counted(
+ estimated,
+ safeUpperBound,
+ estimated == safeUpperBound ? TokenCountAccuracy.EXACT : TokenCountAccuracy.HEURISTIC,
+ scope,
+ ESTIMATOR,
+ basis
+ );
+ }
+
+ private static TokenCountResult unavailableCount() {
+ return TokenCountResult.unavailable(
+ TokenCountUnavailableReason.ESTIMATOR_UNAVAILABLE,
+ TokenCountScope.REQUEST,
+ ESTIMATOR,
+ BASIS
+ );
+ }
+
+ private static PreflightCostResult.Bounded bounded(PreflightCostResult result) {
+ assertThat(result).isInstanceOf(PreflightCostResult.Bounded.class);
+ return (PreflightCostResult.Bounded) result;
+ }
+
+ private static void assertUnavailable(
+ PreflightCostResult result,
+ PreflightCostUnavailableReason reason
+ ) {
+ assertThat(result).isInstanceOf(PreflightCostResult.Unavailable.class);
+ assertThat(((PreflightCostResult.Unavailable) result).reason()).isEqualTo(reason);
+ }
+
+ private static BigDecimal decimal(String value) {
+ return new BigDecimal(value);
+ }
+}
diff --git a/token-pilot-core/src/test/java/io/tokenpilot/core/internal/HeuristicTokenEstimatorTest.java b/token-pilot-core/src/test/java/io/tokenpilot/core/internal/HeuristicTokenEstimatorTest.java
new file mode 100644
index 0000000..a620d09
--- /dev/null
+++ b/token-pilot-core/src/test/java/io/tokenpilot/core/internal/HeuristicTokenEstimatorTest.java
@@ -0,0 +1,177 @@
+package io.tokenpilot.core.internal;
+
+import io.tokenpilot.core.TokenEstimator;
+import io.tokenpilot.core.domain.TokenCountAccuracy;
+import io.tokenpilot.core.domain.TokenCountResult;
+import io.tokenpilot.core.domain.TokenCountScope;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.text.Normalizer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class HeuristicTokenEstimatorTest {
+
+ private final TokenEstimator estimator = LedgerComponents.utf8ByteHeuristicTokenEstimator();
+
+ @Test
+ @DisplayName("빈 문자열은 estimate와 upper bound가 0인 결과로 계산한다")
+ void estimatesEmptyTextWithZeroEstimateAndUpperBound() {
+ TokenCountResult result = estimator.estimate("");
+
+ assertThat(result).isNotNull();
+ assertThat(result.isCounted()).isTrue();
+ assertThat(result.isUnavailable()).isFalse();
+ assertThat(result.tokens()).hasValue(0L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(0L);
+ assertThat(result.isExact()).isFalse();
+ assertThat(result.accuracy()).contains(TokenCountAccuracy.HEURISTIC);
+ assertThat(result.scope()).isEqualTo(TokenCountScope.TEXT_ONLY);
+ }
+
+ @Test
+ @DisplayName("4로 나누어떨어지는 ASCII byte 길이로 estimate를 계산한다")
+ void estimatesAsciiTextWhenUtf8ByteLengthIsDivisibleByFour() {
+ TokenCountResult result = estimator.estimate("four");
+
+ assertThat(result.tokens()).hasValue(1L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(4L);
+ }
+
+ @Test
+ @DisplayName("4보다 짧은 ASCII byte 길이의 estimate를 올림한다")
+ void roundsUpAsciiTextShorterThanFourBytes() {
+ TokenCountResult result = estimator.estimate("abc");
+
+ assertThat(result.tokens()).hasValue(1L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(3L);
+ }
+
+ @Test
+ @DisplayName("나머지가 있는 ASCII byte 길이의 estimate를 올림한다")
+ void roundsUpAsciiTextWhenUtf8ByteLengthHasRemainder() {
+ TokenCountResult result = estimator.estimate("hello");
+
+ assertThat(result.tokens()).hasValue(2L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(5L);
+ }
+
+ @Test
+ @DisplayName("한글 문자열을 UTF-8 byte 길이로 계산한다")
+ void estimatesKoreanTextFromUtf8ByteLength() {
+ TokenCountResult result = estimator.estimate("한글");
+
+ assertThat(result.tokens()).hasValue(2L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(6L);
+ assertHeuristicTextOnlyMetadata(result);
+ }
+
+ @Test
+ @DisplayName("ASCII와 한글이 섞인 문자열을 UTF-8 byte 길이로 계산한다")
+ void estimatesMixedAsciiAndKoreanTextFromUtf8ByteLength() {
+ TokenCountResult result = estimator.estimate("A한");
+
+ assertThat(result.tokens()).hasValue(1L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(4L);
+ assertHeuristicTextOnlyMetadata(result);
+ }
+
+ @Test
+ @DisplayName("모든 결과에 고정된 estimator와 tokenization 기준을 포함한다")
+ void includesFixedEstimatorAndTokenizationMetadata() {
+ TokenCountResult result = estimator.estimate("");
+
+ assertThat(result.estimatorDescriptor().estimatorId())
+ .isEqualTo("tokenpilot-utf8-byte-heuristic");
+ assertThat(result.estimatorDescriptor().estimatorVersion()).isEqualTo("1");
+ assertThat(result.tokenizationBasis().id()).isEqualTo("BYTE_LEVEL_BPE_UTF8");
+ }
+
+ @Test
+ @DisplayName("null 입력을 거부한다")
+ void rejectsNullText() {
+ assertThatThrownBy(() -> estimator.estimate(null))
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ @DisplayName("짝이 없는 high surrogate를 거부한다")
+ void rejectsUnpairedHighSurrogate() {
+ assertThatThrownBy(() -> estimator.estimate("\uD800"))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ @DisplayName("짝이 없는 low surrogate를 거부한다")
+ void rejectsUnpairedLowSurrogate() {
+ assertThatThrownBy(() -> estimator.estimate("\uDC00"))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ @DisplayName("올바른 surrogate pair를 UTF-8 byte 길이로 계산한다")
+ void estimatesValidSurrogatePairFromUtf8ByteLength() {
+ TokenCountResult result = estimator.estimate("\uD83D\uDE00");
+
+ assertThat(result.tokens()).hasValue(1L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(4L);
+ assertHeuristicTextOnlyMetadata(result);
+ }
+
+ @Test
+ @DisplayName("precomposed 문자열을 원문의 UTF-8 byte 길이로 계산한다")
+ void estimatesPrecomposedTextWithoutNormalization() {
+ TokenCountResult result = estimator.estimate("\u00E9");
+
+ assertThat(result.tokens()).hasValue(1L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(2L);
+ }
+
+ @Test
+ @DisplayName("combining 문자열을 원문의 UTF-8 byte 길이로 계산한다")
+ void estimatesCombiningTextWithoutNormalization() {
+ TokenCountResult result = estimator.estimate("e\u0301");
+
+ assertThat(result.tokens()).hasValue(1L);
+ assertThat(result.safeUpperBoundTokens()).hasValue(3L);
+ }
+
+ @Test
+ @DisplayName("canonical equivalent 문자열을 같은 byte 길이로 정규화하지 않는다")
+ void preservesDifferentByteLengthsForCanonicalEquivalentText() {
+ String precomposed = "\u00E9";
+ String combining = "e\u0301";
+ assertThat(Normalizer.normalize(combining, Normalizer.Form.NFC))
+ .isEqualTo(precomposed);
+
+ TokenCountResult precomposedResult = estimator.estimate(precomposed);
+ TokenCountResult combiningResult = estimator.estimate(combining);
+
+ assertThat(precomposedResult.safeUpperBoundTokens()).hasValue(2L);
+ assertThat(combiningResult.safeUpperBoundTokens()).hasValue(3L);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "hello", "한글", "A한", "\uD83D\uDE00", "\u00E9", "e\u0301"})
+ @DisplayName("모든 정상 입력 결과는 HEURISTIC/TEXT_ONLY 계약을 유지한다")
+ void preservesHeuristicTextOnlyContractForValidText(String text) {
+ TokenCountResult result = estimator.estimate(text);
+
+ assertHeuristicTextOnlyMetadata(result);
+ assertThat(result.isExact()).isFalse();
+ assertThat(result.tokens().orElseThrow())
+ .isLessThanOrEqualTo(result.safeUpperBoundTokens().orElseThrow());
+ }
+
+ private static void assertHeuristicTextOnlyMetadata(TokenCountResult result) {
+ assertThat(result.accuracy()).contains(TokenCountAccuracy.HEURISTIC);
+ assertThat(result.scope()).isEqualTo(TokenCountScope.TEXT_ONLY);
+ assertThat(result.estimatorDescriptor().estimatorId())
+ .isEqualTo("tokenpilot-utf8-byte-heuristic");
+ assertThat(result.estimatorDescriptor().estimatorVersion()).isEqualTo("1");
+ assertThat(result.tokenizationBasis().id()).isEqualTo("BYTE_LEVEL_BPE_UTF8");
+ }
+}