feat(models): capability-based Model IR + Transport frontends (RFC #25)

docs/guide/configuration.md

@@ -28,9 +28,10 @@ models:
         args:
             concurrency_limit: 64
 
-    # Derived model with type conversion (ChatModel -> GenModel)
+    # A different kind needs its own base model — a derived model always
+    # inherits its base's kind (cross-kind `type:` removed by RFC #25)
     gen_model:
-        base: base_model
+        name: "gpt-4o"
         type: "gen"
         args:
             concurrency_limit: 32
@@ -53,8 +54,8 @@ tasks:
 
 ### Key Concepts
 
-- **Model derivation**: `base: parent_model` inherits client, limiter, and args
-- **Type conversion**: `type: "gen"` switches between ChatModel and GenModel
+- **Model derivation**: `base: parent_model` inherits client, limiter, args — and always the base's kind
+- **Model kind**: `type: "chat"` / `type: "gen"` selects ChatModel or GenModel on a base model only; cross-kind `type:` conversion on derived models was removed by RFC #25 — define a separate base model instead
 - **Quota allocation**: `concurrency_limit` in `args` reserves capacity from base
 - **Class resolution**: built-in classes (exported by `sieval.tasks` / `sieval.datasets`) use short names; custom classes must use full module paths (`my_pkg.my_module.MyTask`)
 
@@ -81,17 +82,18 @@ preprocess -> infer -> postprocess -> feedback -> report
 Hierarchical concurrency control — derive child models from a base and allocate API quotas:
 
 ```python
-from sieval.core.models import ChatModel, GenModel
+from sieval.core.models import ChatModel
 
 base = ChatModel("gpt-4o", concurrency_limit=128)
 
 math_model = base.with_args(concurrency_limit=64)   # reserves 64
 code_model = base.with_args(concurrency_limit=32)   # reserves 32
-gen_model = base.as_type(GenModel)                   # same quota, different type
 
 # base uses remaining elastic capacity (128 - 64 - 32 = 32)
 ```
 
+A derived model always keeps its base's kind. Cross-kind conversion (`as_type`) was removed by RFC #25 — to reach the same endpoint through a `GenModel`, define a separate base model instead.
+
 ## Anomaly Detection
 
 Built-in anomaly detection runs automatically after each task and saves to `anomalies.json`. Rules are filtered by task tags — custom rules can be added via `@sieval_detection_rule` (see `sieval/core/tasks/anomaly.py`).

pyproject.toml

@@ -177,7 +177,11 @@ also_copy = [
   "sieval/tasks",
   "sieval/infer",
   "sieval/cli",
+  "sieval/meta",
+  "sieval/__init__.py",
   "sieval/__main__.py",
+  "sieval/_version.py",
+  "scripts",
 ]
 
 [tool.pytest]

sieval/cli/leaderboard/session.py

@@ -24,7 +24,7 @@
 
 from sieval.cli.leaderboard.card import AlignmentCard, load_card
 from sieval.core.datasets import Dataset
-from sieval.core.models import ChatModel, GenModel, Model, SglangGenModel
+from sieval.core.models import Capability, ChatModel, GenModel, Model, SglangGenModel
 from sieval.core.runners import MultiTaskRunner, TaskRunnerConfig
 from sieval.core.tasks.context import TaskAction
 from sieval.core.types import JSONValue
@@ -1048,41 +1048,28 @@ def _setup_models(self) -> None:
                 # Extract concurrency_limit separately for with_args
                 concurrency_limit = args.pop("concurrency_limit", None)
 
-                # Check if type conversion is needed
+                # RFC #25 dropped cross-kind model conversion (as_type): a
+                # derived model always inherits its base's kind. `type:` on a
+                # derived model is accepted only when it matches the base.
                 target_type = cfg.get("type")
-                if target_type == "gen":
-                    # An sglang-backed base is already "gen"; as_type(GenModel)
-                    # would swap it to the OpenAI /v1/completions path (which
-                    # rejects echo+logprobs), silently defeating engine: sglang.
-                    # Preserve it. A plain GenModel base is unaffected.
-                    if isinstance(base_model, SglangGenModel):
-                        new_model = base_model
-                    else:
-                        new_model = base_model.as_type(GenModel)
-                    logger.info(
-                        "Created derived model '{}' from '{}' "
-                        "with type conversion to '{}'",
-                        name,
-                        base_name,
-                        target_type,
-                    )
-                elif target_type == "chat":
-                    new_model = base_model.as_type(ChatModel)
-                    logger.info(
-                        "Created derived model '{}' from '{}' "
-                        "with type conversion to '{}'",
-                        name,
-                        base_name,
-                        target_type,
-                    )
-                elif target_type:
-                    raise ValueError(
-                        f"Model '{name}' has invalid type '{target_type}'. "
-                        "Expected 'chat' or 'gen'"
+                if target_type:
+                    if target_type not in ("chat", "gen"):
+                        raise ValueError(
+                            f"Model '{name}' has invalid type '{target_type}'. "
+                            "Expected 'chat' or 'gen'"
+                        )
+                    base_kind = (
+                        "chat" if Capability.Chat in base_model.capabilities else "gen"
                     )
-                else:
-                    # No type conversion, just derive
-                    new_model = base_model
+                    if target_type != base_kind:
+                        raise ValueError(
+                            f"Derived model '{name}' cannot convert base "
+                            f"'{base_name}' from '{base_kind}' to "
+                            f"'{target_type}': cross-kind conversion was "
+                            "removed (RFC #25). Define a separate base model "
+                            "with the desired type instead."
+                        )
+                new_model = base_model
 
                 # Apply additional args (including concurrency_limit)
                 if concurrency_limit is not None or args:

sieval/core/models/__init__.py

@@ -1,15 +1,58 @@
+"""Model backends, the provider-agnostic IR, and Transport frontends.
+
+AI-Generated Code - Claude Fable 5 (Anthropic)
+"""
+
+from .capabilities import Capability
 from .chat_model import ChatModel
+from .exceptions import CapabilityError
 from .gen_model import GenModel
+from .ir import (
+    Citation,
+    GroundingChunk,
+    GroundingMetadata,
+    InputScoringResult,
+    ReasoningOutput,
+    ReasoningParams,
+    Request,
+    Response,
+    SamplingParams,
+    ServerToolSpec,
+    ServerToolType,
+    ServerToolUse,
+    TokenLogprob,
+    TopKEntry,
+    UsageStats,
+)
 from .model import Model, ModelCallMeta, ModelMeta, ModelOutput, ModelUsage
 from .sglang_gen_model import SglangGenModel
+from .transport import Transport
 
 __all__ = [
+    "Capability",
+    "CapabilityError",
     "ChatModel",
+    "Citation",
     "GenModel",
+    "GroundingChunk",
+    "GroundingMetadata",
+    "InputScoringResult",
     "Model",
     "ModelCallMeta",
     "ModelMeta",
     "ModelOutput",
     "ModelUsage",
+    "ReasoningOutput",
+    "ReasoningParams",
+    "Request",
+    "Response",
+    "SamplingParams",
+    "ServerToolSpec",
+    "ServerToolType",
+    "ServerToolUse",
     "SglangGenModel",
+    "TokenLogprob",
+    "TopKEntry",
+    "Transport",
+    "UsageStats",
 ]

sieval/core/models/capabilities.py

@@ -0,0 +1,46 @@
+"""Capability catalog for the Model IR.
+
+Each ``Capability`` represents a feature a Transport can declare support for.
+``Model.assert_capability()`` checks these at setup time — a Request using a
+feature the Transport lacks is rejected immediately, never silently ignored.
+
+AI-Generated Code - Claude Opus 4.8 (Anthropic)
+"""
+
+from enum import Flag, auto
+
+
+class Capability(Flag):
+    # ── Input modality ────────────────────────────────────────
+    Completion = auto()  # input: str (base model)
+    Chat = auto()  # input: list[Message]
+
+    # ── Tools ─────────────────────────────────────────────────
+    FunctionCalling = auto()  # client-side tool call (formerly Tools)
+    ServerTools = auto()  # provider-hosted tools master switch
+    WebSearch = auto()
+    WebFetch = auto()
+    HostedCodeExecution = auto()
+    FileSearch = auto()
+    ComputerUse = auto()
+
+    # ── Reasoning ─────────────────────────────────────────────
+    Reasoning = auto()
+    ReasoningOptional = auto()  # can be turned off (OpenAI o-series, older Claude)
+    ReasoningAlwaysOn = auto()  # forced on (Claude Fable 5 / Mythos 5)
+    ReasoningEffort = auto()  # supports effort enum
+    ReasoningBudget = auto()  # supports exact budget_tokens
+    ReasoningMode = auto()  # supports mode (OpenAI standard/pro)
+    ReasoningContext = auto()  # supports cross-turn reasoning state
+    ReasoningTaskBudget = auto()  # supports cross-call total budget (Anthropic)
+
+    # ── Logprobs / scoring ────────────────────────────────────
+    TopKLogprobs = auto()
+    InputScoring = auto()  # prompt-side logprobs (BPB/PPL)
+    SampledLogprobs = auto()
+    SampledLogprobsWithTokenIds = auto()  # token_id populated (sglang native)
+
+    # ── Other ─────────────────────────────────────────────────
+    StructuredOutput = auto()
+    Prefill = auto()
+    FIM = auto()