Refine LLM tools: modern API params

grafi/tools/llms/impl/claude_tool.py

@@ -48,8 +48,32 @@ class ClaudeTool(LLM):
     api_key: Optional[str] = Field(
         default_factory=lambda: os.getenv("ANTHROPIC_API_KEY")
     )
-    max_tokens: int = Field(default=4096)
-    model: str = Field(default="claude-haiku-4-5-20251001")  # or haiku, opus…
+    # Anthropic recommends not lowballing max_tokens; this is an upper bound, not
+    # a cost floor (you pay for what is generated). Opus 4.x supports up to 128K
+    # when streaming — raise this if you stream long outputs.
+    max_tokens: int = Field(default=16384)
+    # Default to the lowest-cost tier (Haiku). Upgrade for harder tasks, e.g.
+    # "claude-sonnet-4-6" or "claude-opus-4-8".
+    # NOTE: if you switch to Opus 4.7/4.8 they reject temperature/top_p/top_k/
+    # budget_tokens — steer with `effort`/`thinking` instead of sampling params.
+    model: str = Field(default="claude-haiku-4-5")
+    thinking: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description=(
+            "Extended/adaptive thinking config passed to the Messages API, e.g. "
+            "{'type': 'adaptive'} or {'type': 'adaptive', 'display': 'summarized'}. "
+            "Adaptive thinking is the supported mode on Claude 4.6+; "
+            "budget_tokens is rejected on Opus 4.7/4.8. Left off when None."
+        ),
+    )
+    effort: Optional[str] = Field(
+        default=None,
+        description=(
+            "Output/reasoning effort sent inside output_config: "
+            "'low' | 'medium' | 'high' | 'xhigh' | 'max'. Supported on Opus 4.5+ "
+            "and Sonnet 4.6. Left off when None (API default is 'high')."
+        ),
+    )
 
     @classmethod
     def builder(cls) -> "ClaudeToolBuilder":
@@ -146,6 +170,42 @@ def _content_to_text(content: Any) -> str:
             return ""
         return json.dumps(content, default=str)
 
+    def _request_kwargs(
+        self,
+        system: Union[str, Omit],
+        messages: List[MessageParam],
+        tools: Union[List[ToolParam], Omit],
+    ) -> Dict[str, Any]:
+        """Assemble keyword arguments shared by the streaming and non-streaming
+        calls.
+
+        ``thinking`` and ``effort`` are only included when set, so a tool that
+        doesn't use them keeps the API's defaults. ``effort`` is folded into
+        ``output_config`` (merged with any user-supplied ``output_config`` in
+        ``chat_params``). ``chat_params`` is applied last so an explicit caller
+        value always wins over the first-class fields.
+        """
+        kwargs: Dict[str, Any] = {
+            "max_tokens": self.max_tokens,
+            "model": self.model,
+            "system": system,
+            "messages": messages,
+            "tools": tools,
+        }
+
+        chat_params = dict(self.chat_params)
+
+        if self.thinking is not None:
+            kwargs["thinking"] = self.thinking
+
+        if self.effort is not None:
+            output_config = dict(chat_params.pop("output_config", {}) or {})
+            output_config.setdefault("effort", self.effort)
+            kwargs["output_config"] = output_config
+
+        kwargs.update(chat_params)
+        return kwargs
+
     # ------------------------------------------------------------------ #
     # Async call                                                         #
     # ------------------------------------------------------------------ #
@@ -156,29 +216,18 @@ async def invoke(
         input_data: Messages,
     ) -> MsgsAGen:
         system, messages, tools = self.prepare_api_input(input_data)
+        request_kwargs = self._request_kwargs(system, messages, tools)
 
         try:
             async with AsyncAnthropic(api_key=self.api_key) as client:
                 if self.is_streaming:
-                    async with client.messages.stream(
-                        max_tokens=self.max_tokens,
-                        model=self.model,
-                        system=system,
-                        messages=messages,
-                        tools=tools,
-                        **self.chat_params,
-                    ) as stream:
+                    async with client.messages.stream(**request_kwargs) as stream:
                         async for event in stream:
                             if event.type == "text":
                                 yield self.to_stream_messages(event.text)
                 else:
                     resp: AnthropicMessage = await client.messages.create(
-                        max_tokens=self.max_tokens,
-                        model=self.model,
-                        system=system,
-                        messages=messages,
-                        tools=tools,
-                        **self.chat_params,
+                        **request_kwargs
                     )
                     yield self.to_messages(resp)
 
@@ -226,6 +275,16 @@ def to_messages(self, resp: AnthropicMessage) -> Messages:
         if len(tool_calls) > 0:
             message_args["content"] = ""
 
+        # A refused response (HTTP 200, stop_reason="refusal") carries empty or
+        # partial content; surface the reason on the message so callers can react
+        # instead of treating the blank text as a normal answer.
+        if getattr(resp, "stop_reason", None) == "refusal":
+            details = getattr(resp, "stop_details", None)
+            explanation = getattr(details, "explanation", None) if details else None
+            message_args["refusal"] = (
+                explanation or "Request refused by the Anthropic safety system."
+            )
+
         return [Message.model_validate(message_args)]
 
     # ------------------------------------------------------------------ #
@@ -235,6 +294,8 @@ def to_dict(self) -> Dict[str, Any]:
         return {
             **super().to_dict(),
             "max_tokens": self.max_tokens,
+            "thinking": self.thinking,
+            "effort": self.effort,
         }
 
     @classmethod
@@ -261,8 +322,10 @@ async def from_dict(cls, data: Dict[str, Any]) -> "ClaudeTool":
             .is_streaming(data.get("is_streaming", False))
             .system_message(data.get("system_message", ""))
             .api_key(os.getenv("ANTHROPIC_API_KEY"))
-            .model(data.get("model", "claude-haiku-4-5-20251001"))
-            .max_tokens(data.get("max_tokens", 4096))
+            .model(data.get("model", "claude-haiku-4-5"))
+            .max_tokens(data.get("max_tokens", 16384))
+            .thinking(data.get("thinking"))
+            .effort(data.get("effort"))
             .build()
         )
 
@@ -280,3 +343,13 @@ def api_key(self, api_key: Optional[str]) -> Self:
     def max_tokens(self, max_tokens: int) -> Self:
         self.kwargs["max_tokens"] = max_tokens
         return self
+
+    def thinking(self, thinking: Optional[Dict[str, Any]]) -> Self:
+        """Set the extended/adaptive thinking config (e.g. {'type': 'adaptive'})."""
+        self.kwargs["thinking"] = thinking
+        return self
+
+    def effort(self, effort: Optional[str]) -> Self:
+        """Set output effort ('low' | 'medium' | 'high' | 'xhigh' | 'max')."""
+        self.kwargs["effort"] = effort
+        return self

grafi/tools/llms/impl/gemini_tool.py

@@ -57,7 +57,17 @@ class GeminiTool(LLM):
     name: str = Field(default="GeminiTool")
     type: str = Field(default="GeminiTool")
     api_key: Optional[str] = Field(default_factory=lambda: os.getenv("GEMINI_API_KEY"))
+    # Lowest-cost current Gemini tier; upgrade to "gemini-2.5-flash"/"gemini-2.5-pro"
+    # for harder tasks.
     model: str = Field(default="gemini-2.5-flash-lite")
+    thinking_budget: Optional[int] = Field(
+        default=None,
+        description=(
+            "Gemini 2.5 thinking budget in tokens. 0 disables thinking; -1 lets "
+            "the model decide dynamically. Mapped to types.ThinkingConfig and left "
+            "off when None (model default applies)."
+        ),
+    )
 
     @classmethod
     def builder(cls) -> "GeminiToolBuilder":
@@ -80,7 +90,7 @@ def _client(self) -> genai.Client:
 
     def prepare_api_input(
         self, input_data: Messages
-    ) -> tuple[ContentListUnion, Optional[Tool]]:
+    ) -> tuple[ContentListUnion, Optional[List[Tool]], Optional[str]]:
         """
         Map grafi ``Message`` objects -> Gemini *contents* list.
 
@@ -91,21 +101,24 @@ def prepare_api_input(
                 {"role": "model", "parts": [{"text": "Hello!"}]},
             ]
 
-        Function/tool declarations are passed via GenerateContentConfig,
-        so we simply return them for the caller to insert.
+        The system prompt (the instance ``system_message`` plus any ``system``
+        role messages) is returned separately so the caller can pass it via the
+        config's ``system_instruction`` field — the SDK-native way to set a
+        system prompt — rather than faking it as a leading ``user`` turn.
+        Function/tool declarations are likewise returned for the caller to insert.
         """
         contents: List[Content] = []
 
-        # prepend system instruction in Gemini style if present
+        # Collect the system prompt; folded into config.system_instruction below.
+        system_parts: List[str] = []
         if self.system_message:
-            contents.append(
-                Content(
-                    role="user",
-                    parts=[Part(text=self.system_message)],
-                )
-            )
+            system_parts.append(self.system_message)
 
         for m in input_data:
+            if m.role == "system":
+                if isinstance(m.content, str) and m.content:
+                    system_parts.append(m.content)
+                continue
             # Gemini only needs role + parts; we ignore tool_call fields here
             if m.content is not None and isinstance(m.content, str) and m.content != "":
                 contents.append(
@@ -124,11 +137,14 @@ def prepare_api_input(
             )
             function_declarations.append(function_declaration)
 
-        return contents, (
+        tools = (
             [Tool(function_declarations=function_declarations)]
             if function_declarations
             else None
         )
+        system_instruction = "\n\n".join(system_parts) if system_parts else None
+
+        return contents, tools, system_instruction
 
     # --------------------------------------------------------------------- #
     # Asynchronous (async/await) one‑shot call
@@ -139,15 +155,25 @@ async def invoke(
         invoke_context: InvokeContext,
         input_data: Messages,
     ) -> MsgsAGen:  # → async generator just like OpenAITool
-        contents, tools = self.prepare_api_input(input_data)
+        contents, tools, system_instruction = self.prepare_api_input(input_data)
 
         client = genai.Client(api_key=self.api_key)  # same lightweight client
 
-        cfg = (
-            types.GenerateContentConfig(tools=tools, **self.chat_params)  # type: ignore[arg-type]
-            if tools
-            else None
-        )
+        # Always build the config from chat_params (previously it was dropped
+        # entirely when no tools were present), then layer on tools, the system
+        # instruction, and the thinking budget when each is set.
+        config_kwargs: Dict[str, Any] = dict(self.chat_params)
+        if tools:
+            config_kwargs["tools"] = tools
+        if system_instruction:
+            config_kwargs.setdefault("system_instruction", system_instruction)
+        if self.thinking_budget is not None:
+            config_kwargs.setdefault(
+                "thinking_config",
+                types.ThinkingConfig(thinking_budget=self.thinking_budget),
+            )
+
+        cfg = types.GenerateContentConfig(**config_kwargs) if config_kwargs else None
 
         try:
             if self.is_streaming:
@@ -238,6 +264,7 @@ def to_dict(self) -> Dict[str, Any]:
             "type": self.type,
             "api_key": "****************",
             "model": self.model,
+            "thinking_budget": self.thinking_budget,
         }
 
     @classmethod
@@ -263,7 +290,8 @@ async def from_dict(cls, data: Dict[str, Any]) -> "GeminiTool":
             .is_streaming(data.get("is_streaming", False))
             .system_message(data.get("system_message", ""))
             .api_key(os.getenv("GEMINI_API_KEY"))
-            .model(data.get("model", "gemini-2.0-flash-lite"))
+            .model(data.get("model", "gemini-2.5-flash-lite"))
+            .thinking_budget(data.get("thinking_budget"))
             .build()
         )
 
@@ -276,3 +304,8 @@ class GeminiToolBuilder(LLMBuilder[GeminiTool]):
     def api_key(self, api_key: Optional[str]) -> Self:
         self.kwargs["api_key"] = api_key
         return self
+
+    def thinking_budget(self, thinking_budget: Optional[int]) -> Self:
+        """Set the Gemini 2.5 thinking budget in tokens (0 off, -1 dynamic)."""
+        self.kwargs["thinking_budget"] = thinking_budget
+        return self

grafi/tools/llms/impl/openai_tool.py

@@ -3,6 +3,7 @@
 from typing import ClassVar
 from typing import Dict
 from typing import Optional
+from typing import Self
 
 from pydantic import Field
 
@@ -16,13 +17,33 @@ class OpenAITool(OpenAICompatibleTool):
 
     Attributes:
         api_key (str): The API key for authenticating with OpenAI.
-        model (str): The name of the OpenAI model to use (default is 'gpt-4o-mini').
+        model (str): The name of the OpenAI model to use (default is 'gpt-4.1-nano').
+        reasoning_effort (str): Reasoning depth for reasoning models (o-series, gpt-5).
+        verbosity (str): Output verbosity control for gpt-5-class models.
     """
 
     name: str = Field(default="OpenAITool")
     type: str = Field(default="OpenAITool")
     api_key: Optional[str] = Field(default_factory=lambda: os.getenv("OPENAI_API_KEY"))
-    model: str = Field(default="gpt-4o-mini")
+    # Default to the lowest-cost current chat model. Upgrade to "gpt-4.1-mini"/
+    # "gpt-4.1" for harder tasks. For reasoning models (e.g. "gpt-5", "gpt-5-mini",
+    # "o4-mini") set `reasoning_effort` and prefer `max_completion_tokens` over
+    # `temperature`/`max_tokens` via chat_params.
+    model: str = Field(default="gpt-4.1-nano")
+    reasoning_effort: Optional[str] = Field(
+        default=None,
+        description=(
+            "Reasoning effort for reasoning models: 'minimal' | 'low' | 'medium' "
+            "| 'high'. Only meaningful for o-series / gpt-5 models. Left off when None."
+        ),
+    )
+    verbosity: Optional[str] = Field(
+        default=None,
+        description=(
+            "Output verbosity for gpt-5-class models: 'low' | 'medium' | 'high'. "
+            "Left off when None."
+        ),
+    )
 
     _provider_label: ClassVar[str] = "OpenAI"
 
@@ -31,6 +52,22 @@ def builder(cls) -> "OpenAIToolBuilder":
         """Return a builder for OpenAITool."""
         return OpenAIToolBuilder(cls)
 
+    def _extra_create_kwargs(self) -> Dict[str, Any]:
+        """Attach OpenAI-only request params when set (chat_params still wins)."""
+        extra: Dict[str, Any] = {}
+        if self.reasoning_effort is not None:
+            extra["reasoning_effort"] = self.reasoning_effort
+        if self.verbosity is not None:
+            extra["verbosity"] = self.verbosity
+        return extra
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            **super().to_dict(),
+            "reasoning_effort": self.reasoning_effort,
+            "verbosity": self.verbosity,
+        }
+
     @classmethod
     async def from_dict(cls, data: Dict[str, Any]) -> "OpenAITool":
         """Create an OpenAITool instance from a dictionary representation."""
@@ -45,10 +82,22 @@ async def from_dict(cls, data: Dict[str, Any]) -> "OpenAITool":
             .is_streaming(data.get("is_streaming", False))
             .system_message(data.get("system_message", ""))
             .api_key(os.getenv("OPENAI_API_KEY"))
-            .model(data.get("model", "gpt-4o-mini"))
+            .model(data.get("model", "gpt-4.1-nano"))
+            .reasoning_effort(data.get("reasoning_effort"))
+            .verbosity(data.get("verbosity"))
             .build()
         )
 
 
 class OpenAIToolBuilder(OpenAICompatibleToolBuilder[OpenAITool]):
     """Builder for OpenAITool instances."""
+
+    def reasoning_effort(self, reasoning_effort: Optional[str]) -> Self:
+        """Set reasoning effort ('minimal' | 'low' | 'medium' | 'high')."""
+        self.kwargs["reasoning_effort"] = reasoning_effort
+        return self
+
+    def verbosity(self, verbosity: Optional[str]) -> Self:
+        """Set output verbosity ('low' | 'medium' | 'high')."""
+        self.kwargs["verbosity"] = verbosity
+        return self