Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions src/praisonai/praisonai/agents_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,20 @@ def _validate_agents_config(self, config):
'planning_tools', 'planning', 'autonomy', 'guardrails', 'streaming', 'stream',
'approval', 'skills', 'cli_backend', 'runtime', 'reflection', 'handoff', 'web', 'web_fetch'
}

# Collect YAML-local model aliases (valid even if not in global catalogue)
local_model_aliases = set()
models_cfg = config.get("models", {})
if isinstance(models_cfg, dict):
local_model_aliases = {k for k in models_cfg.keys() if isinstance(k, str)}

# Try to load model catalogue for validation
model_catalogue = None
try:
from .llm.catalogue import ModelCatalogue
model_catalogue = ModelCatalogue()
except ImportError:
pass # Catalogue not available

for section_name in ('agents', 'roles'):
section = config.get(section_name, {})
Expand All @@ -548,6 +562,44 @@ def _validate_agents_config(self, config):
self.logger.warning(
f"Unknown field '{field_name}' in {entity_name} '{name}'.{suggestion}"
)

# Validate model/llm values if catalogue available
if model_catalogue:
for model_field in ('llm', 'function_calling_llm'):
model_value = section_config.get(model_field)
if model_value and isinstance(model_value, str):
# Skip validation for local model aliases defined in YAML
if model_value in local_model_aliases:
continue
try:
model_catalogue.validate_model(model_value)
except ValueError as e:
self.logger.warning(
f"Invalid model '{model_value}' in {entity_name} '{name}' field '{model_field}': {e}"
)

# Check for tools configured with non-tool-calling models
if section_config.get('tools'):
llm_value = section_config.get('llm')
if llm_value:
model_info = model_catalogue.describe_model(llm_value)
if model_info and not model_info.get('supports_tools'):
self.logger.warning(
f"{entity_name.capitalize()} '{name}' has tools configured but model '{llm_value}' does not support tool calling"
)

# Also validate top-level llm/model config
if model_catalogue:
for model_field in ('llm', 'model'):
model_value = config.get(model_field)
if model_value and isinstance(model_value, str):
# Skip validation for local model aliases defined in YAML
if model_value in local_model_aliases:
continue
try:
model_catalogue.validate_model(model_value)
except ValueError as e:
self.logger.warning(f"Invalid model '{model_value}' in top-level '{model_field}': {e}")
Comment on lines +566 to +602

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

Skip catalogue validation for YAML-local model aliases.

The new checks validate only against ModelCatalogue. If a role uses a model key defined in top-level models, this logs a false “invalid model” warning.

💡 Proposed fix
-        # Try to load model catalogue for validation
+        # Collect YAML-local model aliases (valid even if not in global catalogue)
+        local_model_aliases = set()
+        models_cfg = config.get("models", {})
+        if isinstance(models_cfg, dict):
+            local_model_aliases = {k for k in models_cfg.keys() if isinstance(k, str)}
+
+        # Try to load model catalogue for validation
         model_catalogue = None
@@
                         model_value = section_config.get(model_field)
                         if model_value and isinstance(model_value, str):
+                            if model_value in local_model_aliases:
+                                continue
                             try:
                                 model_catalogue.validate_model(model_value)
@@
                 model_value = config.get(model_field)
                 if model_value and isinstance(model_value, str):
+                    if model_value in local_model_aliases:
+                        continue
                     try:
                         model_catalogue.validate_model(model_value)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/praisonai/praisonai/agents_generator.py` around lines 560 - 590, Before
calling model_catalogue.validate_model() in both the section-level validation
loop (for 'llm' and 'function_calling_llm' fields) and the top-level validation
loop (for 'llm' and 'model' fields), check if the model_value exists in the
top-level config.get('models', {}) dictionary. If the model is defined locally
in the YAML's models section, skip the catalogue validation call to avoid false
warnings about invalid models that are actually valid local aliases.




Expand Down
1 change: 1 addition & 0 deletions src/praisonai/praisonai/cli/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ class OutputFormat(str, Enum):
"tracker": (".commands.tracker", "app", "Autonomous agent tracking with step-by-step analysis"),
"github": (".commands.github", "app", "GitHub native context tracking and Issue triage"),
"managed": (".commands.managed", "app", "Managed Agents (Anthropic cloud-hosted backend)"),
"models": (".commands.models", "app", "List and describe available models"),

# Moltbot-inspired commands
"bot": (".commands.bot", "app", "Messaging bots with full agent capabilities"),
Expand Down
264 changes: 264 additions & 0 deletions src/praisonai/praisonai/cli/commands/models.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,264 @@
"""
Models command group for PraisonAI CLI.

Provides commands to list and describe available LLM models.
"""

from typing import Optional, List, Dict, Any
import typer
import json

from ..output.console import get_output_controller

app = typer.Typer(help="List and describe available models")


@app.command(name="list")
def list_models(
provider: Optional[str] = typer.Option(None, "--provider", "-p", help="Filter by provider name"),
json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
search: Optional[str] = typer.Argument(None, help="Filter by model name pattern"),
):
"""
List available models with capabilities and limits.

Examples:
praisonai models list
praisonai models list --provider openai
praisonai models list gpt
praisonai models list --json
"""
output = get_output_controller()

try:
from ...llm.catalogue import ModelCatalogue
catalogue = ModelCatalogue()
models = catalogue.list_models(provider=provider, search=search)

if json_output:
output.print(json.dumps(models, indent=2))
return

if not models:
output.print_info("No models found matching your criteria")
return

# Group models by provider for better display
by_provider: Dict[str, List[Dict[str, Any]]] = {}
for model in models:
provider_name = model.get("provider", "unknown")
if provider_name not in by_provider:
by_provider[provider_name] = []
by_provider[provider_name].append(model)

# Display models in a table format
from rich.table import Table
from rich.console import Console

console = Console()

for provider_name, provider_models in sorted(by_provider.items()):
table = Table(title=f"\n{provider_name.upper()} Models", show_header=True, header_style="bold cyan")
table.add_column("Model ID", style="green")
table.add_column("Context", justify="right")
table.add_column("Output", justify="right")
table.add_column("Capabilities", style="yellow")
table.add_column("Cost (1K)", justify="right", style="dim")

for model in sorted(provider_models, key=lambda x: x.get("id", "")):
# Format capabilities
capabilities = []
if model.get("supports_tools"):
capabilities.append("🔧 tools")
if model.get("supports_vision"):
capabilities.append("👁️ vision")
if model.get("supports_reasoning"):
capabilities.append("🧠 reasoning")
cap_str = " ".join(capabilities) if capabilities else "-"

# Format costs
cost_str = "-"
if model.get("input_cost") is not None and model.get("output_cost") is not None:
cost_str = f"${model['input_cost']:.4f}/${model['output_cost']:.4f}"

# Format context/output limits
context = str(model.get("max_context", "-"))
output_limit = str(model.get("max_output", "-"))

table.add_row(
model.get("id", "-"),
context,
output_limit,
cap_str,
cost_str
)

console.print(table)

except ImportError:
# Show basic fallback models
fallback_models = [
{"provider": "OpenAI", "models": ["gpt-4o", "gpt-4o-mini", "gpt-3.5-turbo"]},
{"provider": "Anthropic", "models": ["claude-3-5-sonnet-latest", "claude-3-opus-latest", "claude-3-haiku-latest"]},
{"provider": "Google", "models": ["gemini-1.5-pro", "gemini-1.5-flash"]},
{"provider": "Groq", "models": ["llama-3.3-70b-versatile", "mixtral-8x7b-32768"]},
]

if json_output:
# Provide JSON output for consistency
models_list = []
for prov in fallback_models:
if provider and prov["provider"].lower() != provider.lower():
continue
for model_id in prov['models']:
if not search or search.lower() in model_id.lower():
models_list.append({"id": model_id, "provider": prov["provider"]})
output.print(json.dumps(models_list, indent=2))
else:
output.print_warning("Model catalogue not available. Install litellm for full model listing:")
output.print(" pip install 'praisonai[litellm]'")

if provider:
fallback_models = [p for p in fallback_models if p["provider"].lower() == provider.lower()]

for prov in fallback_models:
output.print_subheader(f"{prov['provider']} Models")
for model in prov['models']:
if not search or search.lower() in model.lower():
output.print(f" • {model}")
except Exception as e:
output.print_error(f"Error listing models: {e}")
raise typer.Exit(1)


@app.command(name="describe")
def describe_model(
model: str = typer.Argument(..., help="Model ID to describe (e.g., gpt-4o, claude-3-5-sonnet)"),
):
"""
Show detailed information for a specific model.

Examples:
praisonai models describe gpt-4o
praisonai models describe claude-3-5-sonnet
praisonai models describe gemini-1.5-pro
"""
output = get_output_controller()

try:
from ...llm.catalogue import ModelCatalogue
catalogue = ModelCatalogue()
info = catalogue.describe_model(model)

if not info:
output.print_error(f"Model '{model}' not found")

# Try to suggest similar models
suggestions = catalogue.get_suggestions(model)
if suggestions:
output.print_info("Did you mean one of these?")
for suggestion in suggestions[:5]:
output.print(f" • {suggestion}")
raise typer.Exit(1)

# Display model details
output.print_subheader(f"Model: {info.get('id', model)}")

if info.get("provider"):
output.print(f"Provider: {info['provider']}")

if info.get("description"):
output.print(f"Description: {info['description']}")

# Capabilities
output.print("\nCapabilities:")
output.print(f" • Tool calling: {'✅' if info.get('supports_tools') else '❌'}")
output.print(f" • Vision: {'✅' if info.get('supports_vision') else '❌'}")
output.print(f" • Reasoning: {'✅' if info.get('supports_reasoning') else '❌'}")
output.print(f" • Streaming: {'✅' if info.get('supports_streaming', True) else '❌'}")

# Limits
output.print("\nLimits:")
if info.get("max_context"):
output.print(f" • Context window: {info['max_context']:,} tokens")
if info.get("max_output"):
output.print(f" • Max output: {info['max_output']:,} tokens")

# Costs
if info.get("input_cost") is not None:
output.print("\nCosts (per 1K tokens):")
output.print(f" • Input: ${info['input_cost']:.6f}")
if info.get("output_cost") is not None:
output.print(f" • Output: ${info['output_cost']:.6f}")

# Notes
if info.get("notes"):
output.print(f"\nNotes: {info['notes']}")

except ImportError:
output.print_warning("Model catalogue not available. Install litellm for detailed model info:")
output.print(" pip install 'praisonai[litellm]'")
except typer.Exit:
# Re-raise typer.Exit without catching it
raise
except Exception as e:
output.print_error(f"Error describing model: {e}")
raise typer.Exit(1) from e


@app.command(name="validate")
def validate_model(
model: str = typer.Argument(..., help="Model ID to validate"),
):
"""
Validate if a model ID is valid and available.

Examples:
praisonai models validate gpt-4o
praisonai models validate invalid-model
"""
output = get_output_controller()

try:
from ...llm.catalogue import ModelCatalogue
catalogue = ModelCatalogue()

if catalogue.is_valid_model(model):
output.print_success(f"✅ '{model}' is a valid model")

# Show basic info if available
info = catalogue.describe_model(model)
if info:
caps = []
if info.get("supports_tools"):
caps.append("tool-calling")
if info.get("supports_vision"):
caps.append("vision")
if info.get("supports_reasoning"):
caps.append("reasoning")
if caps:
output.print(f"Capabilities: {', '.join(caps)}")
else:
output.print_error(f"❌ '{model}' is not a valid model")

# Suggest alternatives
suggestions = catalogue.get_suggestions(model)
if suggestions:
output.print_info("Did you mean one of these?")
for suggestion in suggestions[:5]:
output.print(f" • {suggestion}")
raise typer.Exit(1)

except ImportError:
output.print_warning("Model catalogue not available. Install litellm for model validation:")
output.print(" pip install 'praisonai[litellm]'")
except typer.Exit:
# Re-raise typer.Exit without catching it
raise
except Exception as e:
output.print_error(f"Error validating model: {e}")
raise typer.Exit(1) from e


if __name__ == "__main__":
app()
Loading
Loading