From e7773eafc2e55fa3ed32784bafc8022a82b8ab4b Mon Sep 17 00:00:00 2001 From: eneaxharau Date: Wed, 3 Dec 2025 14:37:38 +0100 Subject: [PATCH 1/5] feat: add dynamic fetching for ollama, lm-studio, anthropic, groq --- cli/commands/config.command.ts | 210 ++++++++------------------------- src/utils/get-models.ts | 117 ++++++++++++++++++ 2 files changed, 165 insertions(+), 162 deletions(-) create mode 100644 src/utils/get-models.ts diff --git a/cli/commands/config.command.ts b/cli/commands/config.command.ts index e294bfc..af68b77 100644 --- a/cli/commands/config.command.ts +++ b/cli/commands/config.command.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import inquirer from 'inquirer'; import chalk from 'chalk'; +import { getModels } from '../../src/utils/get-models'; const CONFIG_FILE = '.codewave.config.json'; @@ -161,176 +162,20 @@ async function initializeConfig(): Promise { value: 'lm-studio', short: 'LM Studio', }, + { name: 'Groq', value: 'groq', short: 'Groq' }, ], default: defaultProvider, }, ]); - // Provider-specific configuration with available models - const providerInfo = { - anthropic: { - defaultModel: 'claude-haiku-4-5-20251001', - models: [ - { - name: 'claude-haiku-4-5-20251001 (recommended) - Cost-optimized for multi-agent discussion (ultra-fast)', - value: 'claude-haiku-4-5-20251001', - }, - { - name: 'claude-sonnet-4-5-20250929 - Latest generation (best quality)', - value: 'claude-sonnet-4-5-20250929', - }, - { - name: 'claude-opus-4-1-20250805 - Most powerful (maximum accuracy)', - value: 'claude-opus-4-1-20250805', - }, - ], - keyFormat: 'sk-ant-...', - url: 'https://console.anthropic.com/', - }, - openai: { - defaultModel: 'gpt-4o-mini', - models: [ - { name: 'gpt-4o-mini (recommended) - Fast and cost-effective', value: 'gpt-4o-mini' }, - { name: 'gpt-4o - Latest multimodal model', value: 'gpt-4o' }, - { name: 'o3-mini - Advanced reasoning (cost-efficient)', value: 'o3-mini-2025-01-31' }, - { name: 'o3 - Most powerful reasoning model', value: 'o3' }, - ], - keyFormat: 'sk-...', - url: 'https://platform.openai.com/', - }, - google: { - defaultModel: 'gemini-2.5-flash', - models: [ - { - name: 'gemini-2.5-flash (recommended) - Best cost-performance ratio', - value: 'gemini-2.5-flash', - }, - { - name: 'gemini-2.5-flash-lite - Fastest and most efficient', - value: 'gemini-2.5-flash-lite', - }, - { name: 'gemini-2.5-pro - Best reasoning capabilities', value: 'gemini-2.5-pro' }, - ], - keyFormat: 'AIza...', - url: 'https://ai.google.dev/', - }, - xai: { - defaultModel: 'grok-4-fast-non-reasoning', - models: [ - { - name: 'grok-4-fast-non-reasoning (recommended) - Latest with 40% fewer tokens', - value: 'grok-4-fast-non-reasoning', - }, - { name: 'grok-4.2 - Polished and refined', value: 'grok-4.2' }, - { name: 'grok-4 - Advanced reasoning model', value: 'grok-4-0709' }, - ], - keyFormat: 'xai-...', - url: 'https://console.x.ai/', - }, - ollama: { - defaultModel: 'gpt-oss-20b', - models: [ - { - name: 'gpt-oss-20b (recommended) - Balanced reasoning and performance', - value: 'gpt-oss-20b', - }, - ], - keyFormat: '(no API key required)', - url: 'https://ollama.com/library', - }, - 'lm-studio': { - defaultModel: 'local-model', // LM Studio often ignores the model name if only one is loaded - models: [ - { - name: 'Load from LM Studio (uses currently loaded model)', - value: 'local-model', - }, - ], - keyFormat: '(no API key required)', - url: 'http://localhost:1234', - }, - groq: { - defaultModel: 'openai/gpt-oss-120b', - models: [ - { - name: 'openai/gpt-oss-120b (recommended) - Balanced reasoning and performance', - value: 'openai/gpt-oss-120b', - }, - ], - keyFormat: 'gsk_...', - url: 'https://console.groq.com/', - }, - }; - - const info = providerInfo[provider as keyof typeof providerInfo]; - - // Select model for the chosen provider - console.log(chalk.cyan(`\n🎯 Available ${provider} models:\n`)); - console.log( - chalk.gray('💡 CodeWave uses multi-agent discussion (3 rounds) to refine evaluations.') - ); - console.log( - chalk.gray(' Cheaper models like Haiku achieve 95%+ quality through discussion refinement.\n') - ); - - // Use existing model as default if it's valid for this provider, otherwise use provider default - let defaultModel = info.defaultModel; - if (config.llm.model && info.models.some((m) => m.value === config.llm.model)) { - defaultModel = config.llm.model; - } - - const { selectedModel } = await inquirer.prompt([ - { - type: 'list', - name: 'selectedModel', - message: `Choose ${provider} model:`, - choices: info.models, - default: defaultModel, - }, - ]); - - // Show cost comparison for selected provider - const costByProvider = { - anthropic: { - 'claude-haiku-4-5-20251001': '$0.025/commit', - 'claude-sonnet-4-5-20250929': '$0.15/commit', - 'claude-opus-4-1-20250805': '$0.40/commit', - }, - openai: { - 'gpt-4o-mini': '$0.008/commit', - 'gpt-4o': '$0.10/commit', - 'o3-mini-2025-01-31': '$0.20/commit', - o3: '$0.40/commit', - }, - google: { - 'gemini-2.5-flash': '$0.010/commit', - 'gemini-2.5-flash-lite': '$0.006/commit', - 'gemini-2.5-pro': '$0.06/commit', - }, - xai: { - 'grok-4-fast-non-reasoning': '$0.08/commit', - 'grok-4.2': '$0.08/commit', - 'grok-4-0709': '$0.08/commit', - }, - }; - - const providerCosts = costByProvider[provider as keyof typeof costByProvider]; - if (providerCosts) { - const cost = providerCosts[selectedModel as keyof typeof providerCosts]; - if (cost) { - console.log(chalk.gray(`\n✓ Selected: ${selectedModel}`)); - console.log(chalk.gray(` Cost: ${cost} (estimated for 3-round multi-agent discussion)`)); - } - } + config.llm.provider = provider; - let apiKey = ''; + let apiKey = null; if (provider !== 'ollama' && provider !== 'lm-studio') { - console.log(chalk.gray(`\nGet your API key at: ${info.url}\n`)); - const existingApiKey = config.apiKeys[provider]; const apiKeyPromptMessage = existingApiKey - ? `Enter ${provider} API key (${info.keyFormat}) [press Enter to keep existing]:` - : `Enter ${provider} API key (${info.keyFormat}):`; + ? `Enter ${provider} API key [press Enter to keep existing]:` + : `Enter ${provider} API key:`; const response = await inquirer.prompt([ { @@ -355,7 +200,48 @@ async function initializeConfig(): Promise { if (apiKey && apiKey.trim().length > 0) { config.apiKeys[provider] = apiKey.trim(); } - config.llm.provider = provider; + + if (provider === 'ollama' || provider === 'lm-studio') { + let existingBaseUrl = config.llm.baseUrl; + const baseUrlPromptMessage = existingBaseUrl + ? `Enter ${provider} base URL [press Enter to keep existing] (${existingBaseUrl}):` + : `Enter ${provider} base URL:`; + + if (!existingBaseUrl) { + existingBaseUrl = + provider === 'ollama' ? 'http://localhost:11434' : 'http://localhost:1234/v1'; + } + const { baseUrl } = await inquirer.prompt([ + { + type: 'input', + name: 'baseUrl', + message: baseUrlPromptMessage, + default: existingBaseUrl, + }, + ]); + config.llm.baseUrl = baseUrl; + } + + const models = await getModels(config); + + // Select model for the chosen provider + console.log(chalk.cyan(`\n🎯 Available ${provider} models:\n`)); + console.log( + chalk.gray('💡 CodeWave uses multi-agent discussion (3 rounds) to refine evaluations.') + ); + console.log( + chalk.gray(' Cheaper models like Haiku achieve 95%+ quality through discussion refinement.\n') + ); + + const { selectedModel } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedModel', + message: `Choose ${provider} model:`, + choices: models, + default: models[0].value, + }, + ]); config.llm.model = selectedModel; console.log(chalk.green(`\n✅ Configured to use: ${provider} (${selectedModel})`)); diff --git a/src/utils/get-models.ts b/src/utils/get-models.ts new file mode 100644 index 0000000..bc00eba --- /dev/null +++ b/src/utils/get-models.ts @@ -0,0 +1,117 @@ +import { AppConfig } from 'config/config.interface'; + +interface GenericModel { + name: string; + value: string; +} + +export async function getModels(config: AppConfig): Promise { + const provider = config.llm.provider; + const baseUrl = config.llm.baseUrl; + + if (!provider) { + throw new Error('Provider is required'); + } + + let models: GenericModel[] = []; + + switch (provider) { + case 'ollama': + models = await getOllamaModels(baseUrl); + break; + case 'lm-studio': + models = await getLMStudioModels(baseUrl); + break; + case 'groq': + models = await getGroqModels(config.apiKeys.groq); + break; + case 'anthropic': + models = await getAnthropicModels(config.apiKeys.anthropic); + break; + } + + return models; +} + +interface OllamaModel { + name: string; + model: string; +} + +async function getOllamaModels(baseUrl: string | undefined): Promise { + const response = await fetch(`${baseUrl}/api/tags`); + + if (!response.ok) { + throw new Error(`Failed to fetch models from ${baseUrl}`); + } + + const data = (await response.json()) as { models: OllamaModel[] }; + return data.models.map((model) => ({ + name: model.name, + value: model.model, + })); +} + +interface LMStudioModel { + id: string; +} + +async function getLMStudioModels(baseUrl: string | undefined): Promise { + const response = await fetch(`${baseUrl}/models`); + + if (!response.ok) { + throw new Error(`Failed to fetch models from ${baseUrl}`); + } + const data = (await response.json()) as { data: LMStudioModel[] }; + console.log('data', { data }); + return data.data.map((model) => ({ + name: model.id, + value: model.id, + })); +} + +interface GroqModel { + id: string; +} + +async function getGroqModels(apiKey: string): Promise { + const response = await fetch('https://api.groq.com/openai/v1/models', { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch models from Groq`); + } + const data = (await response.json()) as { data: GroqModel[] }; + return data.data.map((model) => ({ + name: model.id, + value: model.id, + })); +} + +interface AnthropicModel { + id: string; + display_name: string; +} + +async function getAnthropicModels(apiKey: string): Promise { + const response = await fetch('https://api.anthropic.com/v1/models', { + headers: { + 'X-Api-Key': apiKey, + 'anthropic-version': '2023-06-01', + }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch models from Anthropic`); + } + + const data = (await response.json()) as { data: AnthropicModel[] }; + + return data.data.map((model) => ({ + name: model.display_name, + value: model.id, + })); +} + +async function getOpenAIModels(apiKey: string): Promise {} From d61888b453e23db57dded4499f20e9023b6d0938 Mon Sep 17 00:00:00 2001 From: eneaxharau Date: Wed, 3 Dec 2025 15:16:09 +0100 Subject: [PATCH 2/5] feat: add fetching models dynamically for xai, google, openai --- src/utils/get-models.ts | 75 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/utils/get-models.ts b/src/utils/get-models.ts index bc00eba..cfd3a0a 100644 --- a/src/utils/get-models.ts +++ b/src/utils/get-models.ts @@ -3,6 +3,10 @@ import { AppConfig } from 'config/config.interface'; interface GenericModel { name: string; value: string; + inputTokenLimit?: number; + outputTokenLimit?: number; + pricePerInputToken?: number; + pricePerOutputToken?: number; } export async function getModels(config: AppConfig): Promise { @@ -28,6 +32,15 @@ export async function getModels(config: AppConfig): Promise { case 'anthropic': models = await getAnthropicModels(config.apiKeys.anthropic); break; + case 'openai': + models = await getOpenAIModels(config.apiKeys.openai); + break; + case 'google': + models = await getGoogleModels(config.apiKeys.google); + break; + case 'xai': + models = await getXAIModels(config.apiKeys.xai); + break; } return models; @@ -114,4 +127,64 @@ async function getAnthropicModels(apiKey: string): Promise { })); } -async function getOpenAIModels(apiKey: string): Promise {} +interface OpenAIModel { + id: string; +} + +async function getOpenAIModels(apiKey: string): Promise { + const response = await fetch('https://api.openai.com/v1/models', { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch models from OpenAI`); + } + const data = (await response.json()) as { data: OpenAIModel[] }; + return data.data.map((model) => ({ + name: model.id, + value: model.id, + })); +} + +interface GoogleModel { + name: string; + displayName: string; + baseModelId: string; + inputTokenLimit: number; + outputTokenLimit: number; +} +async function getGoogleModels(apiKey: string): Promise { + const response = await fetch('https://generativelanguage.googleapis.com/v1beta/models', { + headers: { + 'x-goog-api-key': apiKey, + }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch models from Google`); + } + const data = (await response.json()) as { models: GoogleModel[] }; + return data.models.map((model) => ({ + name: model.displayName, + value: model.name, + })); +} + +interface XAIModel { + id: string; +} +async function getXAIModels(apiKey: string): Promise { + const response = await fetch('https://api.x.ai/v1/models', { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch models from XAI`); + } + const data = (await response.json()) as { data: XAIModel[] }; + return data.data.map((model) => ({ + name: model.id, + value: model.id, + })); +} From 921f3bdd9e478fd256f031186f34e1beab5e644e Mon Sep 17 00:00:00 2001 From: eneaxharau Date: Wed, 3 Dec 2025 15:24:19 +0100 Subject: [PATCH 3/5] feat: add new langchain module for xai and fix prettier issues --- README.md | 1 + cli/utils/shared.utils.ts | 5 ++--- package-lock.json | 16 ++++++++++++++++ package.json | 1 + src/llm/llm-service.ts | 10 ++++------ src/orchestrator/commit-evaluation-graph.ts | 5 ++--- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e7eb4b4..18c5729 100644 --- a/README.md +++ b/README.md @@ -972,6 +972,7 @@ Choose your LLM provider and model based on your needs and budget: - **Alternatives**: openai/gpt-oss-20b **Ollama (Local LLMs — Free)** + - Run models entirely on your machine (no API key required) - Supports **Llama 3**, **Mistral**, **Gemma 2**, and more - Works offline once the model is pulled diff --git a/cli/utils/shared.utils.ts b/cli/utils/shared.utils.ts index 3db2ca5..4a0a035 100644 --- a/cli/utils/shared.utils.ts +++ b/cli/utils/shared.utils.ts @@ -433,9 +433,8 @@ export async function createEvaluationDirectory( * Calculate averaged metrics from agent results using weighted averaging (matching report calculations) */ async function calculateAveragedMetrics(evaluationDir: string): Promise { - const { MetricsCalculationService } = await import( - '../../src/services/metrics-calculation.service.js' - ); + const { MetricsCalculationService } = + await import('../../src/services/metrics-calculation.service.js'); return MetricsCalculationService.loadMetricsFromDirectory(evaluationDir); } diff --git a/package-lock.json b/package-lock.json index 6ab1383..d6cd65c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@langchain/langgraph": "^1.0.2", "@langchain/ollama": "^1.0.3", "@langchain/openai": "^1.1.3", + "@langchain/xai": "^1.0.2", "@types/cli-progress": "^3.11.6", "@types/inquirer": "^9.0.9", "chalk": "^4.1.2", @@ -525,6 +526,21 @@ "@langchain/core": "^1.0.0" } }, + "node_modules/@langchain/xai": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@langchain/xai/-/xai-1.0.2.tgz", + "integrity": "sha512-ImwVq5wWzBZXvkUNFE02qw11rMK7a+1f537xVisgoaRODng5oQv4AfPlNEBMT2MKLv4C6POSY1rhC7f/YKriGQ==", + "license": "MIT", + "dependencies": { + "@langchain/openai": "1.1.3" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.0.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/package.json b/package.json index fb54551..4b6d739 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "@langchain/langgraph": "^1.0.2", "@langchain/ollama": "^1.0.3", "@langchain/openai": "^1.1.3", + "@langchain/xai": "^1.0.2", "@types/cli-progress": "^3.11.6", "@types/inquirer": "^9.0.9", "chalk": "^4.1.2", diff --git a/src/llm/llm-service.ts b/src/llm/llm-service.ts index a702f1a..f75d429 100644 --- a/src/llm/llm-service.ts +++ b/src/llm/llm-service.ts @@ -3,6 +3,7 @@ import { ChatAnthropic } from '@langchain/anthropic'; import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; import { ChatOllama } from '@langchain/ollama'; import { ChatGroq } from '@langchain/groq'; +import { ChatXAI } from '@langchain/xai'; import { AppConfig } from '../config/config.interface'; export class LLMService { @@ -76,14 +77,11 @@ export class LLMService { }); case 'xai': - return new ChatOpenAI({ - openAIApiKey: apiKey, + return new ChatXAI({ + apiKey, temperature, maxTokens, - modelName: model, - configuration: { - baseURL: 'https://api.x.ai/v1', - }, + model, }); case 'groq': { return new ChatGroq({ diff --git a/src/orchestrator/commit-evaluation-graph.ts b/src/orchestrator/commit-evaluation-graph.ts index 9b96c55..b5ec78e 100644 --- a/src/orchestrator/commit-evaluation-graph.ts +++ b/src/orchestrator/commit-evaluation-graph.ts @@ -276,9 +276,8 @@ export function createCommitEvaluationGraph(agentRegistry: AgentRegistry, config console.log('📝 Generating developer overview from commit diff...'); - const { DeveloperOverviewGenerator } = await import( - '../services/developer-overview-generator.js' - ); + const { DeveloperOverviewGenerator } = + await import('../services/developer-overview-generator.js'); const { LLMService } = await import('../llm/llm-service.js'); const generator = new DeveloperOverviewGenerator(config); From fd2ae67f626f1ec1daa450dd8bc95ed8d4ccc2dc Mon Sep 17 00:00:00 2001 From: eneaxharau Date: Fri, 5 Dec 2025 18:31:08 +0100 Subject: [PATCH 4/5] feat: create a file for model information --- cli/commands/config.command.ts | 16 + cli/commands/evaluate-command.ts | 2 +- src/config/config.interface.ts | 4 +- src/config/default-config.ts | 2 - src/constants/provider-models.ts | 500 +++++++++++++++++++++++++++++++ src/types/model.types.ts | 48 +++ src/utils/get-models.ts | 151 +--------- 7 files changed, 580 insertions(+), 143 deletions(-) create mode 100644 src/constants/provider-models.ts create mode 100644 src/types/model.types.ts diff --git a/cli/commands/config.command.ts b/cli/commands/config.command.ts index af68b77..da643f7 100644 --- a/cli/commands/config.command.ts +++ b/cli/commands/config.command.ts @@ -240,9 +240,25 @@ async function initializeConfig(): Promise { message: `Choose ${provider} model:`, choices: models, default: models[0].value, + loop: false, }, ]); config.llm.model = selectedModel; + const metadata = models.find((model) => model.value === selectedModel); + + if (metadata) { + const inputPricePerToken = parseFloat(metadata.pricing.input); + const outputPricePerToken = parseFloat(metadata.pricing.output); + const cost = (inputPricePerToken + outputPricePerToken) * DEFAULT_CONFIG.llm.maxTokens * 3; + if (cost) { + console.log(chalk.gray(`\n✓ Selected: ${chalk.cyanBright(selectedModel)}`)); + console.log( + chalk.gray( + ` Cost: ${chalk.green(`$${cost.toFixed(2)} USD`)} per 3-round multi-agent discussion (estimated)` + ) + ); + } + } console.log(chalk.green(`\n✅ Configured to use: ${provider} (${selectedModel})`)); diff --git a/cli/commands/evaluate-command.ts b/cli/commands/evaluate-command.ts index a46b39d..4ef6dd5 100644 --- a/cli/commands/evaluate-command.ts +++ b/cli/commands/evaluate-command.ts @@ -149,7 +149,7 @@ export async function runEvaluateCommand(args: string[]) { // Get API key for selected provider const provider = config.llm.provider; - const apiKey = config.apiKeys[provider]; + const apiKey = config.apiKeys?.[provider as keyof typeof config.apiKeys]; if (!apiKey) { console.log(chalk.red(`\n❌ No API key configured for provider: ${provider}\n`)); diff --git a/src/config/config.interface.ts b/src/config/config.interface.ts index 23ce8b6..bc8f18c 100644 --- a/src/config/config.interface.ts +++ b/src/config/config.interface.ts @@ -9,8 +9,6 @@ export interface AppConfig { openai: string; google: string; xai: string; - ollama: string; - 'lm-studio': string; groq: string; }; llm: { @@ -18,7 +16,7 @@ export interface AppConfig { model: string; temperature: number; maxTokens: number; - baseUrl?: string; + baseUrl?: string | undefined; }; agents: { enabled: string[]; diff --git a/src/config/default-config.ts b/src/config/default-config.ts index f6398e1..a6c41e2 100644 --- a/src/config/default-config.ts +++ b/src/config/default-config.ts @@ -9,8 +9,6 @@ export const DEFAULT_CONFIG: AppConfig = { openai: '', google: '', xai: '', - ollama: '', - 'lm-studio': '', groq: '', }, llm: { diff --git a/src/constants/provider-models.ts b/src/constants/provider-models.ts new file mode 100644 index 0000000..2831939 --- /dev/null +++ b/src/constants/provider-models.ts @@ -0,0 +1,500 @@ +export const PROVIDER_MODELS = { + groq: [ + { + name: 'Qwen 3 32B - [High Intelligence, Low Cost, Super Fast Speed]', + value: 'qwen/qwen3-32b', + pricing: { + input: '0.00000029', + output: '0.00000059', + }, + }, + { + name: 'meta-llama/llama-4-scout-17b-16e-instruct - [Medium Intelligence, Low Cost, Super Fast Speed]', + value: 'meta-llama/llama-4-scout-17b-16e-instruct', + pricing: { + input: '0.00000011', + output: '0.00000034', + }, + }, + { + name: 'llama-3.3-70b-versatile - [High Intelligence, Low Cost, Super Fast Speed]', + value: 'llama-3.3-70b-versatile', + pricing: { + input: '0.00000059', + output: '0.00000079', + }, + }, + { + name: 'moonshotai/kimi-k2-instruct - [Medium Intelligence, Low Cost, Super Fast Speed]', + value: 'moonshotai/kimi-k2-instruct', + pricing: { + input: '0.000001', + output: '0.000003', + }, + }, + { + name: 'moonshotai/kimi-k2-instruct-0905 - [Medium Intelligence, Low Cost, Super Fast Speed]', + value: 'moonshotai/kimi-k2-instruct-0905', + pricing: { + input: '0.000001', + output: '0.000003', + }, + }, + { + name: 'meta-llama/llama-4-maverick-17b-128e-instruct - [Medium Intelligence, Low Cost, Super Fast Speed]', + value: 'meta-llama/llama-4-maverick-17b-128e-instruct', + pricing: { + input: '0.0000002', + output: '0.0000006', + }, + }, + { + name: 'GPT-OSS 120B - [Medium-High Intelligence, Low Cost, Super Fast Speed]', + value: 'openai/gpt-oss-120b', + pricing: { + input: '0.00000015', + output: '0.0000006', + }, + }, + { + name: 'GPT-OSS 20B - [Medium Intelligence, Low Cost, Super Fast Speed]', + value: 'openai/gpt-oss-20b', + pricing: { + input: '0.000000075', + output: '0.0000003', + }, + }, + { + name: 'llama-3.1-8b-instant - [Medium Intelligence, Low Cost, Super Fast Speed]', + value: 'llama-3.1-8b-instant', + pricing: { + input: '0.00000005', + output: '0.00000008', + }, + }, + ], + anthropic: [ + { + name: 'Claude Opus 4.5 - [High Intelligence, High Cost, Slow Speed]', + value: 'claude-opus-4-5-20251101', + pricing: { + input: '0.000005', + output: '0.000025', + }, + }, + { + name: 'Claude Haiku 4.5 - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'claude-haiku-4-5-20251001', + pricing: { + input: '0.000001', + output: '0.000005', + }, + }, + { + name: 'Claude Sonnet 4.5 - [High Intelligence, Medium Cost, Standard Speed]', + value: 'claude-sonnet-4-5-20250929', + pricing: { + input: '0.000003', + output: '0.000015', + }, + }, + { + name: 'Claude Opus 4.1 - [High Intelligence, High Cost, Slow Speed]', + value: 'claude-opus-4-1-20250805', + pricing: { + input: '0.000015', + output: '0.000075', + }, + }, + { + name: 'Claude Opus 4 - [High Intelligence, High Cost, Slow Speed]', + value: 'claude-opus-4-20250514', + pricing: { + input: '0.000015', + output: '0.000075', + }, + }, + { + name: 'Claude Sonnet 4 - [High Intelligence, Medium Cost, Standard Speed]', + value: 'claude-sonnet-4-20250514', + pricing: { + input: '0.000003', + output: '0.000015', + }, + }, + { + name: 'Claude Sonnet 3.7 - [Medium Intelligence, Medium Cost, Standard Speed]', + value: 'claude-3-7-sonnet-20250219', + pricing: { + input: '0.000003', + output: '0.000015', + }, + }, + { + name: 'Claude Haiku 3.5 - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'claude-3-5-haiku-20241022', + pricing: { + input: '0.0000008', + output: '0.000004', + }, + }, + { + name: 'Claude Haiku 3 - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'claude-3-haiku-20240307', + pricing: { + input: '0.00000025', + output: '0.00000125', + }, + }, + { + name: 'Claude Opus 3 - [Medium Intelligence, High Cost, Slow Speed]', + value: 'claude-3-opus-20240229', + pricing: { + input: '0.000015', + output: '0.000075', + }, + }, + ], + openai: [ + { + name: 'gpt-4 - [Medium Intelligence, High Cost, Standard Speed]', + value: 'gpt-4', + pricing: { + input: '0.00003', + output: '0.00006', + }, + }, + { + name: 'gpt-3.5-turbo - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'gpt-3.5-turbo', + pricing: { + input: '0.0000005', + output: '0.0000015', + }, + }, + { + name: 'gpt-5.1-codex-mini - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'gpt-5.1-codex-mini', + pricing: { + input: '0.00000025', + output: '0.000002', + }, + }, + { + name: 'gpt-5.1 - [High Intelligence, High Cost, Standard Speed]', + value: 'gpt-5.1', + pricing: { + input: '0.00000125', + output: '0.00001', + }, + }, + { + name: 'gpt-5.1-codex - [High Intelligence, High Cost, Standard Speed]', + value: 'gpt-5.1-codex', + pricing: { + input: '0.00000125', + output: '0.00001', + }, + }, + { + name: 'gpt-3.5-turbo-instruct - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'gpt-3.5-turbo-instruct', + pricing: { + input: '0.0000015', + output: '0.000002', + }, + }, + { + name: 'gpt-4-1106-preview - [Medium-High Intelligence, Medium Cost, Standard Speed]', + value: 'gpt-4-1106-preview', + pricing: { + input: '0.00001', + output: '0.00003', + }, + }, + { + name: 'gpt-3.5-turbo-1106 - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'gpt-3.5-turbo-1106', + pricing: { + input: '0.000003', + output: '0.000004', + }, + }, + { + name: 'gpt-4-0125-preview - [Medium-High Intelligence, Medium Cost, Standard Speed]', + value: 'gpt-4-0125-preview', + pricing: { + input: '0.00001', + output: '0.00003', + }, + }, + { + name: 'gpt-4-turbo-preview - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'gpt-4-turbo-preview', + pricing: { + input: '0.00001', + output: '0.00003', + }, + }, + { + name: 'gpt-3.5-turbo-0125 - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'gpt-3.5-turbo-0125', + pricing: { + input: '0.000003', + output: '0.000004', + }, + }, + { + name: 'gpt-4-turbo - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'gpt-4-turbo', + pricing: { + input: '0.00001', + output: '0.00003', + }, + }, + { + name: 'gpt-4-turbo-2024-04-09 - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'gpt-4-turbo-2024-04-09', + pricing: { + input: '0.00001', + output: '0.00003', + }, + }, + { + name: 'gpt-4o - [High Intelligence, Medium Cost, Standard Speed]', + value: 'gpt-4o', + pricing: { + input: '0.0000025', + output: '0.00001', + }, + }, + { + name: 'gpt-4o-mini - [Medium-High Intelligence, Low Cost, Fast Speed]', + value: 'gpt-4o-mini', + pricing: { + input: '0.00000015', + output: '0.0000006', + }, + }, + { + name: 'o1 - [High Intelligence, High Cost, Standard Speed, Reasoning]', + value: 'o1', + pricing: { + input: '0.00015', + output: '0.0006', + }, + }, + { + name: 'o3-mini - [Medium-High Intelligence, Medium Cost, Fast Speed, Reasoning]', + value: 'o3-mini', + pricing: { + input: '0.0000011', + output: '0.0000044', + }, + }, + { + name: 'o1-pro - [High Intelligence, High Cost, Standard Speed, Reasoning]', + value: 'o1-pro', + pricing: { + input: '0.00015', + output: '0.0006', + }, + }, + { + name: 'o3 - [High Intelligence, High Cost, Standard Speed, Reasoning]', + value: 'o3', + pricing: { + input: '0.00001', + output: '0.00004', + }, + }, + { + name: 'o4-mini - [Medium-High Intelligence, Medium Cost, Fast Speed, Reasoning]', + value: 'o4-mini', + pricing: { + input: '0.000002', + output: '0.000008', + }, + }, + { + name: 'gpt-4.1 - [Medium Intelligence, High Cost, Standard Speed]', + value: 'gpt-4.1', + pricing: { + input: '0.000002', + output: '0.000008', + }, + }, + { + name: 'gpt-4.1-mini - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'gpt-4.1-mini', + pricing: { + input: '0.0000004', + output: '0.0000016', + }, + }, + { + name: 'gpt-4.1-nano - [Low Intelligence, Low Cost, Fast Speed]', + value: 'gpt-4.1-nano', + pricing: { + input: '0.0000001', + output: '0.0000004', + }, + }, + { + name: 'o3-pro - [High Intelligence, High Cost, Standard Speed, Reasoning]', + value: 'o3-pro', + pricing: { + input: '0.00002', + output: '0.00008', + }, + }, + { + name: 'gpt-5 - [High Intelligence, High Cost, Standard Speed]', + value: 'gpt-5', + pricing: { + input: '0.00000025', + output: '0.000002', + }, + }, + { + name: 'gpt-5-mini - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'gpt-5-mini', + pricing: { + input: '0.00000025', + output: '0.000002', + }, + }, + { + name: 'gpt-5-nano - [Medium-High Intelligence, High Cost, Fast Speed]', + value: 'gpt-5-nano', + pricing: { + input: '0.00000005', + output: '0.0000004', + }, + }, + { + name: 'gpt-5-codex - [High Intelligence, High Cost, Standard Speed]', + value: 'gpt-5-codex', + pricing: { + input: '0.00000125', + output: '0.00001', + }, + }, + { + name: 'gpt-5-pro - [High Intelligence, High Cost, Standard Speed]', + value: 'gpt-5-pro', + pricing: { + input: '0.000015', + output: '0.00012', + }, + }, + ], + google: [ + { + name: 'Gemini 2.5 Flash - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'models/gemini-2.5-flash', + pricing: { + input: '0.0000003', + output: '0.0000025', + }, + }, + { + name: 'Gemini 2.5 Pro - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'models/gemini-2.5-pro', + pricing: { + input: '0.00000125', + output: '0.00001', + }, + }, + { + name: 'Gemini 2.5 Flash-Lite - [Medium Intelligence, Low Cost, Fast Speed]', + value: 'models/gemini-2.5-flash-lite', + pricing: { + input: '0.0000001', + output: '0.0000004', + }, + }, + { + name: 'Gemini 3 Pro Preview - [Medium-High Intelligence, Medium Cost, Fast Speed]', + value: 'models/gemini-3-pro-preview', + pricing: { + input: '0.000002', + output: '0.000012', + }, + }, + ], + xai: [ + { + name: 'grok-2-1212 - [Medium Intelligence, Medium Cost, Standard Speed]', + value: 'grok-2-1212', + pricing: { + input: '0', + output: '0', + }, + }, + { + name: 'grok-3 - [High Intelligence, Medium Cost, Standard Speed]', + value: 'grok-3', + pricing: { + input: '0.000003', + output: '0.000015', + }, + }, + { + name: 'grok-3-mini - [Medium-High Intelligence, Low Cost, Fast Speed]', + value: 'grok-3-mini', + pricing: { + input: '0.0000003', + output: '0.0000005', + }, + }, + { + name: 'Grok 4 (July 2025) - [High Intelligence, Medium Cost, Standard Speed]', + value: 'grok-4-0709', + pricing: { + input: '0.000003', + output: '0.000015', + }, + }, + { + name: 'Grok 4.1 Fast - [High Intelligence, Medium Cost, Fast Speed, Reasoning]', + value: 'grok-4-1-fast-non-reasoning', + pricing: { + input: '0.0000002', + output: '0.0000005', + }, + }, + { + name: 'Grok 4.1 Fast Reasoning - [High Intelligence, Medium Cost, Fast Speed, Reasoning]', + value: 'grok-4-1-fast-reasoning', + pricing: { + input: '0.0000002', + output: '0.0000005', + }, + }, + { + name: 'Grok 4 Fast - [High Intelligence, Medium Cost, Fast Speed, Reasoning]', + value: 'grok-4-fast-non-reasoning', + pricing: { + input: '0.0000002', + output: '0.0000005', + }, + }, + { + name: 'Grok 4 Fast Reasoning - [High Intelligence, Medium Cost, Fast Speed, Reasoning]', + value: 'grok-4-fast-reasoning', + pricing: { + input: '0.0000002', + output: '0.0000005', + }, + }, + { + name: 'Grok Code Fast 1 - [Medium Intelligence, Medium Cost, Fast Speed]', + value: 'grok-code-fast-1', + pricing: { + input: '0.0000002', + output: '0.0000015', + }, + }, + ], +} as const; diff --git a/src/types/model.types.ts b/src/types/model.types.ts new file mode 100644 index 0000000..0e3b9df --- /dev/null +++ b/src/types/model.types.ts @@ -0,0 +1,48 @@ +import { PROVIDER_MODELS } from '../constants/provider-models'; + +export interface GenericModel { + name: string; + value: string; + pricing: { + input: string; + output: string; + }; +} + +export type LLMProvider = keyof typeof PROVIDER_MODELS; +export type LLMProviderModels = (typeof PROVIDER_MODELS)[LLMProvider][number]['value']; +export type Models = { [key in LLMProvider]: GenericModel[] }; + +export interface OllamaModel { + name: string; + model: string; +} + +export interface LMStudioModel { + id: string; +} + +export interface GroqModel { + id: string; +} + +export interface AnthropicModel { + id: string; + display_name: string; +} + +export interface OpenAIModel { + id: string; +} + +export interface GoogleModel { + name: string; + displayName: string; + baseModelId: string; + inputTokenLimit: number; + outputTokenLimit: number; +} + +export interface XAIModel { + id: string; +} diff --git a/src/utils/get-models.ts b/src/utils/get-models.ts index cfd3a0a..42840bf 100644 --- a/src/utils/get-models.ts +++ b/src/utils/get-models.ts @@ -1,13 +1,6 @@ import { AppConfig } from 'config/config.interface'; - -interface GenericModel { - name: string; - value: string; - inputTokenLimit?: number; - outputTokenLimit?: number; - pricePerInputToken?: number; - pricePerOutputToken?: number; -} +import { GenericModel, LLMProvider, LMStudioModel, OllamaModel } from 'types/model.types'; +import { PROVIDER_MODELS } from '../constants/provider-models'; export async function getModels(config: AppConfig): Promise { const provider = config.llm.provider; @@ -26,31 +19,14 @@ export async function getModels(config: AppConfig): Promise { case 'lm-studio': models = await getLMStudioModels(baseUrl); break; - case 'groq': - models = await getGroqModels(config.apiKeys.groq); - break; - case 'anthropic': - models = await getAnthropicModels(config.apiKeys.anthropic); - break; - case 'openai': - models = await getOpenAIModels(config.apiKeys.openai); - break; - case 'google': - models = await getGoogleModels(config.apiKeys.google); - break; - case 'xai': - models = await getXAIModels(config.apiKeys.xai); + default: { + models = (await getProviderModels(provider)) as GenericModel[]; break; + } } - return models; } -interface OllamaModel { - name: string; - model: string; -} - async function getOllamaModels(baseUrl: string | undefined): Promise { const response = await fetch(`${baseUrl}/api/tags`); @@ -62,13 +38,13 @@ async function getOllamaModels(baseUrl: string | undefined): Promise ({ name: model.name, value: model.model, + pricing: { + input: '0', + output: '0', + }, })); } -interface LMStudioModel { - id: string; -} - async function getLMStudioModels(baseUrl: string | undefined): Promise { const response = await fetch(`${baseUrl}/models`); @@ -76,115 +52,16 @@ async function getLMStudioModels(baseUrl: string | undefined): Promise ({ name: model.id, value: model.id, - })); -} - -interface GroqModel { - id: string; -} - -async function getGroqModels(apiKey: string): Promise { - const response = await fetch('https://api.groq.com/openai/v1/models', { - headers: { - Authorization: `Bearer ${apiKey}`, + pricing: { + input: '0', + output: '0', }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch models from Groq`); - } - const data = (await response.json()) as { data: GroqModel[] }; - return data.data.map((model) => ({ - name: model.id, - value: model.id, })); } -interface AnthropicModel { - id: string; - display_name: string; -} - -async function getAnthropicModels(apiKey: string): Promise { - const response = await fetch('https://api.anthropic.com/v1/models', { - headers: { - 'X-Api-Key': apiKey, - 'anthropic-version': '2023-06-01', - }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch models from Anthropic`); - } - - const data = (await response.json()) as { data: AnthropicModel[] }; - - return data.data.map((model) => ({ - name: model.display_name, - value: model.id, - })); -} - -interface OpenAIModel { - id: string; -} - -async function getOpenAIModels(apiKey: string): Promise { - const response = await fetch('https://api.openai.com/v1/models', { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch models from OpenAI`); - } - const data = (await response.json()) as { data: OpenAIModel[] }; - return data.data.map((model) => ({ - name: model.id, - value: model.id, - })); -} - -interface GoogleModel { - name: string; - displayName: string; - baseModelId: string; - inputTokenLimit: number; - outputTokenLimit: number; -} -async function getGoogleModels(apiKey: string): Promise { - const response = await fetch('https://generativelanguage.googleapis.com/v1beta/models', { - headers: { - 'x-goog-api-key': apiKey, - }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch models from Google`); - } - const data = (await response.json()) as { models: GoogleModel[] }; - return data.models.map((model) => ({ - name: model.displayName, - value: model.name, - })); -} - -interface XAIModel { - id: string; -} -async function getXAIModels(apiKey: string): Promise { - const response = await fetch('https://api.x.ai/v1/models', { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch models from XAI`); - } - const data = (await response.json()) as { data: XAIModel[] }; - return data.data.map((model) => ({ - name: model.id, - value: model.id, - })); +async function getProviderModels(provider: LLMProvider): Promise { + return PROVIDER_MODELS[provider]; } From 4e3ff11be4745fae3ac1a07e4d8603db37bad359 Mon Sep 17 00:00:00 2001 From: eneaxharau Date: Tue, 30 Dec 2025 13:11:49 +0100 Subject: [PATCH 5/5] feat: add benchmarking commands and dataset generation functionality --- .gitignore | 2 + benchmarks/benchmark-dataset-opus-4.5.csv | 19 + cli/commands/benchmark-command.ts | 271 ++++++++++++ cli/commands/generate-dataset-command.ts | 199 +++++++++ cli/index.ts | 39 ++ cli/utils/shared.utils.ts | 6 +- package.json | 8 +- src/agents/execution/agent-internal-graph.ts | 4 +- src/benchmark/benchmark-reporter.ts | 357 ++++++++++++++++ src/benchmark/benchmark-runner.ts | 410 +++++++++++++++++++ src/benchmark/dataset-loader.ts | 240 +++++++++++ src/benchmark/index.ts | 9 + src/benchmark/metrics-calculator.ts | 329 +++++++++++++++ src/benchmark/types.ts | 153 +++++++ src/constants/provider-models.ts | 24 +- src/llm/llm-service.ts | 4 +- src/services/metrics-calculation.service.ts | 6 +- 17 files changed, 2063 insertions(+), 17 deletions(-) create mode 100644 benchmarks/benchmark-dataset-opus-4.5.csv create mode 100644 cli/commands/benchmark-command.ts create mode 100644 cli/commands/generate-dataset-command.ts create mode 100644 src/benchmark/benchmark-reporter.ts create mode 100644 src/benchmark/benchmark-runner.ts create mode 100644 src/benchmark/dataset-loader.ts create mode 100644 src/benchmark/index.ts create mode 100644 src/benchmark/metrics-calculator.ts create mode 100644 src/benchmark/types.ts diff --git a/.gitignore b/.gitignore index 7515e4e..de04584 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ index.html .archdoc.config.json .idea + +benchmarks/runs/*.json diff --git a/benchmarks/benchmark-dataset-opus-4.5.csv b/benchmarks/benchmark-dataset-opus-4.5.csv new file mode 100644 index 0000000..6cc3cca --- /dev/null +++ b/benchmarks/benchmark-dataset-opus-4.5.csv @@ -0,0 +1,19 @@ +commit_hash,repo_path,functionalImpact,idealTimeHours,testCoverage,codeQuality,codeComplexity,actualTimeHours,technicalDebtHours,debtReductionHours,notes +e8201970,/Users/eneaxharau/Documents/TechDebtGPT,5.6,6.92,1.7,5,5.2,6.09,4.54,0.74,"Generated by claude-opus-4-5-20251101" +903d6c83,/Users/eneaxharau/Documents/TechDebtGPT,5.3,2.19,2,5.7,2.3,1.44,1.2,0.24,"Generated by claude-opus-4-5-20251101" +13732457,/Users/eneaxharau/Documents/TechDebtGPT,1.4,0.36,1.9,5.1,1.8,0.31,1,0.26,"Generated by claude-opus-4-5-20251101" +883311c7,/Users/eneaxharau/Documents/TechDebtGPT,4.7,3.37,2,5,3.8,3.14,4.09,2.5,"Generated by claude-opus-4-5-20251101" +059bd334,/Users/eneaxharau/Documents/TechDebtGPT,6.4,2.1,1.8,3.6,3.6,1.14,3.59,0.22,"Generated by claude-opus-4-5-20251101" +389915e3,/Users/eneaxharau/Documents/TechDebtGPT,3.5,3.79,1.9,5,4.9,4.41,4.13,2.26,"Generated by claude-opus-4-5-20251101" +4dd230bb,/Users/eneaxharau/Documents/TechDebtGPT,3.8,0.61,1.7,5,3.4,0.68,1.27,0.14,"Generated by claude-opus-4-5-20251101" +c5154cd3,/Users/eneaxharau/Documents/TechDebtGPT,5.6,6,1.2,4.3,5.9,8,4.43,0.84,"Generated by claude-opus-4-5-20251101" +f48a3f22,/Users/eneaxharau/Documents/TechDebtGPT,6.1,7.33,1.2,5.1,4.1,4,3.09,0.35,"Generated by claude-opus-4-5-20251101" +6a07893a,/Users/eneaxharau/Documents/TechDebtGPT,2.8,0.41,2.5,6.1,1.2,0.48,0.27,0.06,"Generated by claude-opus-4-5-20251101" +2a91b7cf,/Users/eneaxharau/Documents/TechDebtGPT,1.5,0.45,1.6,4.4,2.2,0.28,1.22,0,"Generated by claude-opus-4-5-20251101" +69f4988e,/Users/eneaxharau/Documents/TechDebtGPT,5.1,9.41,2,4.7,5.4,8.95,6.55,0,"Generated by claude-opus-4-5-20251101" +a71d319d,/Users/eneaxharau/Documents/TechDebtGPT,3.3,2.4,1.8,4.9,4.7,1.36,2.5,1.82,"Generated by claude-opus-4-5-20251101" +62cea0f6,/Users/eneaxharau/Documents/TechDebtGPT,5.4,1.52,1.9,4.7,3.5,1.04,1.65,0,"Generated by claude-opus-4-5-20251101" +b97b457f,/Users/eneaxharau/Documents/TechDebtGPT,5.6,3.79,1.7,4.5,4.2,2.95,2.33,0,"Generated by claude-opus-4-5-20251101" +a277f897,/Users/eneaxharau/Documents/TechDebtGPT,1.1,0.81,1.8,4.5,3.8,0.91,1.78,0.07,"Generated by claude-opus-4-5-20251101" +3216c050,/Users/eneaxharau/Documents/TechDebtGPT,5.7,2.21,1.2,5.4,4.3,1.11,1.93,0.2,"Generated by claude-opus-4-5-20251101" +bd6a1199,/Users/eneaxharau/Documents/TechDebtGPT,3.5,1.79,1,4.1,3.3,1.66,2.33,0.22,"Generated by claude-opus-4-5-20251101" \ No newline at end of file diff --git a/cli/commands/benchmark-command.ts b/cli/commands/benchmark-command.ts new file mode 100644 index 0000000..ee76064 --- /dev/null +++ b/cli/commands/benchmark-command.ts @@ -0,0 +1,271 @@ +// cli/commands/benchmark-command.ts +// CLI command for running benchmarks and comparing model performance + +import chalk from 'chalk'; +import * as fs from 'fs'; +import * as path from 'path'; +import { + runBenchmark, + listBenchmarkRuns, + loadBenchmarkRuns, + getBenchmarkRunsDir, +} from '../../src/benchmark/benchmark-runner'; +import { + printBenchmarkResult, + printBenchmarkList, + printModelComparison, + generateComparisonJSON, +} from '../../src/benchmark/benchmark-reporter'; +import { BenchmarkOptions, CompareOptions } from '../../src/benchmark/types'; + +/** + * Parse CLI arguments for benchmark command + */ +function parseArgs(args: string[]): { + subcommand: 'run' | 'compare' | 'list'; + options: any; +} { + // Check for subcommands + if (args[0] === 'compare') { + return { + subcommand: 'compare', + options: parseCompareArgs(args.slice(1)), + }; + } + + if (args[0] === 'list') { + return { + subcommand: 'list', + options: {}, + }; + } + + // Default: run benchmark + return { + subcommand: 'run', + options: parseRunArgs(args), + }; +} + +/** + * Parse arguments for benchmark run + */ +function parseRunArgs(args: string[]): BenchmarkOptions { + const options: BenchmarkOptions = { + datasetPath: '', + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--dataset' || arg === '-d') { + options.datasetPath = args[++i]; + } else if (arg === '--name' || arg === '-n') { + options.name = args[++i]; + } else if (arg === '--output' || arg === '-o') { + options.outputPath = args[++i]; + } else if (arg === '--concurrency' || arg === '-c') { + const raw = args[++i]; + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed >= 1) { + options.concurrency = parsed; + } + } else if (arg === '--depth') { + const depth = args[++i]; + if (['fast', 'normal', 'deep'].includes(depth)) { + options.depthMode = depth as 'fast' | 'normal' | 'deep'; + } + } else if (arg === '--silent' || arg === '-s') { + options.silent = true; + } + } + + return options; +} + +/** + * Parse arguments for benchmark compare + */ +function parseCompareArgs(args: string[]): CompareOptions { + const options: CompareOptions = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--runs' || arg === '-r') { + options.runNames = args[++i].split(',').map((s) => s.trim()); + } else if (arg === '--all' || arg === '-a') { + options.all = true; + } + } + + return options; +} + +/** + * Print usage help + */ +function printUsage(): void { + console.log(chalk.cyan('\n📊 Codewave Benchmark Tool\n')); + console.log(chalk.white('Usage:')); + console.log( + chalk.gray(' codewave benchmark --dataset Run benchmark against dataset') + ); + console.log( + chalk.gray(' codewave benchmark --dataset --name Run with custom name') + ); + console.log(chalk.gray(' codewave benchmark compare --runs a,b,c Compare specific runs')); + console.log(chalk.gray(' codewave benchmark compare --all Compare all saved runs')); + console.log(chalk.gray(' codewave benchmark list List all saved runs')); + console.log(''); + console.log(chalk.white('Options:')); + console.log(chalk.gray(' --dataset, -d Path to ground truth CSV dataset')); + console.log(chalk.gray(' --name, -n Custom name for this benchmark run')); + console.log(chalk.gray(' --output, -o Path to save JSON results')); + console.log( + chalk.gray(' --concurrency, -c Number of commits to evaluate in parallel (default: 1)') + ); + console.log(chalk.gray(' --depth Analysis depth: fast, normal, deep')); + console.log(chalk.gray(' --silent, -s Suppress progress output')); + console.log(''); + console.log(chalk.white('Compare Options:')); + console.log(chalk.gray(' --runs, -r Comma-separated run names to compare')); + console.log(chalk.gray(' --all, -a Compare all saved benchmark runs')); + console.log(''); + console.log(chalk.white('Examples:')); + console.log(chalk.gray(' codewave benchmark --dataset ./ground-truth.csv')); + console.log(chalk.gray(' codewave benchmark --dataset ./data.csv --name "claude-baseline"')); + console.log(chalk.gray(' codewave benchmark --dataset ./data.csv --concurrency 4')); + console.log(chalk.gray(' codewave benchmark compare --runs claude-baseline,gpt4-test')); + console.log(chalk.gray(' codewave benchmark list')); + console.log(''); +} + +/** + * Run benchmark command + */ +async function runBenchmarkCommand(options: BenchmarkOptions): Promise { + if (!options.datasetPath) { + console.log(chalk.red('\n❌ Error: --dataset is required\n')); + printUsage(); + process.exit(1); + } + + // Resolve dataset path + const datasetPath = path.resolve(options.datasetPath); + if (!fs.existsSync(datasetPath)) { + console.log(chalk.red(`\n❌ Error: Dataset file not found: ${datasetPath}\n`)); + process.exit(1); + } + + options.datasetPath = datasetPath; + + console.log(chalk.cyan('\n🚀 Starting benchmark run...\n')); + + try { + const result = await runBenchmark(options, (message) => { + if (!options.silent) { + console.log(message); + } + }); + + // Print results + printBenchmarkResult(result); + + // Save JSON output if requested + if (options.outputPath) { + const outputPath = path.resolve(options.outputPath); + fs.writeFileSync(outputPath, JSON.stringify(result, null, 2)); + console.log(chalk.green(`\n💾 Results saved to: ${outputPath}\n`)); + } + } catch (error) { + console.log( + chalk.red( + `\n❌ Benchmark failed: ${error instanceof Error ? error.message : String(error)}\n` + ) + ); + process.exit(1); + } +} + +/** + * Run compare command + */ +async function runCompareCommand(options: CompareOptions): Promise { + let runNames: string[]; + + if (options.all) { + // Load all runs + const runs = listBenchmarkRuns(); + if (runs.length < 2) { + console.log(chalk.yellow('\n⚠️ Need at least 2 benchmark runs to compare.\n')); + console.log( + chalk.gray(' Run: codewave benchmark --dataset to create benchmark runs.\n') + ); + process.exit(1); + } + runNames = runs.map((r) => r.name); + } else if (options.runNames && options.runNames.length > 0) { + runNames = options.runNames; + } else { + console.log(chalk.red('\n❌ Error: Specify --runs or --all for comparison\n')); + printUsage(); + process.exit(1); + } + + console.log(chalk.cyan(`\n🔍 Loading ${runNames.length} benchmark runs for comparison...\n`)); + + const results = loadBenchmarkRuns(runNames); + + if (results.length < 2) { + console.log(chalk.red('\n❌ Error: Could not load enough benchmark runs\n')); + console.log( + chalk.gray( + ' Make sure the run names are correct. Use "codewave benchmark list" to see available runs.\n' + ) + ); + process.exit(1); + } + + // Print comparison + printModelComparison(results); + + // Also output JSON comparison to file + const comparison = generateComparisonJSON(results); + const comparisonPath = path.join(getBenchmarkRunsDir(), `comparison-${Date.now()}.json`); + fs.writeFileSync(comparisonPath, JSON.stringify(comparison, null, 2)); + console.log(chalk.gray(` 📄 Comparison JSON saved to: ${comparisonPath}\n`)); +} + +/** + * Run list command + */ +async function runListCommand(): Promise { + const runs = listBenchmarkRuns(); + printBenchmarkList(runs); +} + +/** + * Main entry point for benchmark command + */ +export async function runBenchmarkCommandHandler(args: string[]): Promise { + // Handle help + if (args.includes('--help') || args.includes('-h') || args.length === 0) { + printUsage(); + return; + } + + const { subcommand, options } = parseArgs(args); + + switch (subcommand) { + case 'run': + await runBenchmarkCommand(options); + break; + case 'compare': + await runCompareCommand(options); + break; + case 'list': + await runListCommand(); + break; + } +} diff --git a/cli/commands/generate-dataset-command.ts b/cli/commands/generate-dataset-command.ts new file mode 100644 index 0000000..4f9e283 --- /dev/null +++ b/cli/commands/generate-dataset-command.ts @@ -0,0 +1,199 @@ +// cli/commands/generate-dataset-command.ts +// CLI command for generating benchmark datasets from commits + +import chalk from 'chalk'; +import * as fs from 'fs'; +import * as path from 'path'; +import { generateDataset } from '../../src/benchmark/benchmark-runner'; +import { printDatasetComplete } from '../../src/benchmark/benchmark-reporter'; +import { GenerateDatasetOptions } from '../../src/benchmark/types'; + +/** + * Parse CLI arguments for generate-dataset command + */ +function parseArgs(args: string[]): GenerateDatasetOptions & { commitsFile?: string } { + const options: GenerateDatasetOptions & { commitsFile?: string } = { + commits: [], + repoPath: '.', + outputPath: './benchmark-dataset.csv', + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--commits' || arg === '-c') { + options.commits = args[++i].split(',').map((s) => s.trim()); + } else if (arg === '--commits-file' || arg === '-f') { + options.commitsFile = args[++i]; + } else if (arg === '--repo' || arg === '-r') { + options.repoPath = args[++i]; + } else if (arg === '--output' || arg === '-o') { + options.outputPath = args[++i]; + } else if (arg === '--concurrency') { + const raw = args[++i]; + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed >= 1) { + options.concurrency = parsed; + } + } else if (arg === '--depth') { + const depth = args[++i]; + if (['fast', 'normal', 'deep'].includes(depth)) { + options.depthMode = depth as 'fast' | 'normal' | 'deep'; + } + } + } + + return options; +} + +/** + * Load commits from a file (one commit hash per line) + */ +function loadCommitsFromFile(filePath: string): string[] { + const absolutePath = path.resolve(filePath); + + if (!fs.existsSync(absolutePath)) { + throw new Error(`Commits file not found: ${absolutePath}`); + } + + const content = fs.readFileSync(absolutePath, 'utf-8'); + const commits = content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith('#')); // Ignore empty lines and comments + + return commits; +} + +/** + * Print usage help + */ +function printUsage(): void { + console.log(chalk.cyan('\n📝 Generate Benchmark Dataset\n')); + console.log(chalk.white('Description:')); + console.log(chalk.gray(' Evaluate commits and generate a CSV file with model predictions.')); + console.log(chalk.gray(' Review and adjust values to create ground truth for benchmarking.\n')); + console.log(chalk.white('Usage:')); + console.log( + chalk.gray(' codewave generate-dataset --commits --output ') + ); + console.log(chalk.gray(' codewave generate-dataset --commits-file --output ')); + console.log(''); + console.log(chalk.white('Options:')); + console.log(chalk.gray(' --commits, -c Comma-separated commit hashes to evaluate')); + console.log(chalk.gray(' --commits-file, -f File with commit hashes (one per line)')); + console.log( + chalk.gray(' --repo, -r Repository path (default: current directory)') + ); + console.log( + chalk.gray(' --output, -o Output CSV path (default: ./benchmark-dataset.csv)') + ); + console.log( + chalk.gray(' --concurrency Number of commits to evaluate in parallel (default: 1)') + ); + console.log(chalk.gray(' --depth Analysis depth: fast, normal, deep')); + console.log(''); + console.log(chalk.white('Examples:')); + console.log( + chalk.gray(' codewave generate-dataset --commits abc1234,def5678 --output ./data.csv') + ); + console.log( + chalk.gray(' codewave generate-dataset --commits-file ./commits.txt --repo /path/to/repo') + ); + console.log( + chalk.gray(' codewave generate-dataset --commits-file ./commits.txt --concurrency 4') + ); + console.log(''); + console.log(chalk.white('Workflow:')); + console.log(chalk.gray(' 1. Run generate-dataset to create CSV with model predictions')); + console.log(chalk.gray(' 2. Open CSV in spreadsheet, review and adjust values')); + console.log(chalk.gray(' 3. Save as ground truth dataset')); + console.log(chalk.gray(' 4. Run: codewave benchmark --dataset ')); + console.log(''); +} + +/** + * Main entry point for generate-dataset command + */ +export async function runGenerateDatasetCommand(args: string[]): Promise { + // Handle help + if (args.includes('--help') || args.includes('-h') || args.length === 0) { + printUsage(); + return; + } + + const options = parseArgs(args); + + // Load commits from file if specified + if (options.commitsFile) { + try { + const fileCommits = loadCommitsFromFile(options.commitsFile); + options.commits = [...options.commits, ...fileCommits]; + } catch (error) { + console.log( + chalk.red( + `\n❌ Error loading commits file: ${error instanceof Error ? error.message : String(error)}\n` + ) + ); + process.exit(1); + } + } + + // Validate commits + if (options.commits.length === 0) { + console.log(chalk.red('\n❌ Error: No commits specified\n')); + console.log(chalk.gray(' Use --commits or --commits-file to specify commits to evaluate.\n')); + printUsage(); + process.exit(1); + } + + // Resolve paths + options.repoPath = path.resolve(options.repoPath); + options.outputPath = path.resolve(options.outputPath); + + // Validate repo path + if (!fs.existsSync(options.repoPath)) { + console.log(chalk.red(`\n❌ Error: Repository path not found: ${options.repoPath}\n`)); + process.exit(1); + } + + // Check if .git exists + const gitPath = path.join(options.repoPath, '.git'); + if (!fs.existsSync(gitPath)) { + console.log(chalk.red(`\n❌ Error: Not a git repository: ${options.repoPath}\n`)); + process.exit(1); + } + + console.log(chalk.cyan('\n🚀 Starting dataset generation...\n')); + console.log(chalk.white(` 📁 Repository: ${options.repoPath}`)); + console.log(chalk.white(` 📊 Commits: ${options.commits.length}`)); + console.log(chalk.white(` 📄 Output: ${options.outputPath}`)); + if (options.depthMode) { + console.log(chalk.white(` 🎯 Depth: ${options.depthMode}`)); + } + console.log(''); + + try { + await generateDataset( + { + commits: options.commits, + repoPath: options.repoPath, + outputPath: options.outputPath, + depthMode: options.depthMode, + concurrency: options.concurrency, + }, + (message) => { + console.log(message); + } + ); + + printDatasetComplete(options.outputPath, options.commits.length); + } catch (error) { + console.log( + chalk.red( + `\n❌ Dataset generation failed: ${error instanceof Error ? error.message : String(error)}\n` + ) + ); + process.exit(1); + } +} diff --git a/cli/index.ts b/cli/index.ts index f2297fe..e906521 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -21,6 +21,8 @@ import { runEvaluateCommand } from './commands/evaluate-command'; import { runConfigCommand } from './commands/config.command'; import { runBatchEvaluateCommand } from './commands/batch-evaluate-command'; import { runGenerateOkrCommand } from './commands/generate-okr-command'; +import { runBenchmarkCommandHandler } from './commands/benchmark-command'; +import { runGenerateDatasetCommand } from './commands/generate-dataset-command'; async function main() { const [, , command, ...args] = process.argv; @@ -59,6 +61,12 @@ async function main() { case 'config': await runConfigCommand(args); break; + case 'benchmark': + await runBenchmarkCommandHandler(args); + break; + case 'generate-dataset': + await runGenerateDatasetCommand(args); + break; default: printUsage(); process.exit(1); @@ -90,6 +98,12 @@ function printUsage() { console.log( ' generate-okr [options] Generate OKRs and action points from history' ); + console.log( + ' benchmark [options] Run benchmark against labeled dataset' + ); + console.log( + ' generate-dataset [options] Generate dataset CSV for manual labeling' + ); console.log(''); console.log('Evaluate Options:'); console.log(' Evaluate a specific commit (default)'); @@ -110,6 +124,21 @@ function printUsage() { console.log(' --depth Analysis depth: fast, normal, deep (default: normal)'); console.log(' --no-stream Disable streaming output (silent mode)'); console.log(''); + console.log('Benchmark Options:'); + console.log(' --dataset Path to ground truth CSV dataset'); + console.log(' --name Custom name for this benchmark run'); + console.log(' --output Path to save JSON results'); + console.log(' --depth Analysis depth: fast, normal, deep'); + console.log(' compare --runs Compare comma-separated benchmark runs'); + console.log(' compare --all Compare all saved benchmark runs'); + console.log(' list List all saved benchmark runs'); + console.log(''); + console.log('Generate Dataset Options:'); + console.log(' --commits Comma-separated commit hashes to evaluate'); + console.log(' --commits-file File with commit hashes (one per line)'); + console.log(' --repo Repository path (default: current directory)'); + console.log(' --output Output CSV path (default: ./benchmark-dataset.csv)'); + console.log(''); console.log('Examples:'); console.log(' # Setup configuration'); console.log(' codewave config --init'); @@ -132,6 +161,16 @@ function printUsage() { console.log(' # Batch evaluate date range with fast mode'); console.log(' codewave batch --since "2024-01-01" --until "2024-01-31" --depth fast'); console.log(''); + console.log(' # Generate dataset from commits for manual labeling'); + console.log(' codewave generate-dataset --commits abc123,def456 --output ./data.csv'); + console.log(''); + console.log(' # Run benchmark against ground truth dataset'); + console.log(' codewave benchmark --dataset ./ground-truth.csv --name "claude-baseline"'); + console.log(''); + console.log(' # Compare benchmark runs'); + console.log(' codewave benchmark compare --runs claude-baseline,gpt4-test'); + console.log(' codewave benchmark list'); + console.log(''); console.log('📖 Docs: https://github.com/techdebtgpt/codewave'); } diff --git a/cli/utils/shared.utils.ts b/cli/utils/shared.utils.ts index 4a0a035..3125665 100644 --- a/cli/utils/shared.utils.ts +++ b/cli/utils/shared.utils.ts @@ -213,8 +213,10 @@ function extractMetricsSnapshot(agentResults: AgentResult[]): MetricsSnapshot { if (contributors.length > 0) { const weightedValue = calculateWeightedAverage(contributors, metricName); - // Determine decimal places based on metric - if (metricName.includes('Hours') || metricName.includes('Time')) { + // Handle null when all agents returned null for this metric + if (weightedValue === null) { + result[metricName] = 0; // Default to 0 when all agents returned null + } else if (metricName.includes('Hours') || metricName.includes('Time')) { result[metricName] = Number(weightedValue.toFixed(2)); } else { result[metricName] = Number(weightedValue.toFixed(1)); diff --git a/package.json b/package.json index 4b6d739..3d1818c 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,11 @@ "prepublishOnly": "npm run validate:publish", "prepack": "npm run build", "postpack": "echo '✅ Package created successfully'", - "batch": "node ./dist/cli/index.js batch --count 3" + "batch": "node ./dist/cli/index.js batch --count 3", + "benchmark": "node ./dist/cli/index.js benchmark", + "benchmark:list": "node ./dist/cli/index.js benchmark list", + "benchmark:compare": "node ./dist/cli/index.js benchmark compare --all", + "generate-dataset": "node ./dist/cli/index.js generate-dataset" }, "bin": { "codewave": "./dist/cli/index.js" @@ -109,4 +113,4 @@ "p-limit": "^5.0.0", "table": "^6.9.0" } -} +} \ No newline at end of file diff --git a/src/agents/execution/agent-internal-graph.ts b/src/agents/execution/agent-internal-graph.ts index bb26f2e..2b8e883 100644 --- a/src/agents/execution/agent-internal-graph.ts +++ b/src/agents/execution/agent-internal-graph.ts @@ -177,10 +177,12 @@ async function refineAnalysis( const response = (await model.invoke(messages)) as any; const tokenUsage = extractTokenUsage(response); + // Return only the NEW messages - the reducer will append them to state.messages + // Previously this was [...state.messages, user, assistant] which caused duplication + // because the reducer does [...oldState, ...update] return { currentAnalysis: response, messages: [ - ...state.messages, { role: 'user', content: refinementPrompt }, { role: 'assistant', content: response.content }, ] as any, diff --git a/src/benchmark/benchmark-reporter.ts b/src/benchmark/benchmark-reporter.ts new file mode 100644 index 0000000..53bdb80 --- /dev/null +++ b/src/benchmark/benchmark-reporter.ts @@ -0,0 +1,357 @@ +// src/benchmark/benchmark-reporter.ts +// Console table and JSON report generation for benchmarks + +import chalk from 'chalk'; +import { table } from 'table'; +import { BenchmarkResult, BenchmarkRunInfo, BENCHMARK_METRICS } from './types'; +import { compareRuns } from './metrics-calculator'; + +/** + * Format a number with specified decimal places + */ +function formatNum(value: number | undefined, decimals: number = 2): string { + if (value === undefined || isNaN(value)) return 'N/A'; + return value.toFixed(decimals); +} + +/** + * Format percentage + */ +function formatPct(value: number | undefined): string { + if (value === undefined || isNaN(value)) return 'N/A'; + return `${value.toFixed(1)}%`; +} + +/** + * Print a single benchmark result to console + */ +export function printBenchmarkResult(result: BenchmarkResult): void { + console.log('\n'); + console.log(chalk.bold.cyan('═'.repeat(80))); + console.log(chalk.bold.cyan(' Codewave Benchmark Results')); + console.log(chalk.bold.cyan('═'.repeat(80))); + console.log(''); + + // Metadata + console.log( + chalk.white(` Dataset: ${chalk.bold(result.datasetPath)} (${result.commitCount} commits)`) + ); + console.log(chalk.white(` Model: ${chalk.bold(result.model)} | Provider: ${result.provider}`)); + console.log(chalk.white(` Depth: ${result.depthMode} | Run: ${result.timestamp}`)); + console.log(chalk.white(` Name: ${chalk.bold(result.name)}`)); + console.log(''); + + // Header explanation + console.log(chalk.gray(' ↓ = Lower is better (less error)')); + console.log(chalk.gray(' ↑ = Higher is better (more accuracy)')); + console.log(chalk.gray(' Dir% = Direction Accuracy (only for +/- debt metrics)')); + console.log(''); + + // Build metrics table + const tableData: string[][] = [ + [ + chalk.bold('Metric'), + chalk.bold('MAE ↓'), + chalk.bold('RMSE ↓'), + chalk.bold('NMAE ↓'), + chalk.bold('R² ↑'), + chalk.bold('Max ↓'), + chalk.bold('Dir% ↑'), + chalk.bold('N'), + ], + ]; + + for (const metric of BENCHMARK_METRICS) { + const m = result.metrics[metric]; + const row = [ + metric, + formatNum(m.mae), + formatNum(m.rmse), + formatPct(m.nmae), + formatNum(m.r2), + formatNum(m.maxError), + m.directionAccuracy !== undefined ? formatPct(m.directionAccuracy) : '-', + m.sampleCount.toString(), + ]; + tableData.push(row); + } + + // Table config + const tableConfig = { + border: { + topBody: '─', + topJoin: '┬', + topLeft: '┌', + topRight: '┐', + bottomBody: '─', + bottomJoin: '┴', + bottomLeft: '└', + bottomRight: '┘', + bodyLeft: '│', + bodyRight: '│', + bodyJoin: '│', + joinBody: '─', + joinLeft: '├', + joinRight: '┤', + joinJoin: '┼', + }, + columns: { + 0: { alignment: 'left' as const, width: 20 }, + 1: { alignment: 'right' as const, width: 8 }, + 2: { alignment: 'right' as const, width: 8 }, + 3: { alignment: 'right' as const, width: 8 }, + 4: { alignment: 'right' as const, width: 8 }, + 5: { alignment: 'right' as const, width: 8 }, + 6: { alignment: 'right' as const, width: 8 }, + 7: { alignment: 'right' as const, width: 5 }, + }, + }; + + console.log(table(tableData, tableConfig)); + + // Summary + console.log(chalk.bold.cyan('─'.repeat(80))); + console.log( + chalk.white( + ` Overall NMAE: ${chalk.bold(formatPct(result.overallNmae))} (↓ lower better) | ` + + `Avg R²: ${chalk.bold(formatNum(result.overallR2))} (↑ higher better)` + ) + ); + console.log(chalk.bold.cyan('─'.repeat(80))); + + // Performance stats + console.log(''); + console.log( + chalk.gray(` ⏱️ Total time: ${(result.totalEvaluationTime / 1000 / 60).toFixed(1)} min`) + ); + console.log( + chalk.gray(` ⏱️ Avg per commit: ${(result.averageEvaluationTime / 1000).toFixed(1)}s`) + ); + if (result.totalTokens) { + console.log(chalk.gray(` 🎟️ Total tokens: ${result.totalTokens.toLocaleString()}`)); + } + console.log(''); +} + +/** + * Print benchmark runs list to console + */ +export function printBenchmarkList(runs: BenchmarkRunInfo[]): void { + if (runs.length === 0) { + console.log(chalk.yellow('\n No benchmark runs found.')); + console.log(chalk.gray(' Run: codewave benchmark --dataset to create one.\n')); + return; + } + + console.log('\n'); + console.log(chalk.bold.cyan('═'.repeat(100))); + console.log(chalk.bold.cyan(' Saved Benchmark Runs')); + console.log(chalk.bold.cyan('═'.repeat(100))); + console.log(''); + + const tableData: string[][] = [ + [ + chalk.bold('Name'), + chalk.bold('Model'), + chalk.bold('Commits'), + chalk.bold('NMAE ↓'), + chalk.bold('R² ↑'), + chalk.bold('Timestamp'), + ], + ]; + + for (const run of runs) { + tableData.push([ + run.name.length > 35 ? run.name.substring(0, 32) + '...' : run.name, + run.model.length > 25 ? run.model.substring(0, 22) + '...' : run.model, + run.commitCount.toString(), + formatPct(run.overallNmae), + formatNum(run.overallR2), + new Date(run.timestamp).toLocaleString(), + ]); + } + + const tableConfig = { + border: { + topBody: '─', + topJoin: '┬', + topLeft: '┌', + topRight: '┐', + bottomBody: '─', + bottomJoin: '┴', + bottomLeft: '└', + bottomRight: '┘', + bodyLeft: '│', + bodyRight: '│', + bodyJoin: '│', + joinBody: '─', + joinLeft: '├', + joinRight: '┤', + joinJoin: '┼', + }, + columns: { + 0: { alignment: 'left' as const, width: 35 }, + 1: { alignment: 'left' as const, width: 25 }, + 2: { alignment: 'right' as const, width: 8 }, + 3: { alignment: 'right' as const, width: 8 }, + 4: { alignment: 'right' as const, width: 8 }, + 5: { alignment: 'left' as const, width: 20 }, + }, + }; + + console.log(table(tableData, tableConfig)); + console.log(chalk.gray(` Total: ${runs.length} benchmark runs\n`)); +} + +/** + * Print model comparison to console + */ +export function printModelComparison(results: BenchmarkResult[]): void { + if (results.length < 2) { + console.log(chalk.yellow('\n Need at least 2 benchmark runs to compare.\n')); + return; + } + + console.log('\n'); + console.log(chalk.bold.cyan('═'.repeat(100))); + console.log(chalk.bold.cyan(' Model Comparison Results')); + console.log(chalk.bold.cyan('═'.repeat(100))); + console.log(''); + + // Runs being compared + console.log(chalk.white(' Comparing:')); + for (const r of results) { + console.log(chalk.gray(` • ${r.name} (${r.model})`)); + } + console.log(''); + + // Calculate comparison + const runsData = results.map((r) => ({ name: r.name, metrics: r.metrics })); + const { comparisons, rankings, overallBest } = compareRuns(runsData); + + // Header explanation + console.log(chalk.gray(' ↓ = Lower is better | ↑ = Higher is better')); + console.log(chalk.gray(' 🏆 = Best performer for that metric')); + console.log(''); + + // Per-metric comparison + console.log(chalk.bold.white(' Per-Metric Comparison (NMAE - lower is better):')); + console.log(''); + + for (const comp of comparisons) { + const metricLabel = comp.metric.padEnd(20); + const runValues = comp.runs + .map((r) => { + const isBest = r.name === comp.bestRun; + const value = formatPct(r.nmae); + return isBest ? chalk.green(`${r.name}: ${value} 🏆`) : chalk.white(`${r.name}: ${value}`); + }) + .join(' | '); + + console.log(` ${chalk.cyan(metricLabel)} ${runValues}`); + } + + console.log(''); + console.log(chalk.bold.cyan('─'.repeat(100))); + console.log(''); + + // Overall rankings + console.log(chalk.bold.white(' Overall Rankings (by average NMAE):')); + console.log(''); + + const rankTableData: string[][] = [ + [chalk.bold('Rank'), chalk.bold('Run Name'), chalk.bold('Avg NMAE ↓'), chalk.bold('Avg R² ↑')], + ]; + + for (const r of rankings) { + const rankEmoji = + r.rank === 1 ? '🥇' : r.rank === 2 ? '🥈' : r.rank === 3 ? '🥉' : `#${r.rank}`; + const isWinner = r.rank === 1; + + rankTableData.push([ + rankEmoji, + isWinner ? chalk.green.bold(r.name) : r.name, + isWinner ? chalk.green.bold(formatPct(r.avgNmae)) : formatPct(r.avgNmae), + isWinner ? chalk.green.bold(formatNum(r.avgR2)) : formatNum(r.avgR2), + ]); + } + + const rankTableConfig = { + border: { + topBody: '─', + topJoin: '┬', + topLeft: '┌', + topRight: '┐', + bottomBody: '─', + bottomJoin: '┴', + bottomLeft: '└', + bottomRight: '┘', + bodyLeft: '│', + bodyRight: '│', + bodyJoin: '│', + joinBody: '─', + joinLeft: '├', + joinRight: '┤', + joinJoin: '┼', + }, + columns: { + 0: { alignment: 'center' as const, width: 6 }, + 1: { alignment: 'left' as const, width: 40 }, + 2: { alignment: 'right' as const, width: 12 }, + 3: { alignment: 'right' as const, width: 12 }, + }, + }; + + console.log(table(rankTableData, rankTableConfig)); + + console.log(chalk.bold.green(` 🏆 Overall Best: ${overallBest}`)); + console.log(''); +} + +/** + * Generate JSON comparison report + */ +export function generateComparisonJSON(results: BenchmarkResult[]): object { + const runsData = results.map((r) => ({ name: r.name, metrics: r.metrics })); + const { comparisons, rankings, overallBest } = compareRuns(runsData); + + return { + timestamp: new Date().toISOString(), + runs: results.map((r) => ({ + name: r.name, + model: r.model, + provider: r.provider, + depthMode: r.depthMode, + commitCount: r.commitCount, + overallNmae: r.overallNmae, + overallR2: r.overallR2, + })), + comparisons: comparisons.map((c) => ({ + metric: c.metric, + bestRun: c.bestRun, + runs: c.runs, + })), + rankings, + overallBest, + }; +} + +/** + * Print dataset generation completion + */ +export function printDatasetComplete(outputPath: string, count: number): void { + console.log(''); + console.log(chalk.bold.green('═'.repeat(60))); + console.log(chalk.bold.green(' Dataset Generation Complete')); + console.log(chalk.bold.green('═'.repeat(60))); + console.log(''); + console.log(chalk.white(` 📄 Output: ${chalk.bold(outputPath)}`)); + console.log(chalk.white(` 📊 Commits: ${count}`)); + console.log(''); + console.log(chalk.yellow(' Next steps:')); + console.log(chalk.gray(' 1. Open the CSV in a spreadsheet')); + console.log(chalk.gray(' 2. Review and adjust the values based on your judgment')); + console.log(chalk.gray(' 3. Save as your ground truth dataset')); + console.log(chalk.gray(' 4. Run: codewave benchmark --dataset ')); + console.log(''); +} diff --git a/src/benchmark/benchmark-runner.ts b/src/benchmark/benchmark-runner.ts new file mode 100644 index 0000000..35841cf --- /dev/null +++ b/src/benchmark/benchmark-runner.ts @@ -0,0 +1,410 @@ +// src/benchmark/benchmark-runner.ts +// Core benchmark execution logic + +import * as fs from 'fs'; +import * as path from 'path'; +import { CommitEvaluationOrchestrator } from '../orchestrator/commit-evaluation-orchestrator'; +import { loadConfig } from '../config/config-loader'; +import { createAgentRegistry } from '../../cli/utils/shared.utils'; +import { getCommitDiff } from '../../cli/utils/git-utils'; +import { MetricsCalculationService } from '../services/metrics-calculation.service'; +import { loadDataset, saveDatasetCSV } from './dataset-loader'; +import { calculateAllMetrics } from './metrics-calculator'; +import { + DatasetEntry, + PredictionEntry, + BenchmarkResult, + BenchmarkOptions, + GenerateDatasetOptions, + BenchmarkRunInfo, +} from './types'; + +function normalizeConcurrency(value: unknown, fallback: number): number { + if (typeof value !== 'number' || !Number.isFinite(value)) return fallback; + const n = Math.floor(value); + return n >= 1 ? n : fallback; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise { + const safeConcurrency = Math.max(1, Math.floor(concurrency)); + const results = new Array(items.length); + let nextIndex = 0; + + const runWorker = async () => { + while (nextIndex < items.length) { + const index = nextIndex++; + if (index >= items.length) { + return; + } + results[index] = await worker(items[index], index); + } + }; + + const workerCount = Math.min(safeConcurrency, items.length); + await Promise.all(Array.from({ length: workerCount }, () => runWorker())); + return results; +} + +/** + * Get the benchmark runs directory + */ +export function getBenchmarkRunsDir(): string { + return path.join(process.cwd(), 'benchmarks', 'runs'); +} + +/** + * Ensure benchmark directories exist + */ +function ensureBenchmarkDirs(): void { + const runsDir = getBenchmarkRunsDir(); + if (!fs.existsSync(runsDir)) { + fs.mkdirSync(runsDir, { recursive: true }); + } +} + +/** + * Generate a unique run name if not provided + */ +function generateRunName(config: any): string { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); + const model = config.llm.model.replace(/[^a-zA-Z0-9]/g, '-'); + return `${model}_${timestamp}`; +} + +/** + * Run a single commit evaluation and extract metrics + */ +async function evaluateCommit( + orchestrator: CommitEvaluationOrchestrator, + entry: DatasetEntry, + config: any, + onProgress?: (message: string) => void +): Promise { + const startTime = Date.now(); + + try { + // Get the diff for the commit + const diff = getCommitDiff(entry.commitHash, entry.repoPath); + + if (!diff || diff.trim().length === 0) { + onProgress?.(`⚠️ No diff found for commit ${entry.commitHash}`); + return null; + } + + const context = { + commitDiff: diff, + filesChanged: [], + commitHash: entry.commitHash, + config, + }; + + // Run evaluation with streaming disabled for benchmark + const result = await orchestrator.evaluateCommit(context, { + streaming: false, + disableTracing: true, + threadId: `benchmark-${entry.commitHash}-${Date.now()}`, + }); + + const evaluationTime = Date.now() - startTime; + + // Calculate weighted metrics from agent results + const agentResults = result.agentResults || []; + const metrics = MetricsCalculationService.calculateWeightedMetrics(agentResults); + + // Calculate total token usage + let totalInputTokens = 0; + let totalOutputTokens = 0; + for (const agent of agentResults) { + if (agent.tokenUsage) { + totalInputTokens += agent.tokenUsage.inputTokens || 0; + totalOutputTokens += agent.tokenUsage.outputTokens || 0; + } + } + + const prediction: PredictionEntry = { + commitHash: entry.commitHash, + repoPath: entry.repoPath, + functionalImpact: metrics.functionalImpact ?? null, + idealTimeHours: metrics.idealTimeHours ?? null, + testCoverage: metrics.testCoverage ?? null, + codeQuality: metrics.codeQuality ?? null, + codeComplexity: metrics.codeComplexity ?? null, + actualTimeHours: metrics.actualTimeHours ?? null, + technicalDebtHours: metrics.technicalDebtHours ?? null, + debtReductionHours: metrics.debtReductionHours ?? null, + evaluationTime, + tokenUsage: { + inputTokens: totalInputTokens, + outputTokens: totalOutputTokens, + totalTokens: totalInputTokens + totalOutputTokens, + }, + }; + + return prediction; + } catch (error) { + onProgress?.( + `❌ Error evaluating commit ${entry.commitHash}: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +} + +/** + * Run benchmark against a dataset + */ +export async function runBenchmark( + options: BenchmarkOptions, + onProgress?: (message: string) => void +): Promise { + ensureBenchmarkDirs(); + + // Load config + const config = loadConfig(); + if (!config) { + throw new Error('No configuration found. Run: codewave config --init'); + } + + // Apply depth mode + if (options.depthMode) { + config.agents.depthMode = options.depthMode; + } + + // Load dataset + onProgress?.(`📂 Loading dataset: ${options.datasetPath}`); + const { entries, errors, warnings } = loadDataset(options.datasetPath); + + if (errors.length > 0) { + throw new Error(`Dataset errors:\n${errors.join('\n')}`); + } + + if (warnings.length > 0) { + warnings.forEach((w) => onProgress?.(`⚠️ ${w}`)); + } + + onProgress?.(`✅ Loaded ${entries.length} commits from dataset`); + + // Create orchestrator + const agentRegistry = createAgentRegistry(config); + const orchestrator = new CommitEvaluationOrchestrator(agentRegistry, config); + + // Run evaluations + const concurrency = normalizeConcurrency(options.concurrency, 1); + if (concurrency > 1) { + onProgress?.(`⚡ Running commit evaluations in parallel (concurrency=${concurrency})`); + } + const startTime = Date.now(); + + const predictionResults = await mapWithConcurrency(entries, concurrency, async (entry, i) => { + onProgress?.(`\n[${i + 1}/${entries.length}] Evaluating commit ${entry.commitHash}...`); + const prediction = await evaluateCommit(orchestrator, entry, config, onProgress); + if (prediction) { + onProgress?.( + `✅ [${i + 1}/${entries.length}] Completed in ${(prediction.evaluationTime / 1000).toFixed(1)}s` + ); + } else { + onProgress?.(`⚠️ [${i + 1}/${entries.length}] Failed`); + } + return prediction; + }); + + const predictions = predictionResults.filter((p): p is PredictionEntry => p !== null); + + const totalEvaluationTime = Date.now() - startTime; + + // Calculate metrics + onProgress?.(`\n📊 Calculating benchmark metrics...`); + const { metrics, overallNmae, overallR2, commitErrors } = calculateAllMetrics( + entries, + predictions + ); + + // Calculate total tokens + const totalTokens = predictions.reduce((sum, p) => sum + (p.tokenUsage?.totalTokens || 0), 0); + + // Build result + const runName = options.name || generateRunName(config); + const result: BenchmarkResult = { + name: runName, + timestamp: new Date().toISOString(), + datasetPath: options.datasetPath, + commitCount: entries.length, + model: config.llm.model, + provider: config.llm.provider, + depthMode: config.agents.depthMode || 'normal', + metrics, + overallNmae, + overallR2, + predictions, + commitErrors, + totalEvaluationTime, + averageEvaluationTime: predictions.length > 0 ? totalEvaluationTime / predictions.length : 0, + totalTokens, + }; + + // Save result + const resultPath = path.join(getBenchmarkRunsDir(), `${runName}.json`); + fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); + onProgress?.(`\n💾 Saved benchmark result to: ${resultPath}`); + + return result; +} + +/** + * Generate a dataset by evaluating commits + */ +export async function generateDataset( + options: GenerateDatasetOptions, + onProgress?: (message: string) => void +): Promise { + // Load config + const config = loadConfig(); + if (!config) { + throw new Error('No configuration found. Run: codewave config --init'); + } + + // Apply depth mode + if (options.depthMode) { + config.agents.depthMode = options.depthMode; + } + + onProgress?.(`🚀 Generating dataset for ${options.commits.length} commits...`); + onProgress?.(`📁 Repository: ${options.repoPath}`); + onProgress?.(`🎯 Output: ${options.outputPath}`); + + // Create orchestrator + const agentRegistry = createAgentRegistry(config); + const orchestrator = new CommitEvaluationOrchestrator(agentRegistry, config); + + // Evaluate each commit + const concurrency = normalizeConcurrency(options.concurrency, 1); + if (concurrency > 1) { + onProgress?.(`⚡ Running commit evaluations in parallel (concurrency=${concurrency})`); + } + + const entries = new Array(options.commits.length); + await mapWithConcurrency(options.commits, concurrency, async (commitHash, i) => { + onProgress?.(`\n[${i + 1}/${options.commits.length}] Evaluating commit ${commitHash}...`); + + const baseEntry: DatasetEntry = { + commitHash, + repoPath: options.repoPath, + functionalImpact: null, + idealTimeHours: null, + testCoverage: null, + codeQuality: null, + codeComplexity: null, + actualTimeHours: null, + technicalDebtHours: null, + debtReductionHours: null, + notes: '', + }; + + const prediction = await evaluateCommit(orchestrator, baseEntry, config, onProgress); + + if (prediction) { + entries[i] = { + commitHash, + repoPath: options.repoPath, + functionalImpact: prediction.functionalImpact, + idealTimeHours: prediction.idealTimeHours, + testCoverage: prediction.testCoverage, + codeQuality: prediction.codeQuality, + codeComplexity: prediction.codeComplexity, + actualTimeHours: prediction.actualTimeHours, + technicalDebtHours: prediction.technicalDebtHours, + debtReductionHours: prediction.debtReductionHours, + notes: `Generated by ${config.llm.model}`, + }; + onProgress?.(`✅ [${i + 1}/${options.commits.length}] Completed`); + } else { + entries[i] = { + ...baseEntry, + notes: 'EVALUATION_FAILED - please fill manually', + }; + onProgress?.(`⚠️ [${i + 1}/${options.commits.length}] Failed - added placeholder entry`); + } + }); + + // Save to CSV + saveDatasetCSV(entries, options.outputPath); + onProgress?.(`\n✅ Dataset saved to: ${options.outputPath}`); + onProgress?.(`📝 Review and adjust values, then use as ground truth for benchmarking.`); +} + +/** + * List all benchmark runs + */ +export function listBenchmarkRuns(): BenchmarkRunInfo[] { + const runsDir = getBenchmarkRunsDir(); + + if (!fs.existsSync(runsDir)) { + return []; + } + + const files = fs.readdirSync(runsDir).filter((f) => f.endsWith('.json')); + const runs: BenchmarkRunInfo[] = []; + + for (const file of files) { + try { + const content = fs.readFileSync(path.join(runsDir, file), 'utf-8'); + const result: BenchmarkResult = JSON.parse(content); + + runs.push({ + name: result.name, + timestamp: result.timestamp, + model: result.model, + provider: result.provider, + commitCount: result.commitCount, + overallNmae: result.overallNmae, + overallR2: result.overallR2, + filePath: path.join(runsDir, file), + }); + } catch { + // Skip invalid files + } + } + + // Sort by timestamp (newest first) + runs.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()); + + return runs; +} + +/** + * Load a benchmark result by name + */ +export function loadBenchmarkRun(name: string): BenchmarkResult | null { + const runsDir = getBenchmarkRunsDir(); + const filePath = path.join(runsDir, `${name}.json`); + + if (!fs.existsSync(filePath)) { + return null; + } + + try { + const content = fs.readFileSync(filePath, 'utf-8'); + return JSON.parse(content); + } catch { + return null; + } +} + +/** + * Load multiple benchmark runs for comparison + */ +export function loadBenchmarkRuns(names: string[]): BenchmarkResult[] { + const results: BenchmarkResult[] = []; + + for (const name of names) { + const result = loadBenchmarkRun(name); + if (result) { + results.push(result); + } + } + + return results; +} diff --git a/src/benchmark/dataset-loader.ts b/src/benchmark/dataset-loader.ts new file mode 100644 index 0000000..9143caa --- /dev/null +++ b/src/benchmark/dataset-loader.ts @@ -0,0 +1,240 @@ +// src/benchmark/dataset-loader.ts +// CSV dataset parsing and validation for benchmark ground truth + +import * as fs from 'fs'; +import * as path from 'path'; +import { DatasetEntry, BenchmarkMetricName } from './types'; + +/** + * CSV column names mapping to DatasetEntry fields + */ +const CSV_COLUMNS = [ + 'commit_hash', + 'repo_path', + 'functionalImpact', + 'idealTimeHours', + 'testCoverage', + 'codeQuality', + 'codeComplexity', + 'actualTimeHours', + 'technicalDebtHours', + 'debtReductionHours', + 'notes', +] as const; + +const REQUIRED_COLUMNS = [ + 'commit_hash', + 'repo_path', + 'functionalImpact', + 'idealTimeHours', + 'testCoverage', + 'codeQuality', + 'codeComplexity', + 'actualTimeHours', + 'technicalDebtHours', + 'debtReductionHours', +]; + +/** + * Parse a CSV value, handling nulls and empty strings + */ +function parseNumericValue(value: string): number | null { + const trimmed = value.trim(); + if (trimmed === '' || trimmed.toLowerCase() === 'null' || trimmed === '-') { + return null; + } + const num = parseFloat(trimmed); + if (isNaN(num)) { + return null; + } + return num; +} + +/** + * Parse a single CSV line, handling quoted values + */ +function parseCSVLine(line: string): string[] { + const values: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + + if (char === '"') { + if (inQuotes && line[i + 1] === '"') { + // Escaped quote + current += '"'; + i++; + } else { + // Toggle quote mode + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + values.push(current.trim()); + current = ''; + } else { + current += char; + } + } + values.push(current.trim()); + + return values; +} + +/** + * Validate a dataset entry + */ +function validateEntry(entry: DatasetEntry, lineNumber: number): string[] { + const errors: string[] = []; + + if (!entry.commitHash || entry.commitHash.trim() === '') { + errors.push(`Line ${lineNumber}: commit_hash is required`); + } + + if (!entry.repoPath || entry.repoPath.trim() === '') { + errors.push(`Line ${lineNumber}: repo_path is required`); + } + + // Validate numeric ranges for score-based metrics (1-10) + const scoreMetrics: BenchmarkMetricName[] = [ + 'functionalImpact', + 'testCoverage', + 'codeQuality', + 'codeComplexity', + ]; + + for (const metric of scoreMetrics) { + const value = entry[metric]; + if (value !== null && (value < 1 || value > 10)) { + errors.push(`Line ${lineNumber}: ${metric} must be between 1 and 10 (got ${value})`); + } + } + + // Validate time-based metrics (must be non-negative for most) + const timeMetrics: BenchmarkMetricName[] = ['idealTimeHours', 'actualTimeHours']; + for (const metric of timeMetrics) { + const value = entry[metric]; + if (value !== null && value < 0) { + errors.push(`Line ${lineNumber}: ${metric} must be non-negative (got ${value})`); + } + } + + return errors; +} + +/** + * Load and parse a CSV dataset file + */ +export function loadDataset(filePath: string): { + entries: DatasetEntry[]; + errors: string[]; + warnings: string[]; +} { + const absolutePath = path.resolve(filePath); + const errors: string[] = []; + const warnings: string[] = []; + const entries: DatasetEntry[] = []; + + // Check file exists + if (!fs.existsSync(absolutePath)) { + errors.push(`Dataset file not found: ${absolutePath}`); + return { entries, errors, warnings }; + } + + // Read and parse file + const content = fs.readFileSync(absolutePath, 'utf-8'); + const lines = content.split('\n').filter((line) => line.trim() !== ''); + + if (lines.length < 2) { + errors.push('Dataset file must have at least a header row and one data row'); + return { entries, errors, warnings }; + } + + // Parse header + const header = parseCSVLine(lines[0]).map((h) => h.toLowerCase().replace(/\s+/g, '_')); + + // Validate required columns + for (const required of REQUIRED_COLUMNS) { + const normalizedRequired = required.toLowerCase(); + if (!header.some((h) => h === normalizedRequired)) { + errors.push(`Missing required column: ${required}`); + } + } + + if (errors.length > 0) { + return { entries, errors, warnings }; + } + + // Create column index map + const columnIndex: Record = {}; + header.forEach((col, idx) => { + columnIndex[col] = idx; + }); + + // Parse data rows + for (let i = 1; i < lines.length; i++) { + const lineNumber = i + 1; + const values = parseCSVLine(lines[i]); + + if (values.length < REQUIRED_COLUMNS.length) { + warnings.push(`Line ${lineNumber}: Incomplete row, skipping`); + continue; + } + + const entry: DatasetEntry = { + commitHash: values[columnIndex['commit_hash']] || '', + repoPath: values[columnIndex['repo_path']] || '', + functionalImpact: parseNumericValue(values[columnIndex['functionalimpact']] || ''), + idealTimeHours: parseNumericValue(values[columnIndex['idealtimehours']] || ''), + testCoverage: parseNumericValue(values[columnIndex['testcoverage']] || ''), + codeQuality: parseNumericValue(values[columnIndex['codequality']] || ''), + codeComplexity: parseNumericValue(values[columnIndex['codecomplexity']] || ''), + actualTimeHours: parseNumericValue(values[columnIndex['actualtimehours']] || ''), + technicalDebtHours: parseNumericValue(values[columnIndex['technicaldebthours']] || ''), + debtReductionHours: parseNumericValue(values[columnIndex['debtreductionhours']] || ''), + notes: values[columnIndex['notes']] || undefined, + }; + + // Validate entry + const entryErrors = validateEntry(entry, lineNumber); + if (entryErrors.length > 0) { + errors.push(...entryErrors); + } else { + entries.push(entry); + } + } + + if (entries.length === 0 && errors.length === 0) { + errors.push('No valid entries found in dataset'); + } + + return { entries, errors, warnings }; +} + +/** + * Save predictions to CSV format (for generate-dataset command) + */ +export function saveDatasetCSV(entries: DatasetEntry[], outputPath: string): void { + const header = CSV_COLUMNS.join(','); + + const rows = entries.map((entry) => { + const values = [ + entry.commitHash, + entry.repoPath, + entry.functionalImpact?.toString() ?? '', + entry.idealTimeHours?.toString() ?? '', + entry.testCoverage?.toString() ?? '', + entry.codeQuality?.toString() ?? '', + entry.codeComplexity?.toString() ?? '', + entry.actualTimeHours?.toString() ?? '', + entry.technicalDebtHours?.toString() ?? '', + entry.debtReductionHours?.toString() ?? '', + entry.notes ? `"${entry.notes.replace(/"/g, '""')}"` : '', + ]; + return values.join(','); + }); + + const content = [header, ...rows].join('\n'); + fs.writeFileSync(outputPath, content, 'utf-8'); +} diff --git a/src/benchmark/index.ts b/src/benchmark/index.ts new file mode 100644 index 0000000..f316296 --- /dev/null +++ b/src/benchmark/index.ts @@ -0,0 +1,9 @@ +// src/benchmark/index.ts +// Benchmark module exports + +export * from './types'; +export * from './dataset-loader'; +export * from './metrics-calculator'; +export * from './benchmark-runner'; +export * from './benchmark-reporter'; + diff --git a/src/benchmark/metrics-calculator.ts b/src/benchmark/metrics-calculator.ts new file mode 100644 index 0000000..d64c736 --- /dev/null +++ b/src/benchmark/metrics-calculator.ts @@ -0,0 +1,329 @@ +// src/benchmark/metrics-calculator.ts +// Statistical metrics computation for benchmark evaluation + +import { + DatasetEntry, + PredictionEntry, + PillarMetrics, + CommitError, + BenchmarkMetricName, + BENCHMARK_METRICS, +} from './types'; + +/** + * Metrics that can have positive or negative values (for direction accuracy) + */ +const SIGNED_METRICS: BenchmarkMetricName[] = ['technicalDebtHours', 'debtReductionHours']; + +/** + * Calculate Mean Absolute Error + * Lower is better + */ +function calculateMAE(actual: number[], predicted: number[]): number { + if (actual.length === 0) return 0; + const sum = actual.reduce((acc, val, idx) => acc + Math.abs(val - predicted[idx]), 0); + return sum / actual.length; +} + +/** + * Calculate Root Mean Squared Error + * Lower is better - penalizes large errors more than MAE + */ +function calculateRMSE(actual: number[], predicted: number[]): number { + if (actual.length === 0) return 0; + const sumSquares = actual.reduce((acc, val, idx) => acc + Math.pow(val - predicted[idx], 2), 0); + return Math.sqrt(sumSquares / actual.length); +} + +/** + * Calculate Normalized Mean Absolute Error + * Lower is better - expressed as percentage of the data range + */ +function calculateNMAE(actual: number[], predicted: number[]): number { + if (actual.length === 0) return 0; + const mae = calculateMAE(actual, predicted); + const range = Math.max(...actual) - Math.min(...actual); + if (range === 0) return 0; // All values are the same + return (mae / range) * 100; +} + +/** + * Calculate R-squared (coefficient of determination) + * Higher is better - 1.0 means perfect prediction + */ +function calculateR2(actual: number[], predicted: number[]): number { + if (actual.length === 0) return 0; + + const mean = actual.reduce((a, b) => a + b, 0) / actual.length; + + // Total sum of squares + const ssTot = actual.reduce((acc, val) => acc + Math.pow(val - mean, 2), 0); + + // Residual sum of squares + const ssRes = actual.reduce((acc, val, idx) => acc + Math.pow(val - predicted[idx], 2), 0); + + if (ssTot === 0) return 1; // Perfect prediction when all values are the same + return 1 - ssRes / ssTot; +} + +/** + * Calculate Maximum Absolute Error + * Lower is better - worst-case deviation + */ +function calculateMaxError(actual: number[], predicted: number[]): number { + if (actual.length === 0) return 0; + let maxError = 0; + for (let i = 0; i < actual.length; i++) { + const error = Math.abs(actual[i] - predicted[i]); + if (error > maxError) maxError = error; + } + return maxError; +} + +/** + * Calculate Direction Accuracy + * Higher is better - percentage of predictions with correct sign + * Only applicable for metrics that can be positive or negative + */ +function calculateDirectionAccuracy(actual: number[], predicted: number[]): number { + if (actual.length === 0) return 0; + + let correct = 0; + for (let i = 0; i < actual.length; i++) { + // Check if signs match (or both are zero) + const actualSign = Math.sign(actual[i]); + const predictedSign = Math.sign(predicted[i]); + if (actualSign === predictedSign) { + correct++; + } + } + + return (correct / actual.length) * 100; +} + +/** + * Calculate all metrics for a single pillar + */ +export function calculatePillarMetrics( + actual: (number | null)[], + predicted: (number | null)[], + metricName: BenchmarkMetricName +): PillarMetrics { + // Filter out pairs where either value is null + const validPairs: { actual: number; predicted: number }[] = []; + for (let i = 0; i < actual.length; i++) { + if (actual[i] !== null && predicted[i] !== null) { + validPairs.push({ actual: actual[i]!, predicted: predicted[i]! }); + } + } + + const actualValues = validPairs.map((p) => p.actual); + const predictedValues = validPairs.map((p) => p.predicted); + + const metrics: PillarMetrics = { + mae: calculateMAE(actualValues, predictedValues), + rmse: calculateRMSE(actualValues, predictedValues), + nmae: calculateNMAE(actualValues, predictedValues), + r2: calculateR2(actualValues, predictedValues), + maxError: calculateMaxError(actualValues, predictedValues), + sampleCount: validPairs.length, + }; + + // Add direction accuracy for signed metrics + if (SIGNED_METRICS.includes(metricName)) { + metrics.directionAccuracy = calculateDirectionAccuracy(actualValues, predictedValues); + } + + return metrics; +} + +/** + * Calculate per-commit errors + */ +export function calculateCommitErrors( + groundTruth: DatasetEntry[], + predictions: PredictionEntry[] +): CommitError[] { + const errors: CommitError[] = []; + + // Create a map of predictions by commit hash + const predictionMap = new Map(); + for (const pred of predictions) { + predictionMap.set(pred.commitHash, pred); + } + + for (const truth of groundTruth) { + const prediction = predictionMap.get(truth.commitHash); + if (!prediction) continue; + + const commitError: CommitError = { + commitHash: truth.commitHash, + errors: {} as Record, + absoluteErrors: {} as Record, + }; + + for (const metric of BENCHMARK_METRICS) { + const actual = truth[metric]; + const predicted = prediction[metric]; + + if (actual !== null && predicted !== null) { + commitError.errors[metric] = predicted - actual; + commitError.absoluteErrors[metric] = Math.abs(predicted - actual); + } else { + commitError.errors[metric] = null; + commitError.absoluteErrors[metric] = null; + } + } + + errors.push(commitError); + } + + return errors; +} + +/** + * Calculate all benchmark metrics across all pillars + */ +export function calculateAllMetrics( + groundTruth: DatasetEntry[], + predictions: PredictionEntry[] +): { + metrics: Record; + overallNmae: number; + overallR2: number; + commitErrors: CommitError[]; +} { + // Create a map of predictions by commit hash for matching + const predictionMap = new Map(); + for (const pred of predictions) { + predictionMap.set(pred.commitHash, pred); + } + + // Extract paired values for each metric + const metrics: Record = {}; + let totalNmae = 0; + let totalR2 = 0; + let validMetricCount = 0; + + for (const metric of BENCHMARK_METRICS) { + const actual: (number | null)[] = []; + const predicted: (number | null)[] = []; + + for (const truth of groundTruth) { + const prediction = predictionMap.get(truth.commitHash); + if (prediction) { + actual.push(truth[metric]); + predicted.push(prediction[metric]); + } + } + + const pillarMetrics = calculatePillarMetrics(actual, predicted, metric); + metrics[metric] = pillarMetrics; + + if (pillarMetrics.sampleCount > 0) { + totalNmae += pillarMetrics.nmae; + totalR2 += pillarMetrics.r2; + validMetricCount++; + } + } + + const commitErrors = calculateCommitErrors(groundTruth, predictions); + + return { + metrics: metrics as Record, + overallNmae: validMetricCount > 0 ? totalNmae / validMetricCount : 0, + overallR2: validMetricCount > 0 ? totalR2 / validMetricCount : 0, + commitErrors, + }; +} + +/** + * Compare multiple benchmark runs and rank them + */ +export function compareRuns( + runs: { name: string; metrics: Record }[] +): { + comparisons: { + metric: BenchmarkMetricName; + runs: { + name: string; + mae: number; + rmse: number; + nmae: number; + r2: number; + maxError: number; + directionAccuracy?: number; + }[]; + bestRun: string; + }[]; + rankings: { name: string; avgNmae: number; avgR2: number; rank: number }[]; + overallBest: string; +} { + const comparisons: { + metric: BenchmarkMetricName; + runs: { + name: string; + mae: number; + rmse: number; + nmae: number; + r2: number; + maxError: number; + directionAccuracy?: number; + }[]; + bestRun: string; + }[] = []; + + // Compare each metric + for (const metric of BENCHMARK_METRICS) { + const metricRuns = runs.map((run) => ({ + name: run.name, + mae: run.metrics[metric].mae, + rmse: run.metrics[metric].rmse, + nmae: run.metrics[metric].nmae, + r2: run.metrics[metric].r2, + maxError: run.metrics[metric].maxError, + directionAccuracy: run.metrics[metric].directionAccuracy, + })); + + // Best run is the one with lowest NMAE (normalized error) + const bestRun = metricRuns.reduce((best, current) => + current.nmae < best.nmae ? current : best + ).name; + + comparisons.push({ + metric, + runs: metricRuns, + bestRun, + }); + } + + // Calculate overall rankings + const rankings = runs + .map((run) => { + let totalNmae = 0; + let totalR2 = 0; + let count = 0; + + for (const metric of BENCHMARK_METRICS) { + if (run.metrics[metric].sampleCount > 0) { + totalNmae += run.metrics[metric].nmae; + totalR2 += run.metrics[metric].r2; + count++; + } + } + + return { + name: run.name, + avgNmae: count > 0 ? totalNmae / count : Infinity, + avgR2: count > 0 ? totalR2 / count : 0, + rank: 0, + }; + }) + .sort((a, b) => a.avgNmae - b.avgNmae) + .map((r, idx) => ({ ...r, rank: idx + 1 })); + + const overallBest = rankings.length > 0 ? rankings[0].name : ''; + + return { comparisons, rankings, overallBest }; +} + diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts new file mode 100644 index 0000000..5872262 --- /dev/null +++ b/src/benchmark/types.ts @@ -0,0 +1,153 @@ +// src/benchmark/types.ts +// Type definitions for the benchmark system + +/** + * The 8 pillar metrics that are benchmarked + */ +export const BENCHMARK_METRICS = [ + 'functionalImpact', + 'idealTimeHours', + 'testCoverage', + 'codeQuality', + 'codeComplexity', + 'actualTimeHours', + 'technicalDebtHours', + 'debtReductionHours', +] as const; + +export type BenchmarkMetricName = (typeof BENCHMARK_METRICS)[number]; + +/** + * A single row in the ground truth dataset + */ +export interface DatasetEntry { + commitHash: string; + repoPath: string; + functionalImpact: number | null; + idealTimeHours: number | null; + testCoverage: number | null; + codeQuality: number | null; + codeComplexity: number | null; + actualTimeHours: number | null; + technicalDebtHours: number | null; + debtReductionHours: number | null; + notes?: string; +} + +/** + * Prediction result for a single commit + */ +export interface PredictionEntry { + commitHash: string; + repoPath: string; + functionalImpact: number | null; + idealTimeHours: number | null; + testCoverage: number | null; + codeQuality: number | null; + codeComplexity: number | null; + actualTimeHours: number | null; + technicalDebtHours: number | null; + debtReductionHours: number | null; + evaluationTime: number; // Time taken to evaluate in ms + tokenUsage?: { + inputTokens: number; + outputTokens: number; + totalTokens: number; + }; +} + +/** + * Error metrics for a single pillar + */ +export interface PillarMetrics { + mae: number; // Mean Absolute Error (lower is better) + rmse: number; // Root Mean Squared Error (lower is better) + nmae: number; // Normalized MAE as percentage (lower is better) + r2: number; // R-squared / coefficient of determination (higher is better) + maxError: number; // Maximum absolute error (lower is better) + directionAccuracy?: number; // % correct direction for +/- metrics (higher is better) + sampleCount: number; // Number of non-null comparisons +} + +/** + * Per-commit error details + */ +export interface CommitError { + commitHash: string; + errors: Record; // Prediction - Actual + absoluteErrors: Record; +} + +/** + * Complete benchmark results for a single run + */ +export interface BenchmarkResult { + name: string; // Run name (e.g., "claude-sonnet-baseline") + timestamp: string; // ISO timestamp + datasetPath: string; // Path to ground truth CSV + commitCount: number; // Number of commits evaluated + model: string; // Model used (e.g., "claude-sonnet-4-20250514") + provider: string; // Provider (e.g., "anthropic") + depthMode: 'fast' | 'normal' | 'deep'; + + // Aggregate metrics per pillar + metrics: Record; + + // Overall summary metrics + overallNmae: number; // Average NMAE across all pillars + overallR2: number; // Average R² across all pillars + + // Per-commit details + predictions: PredictionEntry[]; + commitErrors: CommitError[]; + + // Performance stats + totalEvaluationTime: number; // Total time in ms + averageEvaluationTime: number; // Average time per commit in ms + totalTokens?: number; +} + +/** + * Options for running a benchmark + */ +export interface BenchmarkOptions { + datasetPath: string; + name?: string; // Optional run name + outputPath?: string; // Optional JSON output path + depthMode?: 'fast' | 'normal' | 'deep'; + silent?: boolean; // Suppress progress output + concurrency?: number; // Number of commits to evaluate in parallel (default: 1) +} + +/** + * Options for comparing benchmark runs + */ +export interface CompareOptions { + runNames?: string[]; // Specific runs to compare + all?: boolean; // Compare all runs in benchmarks/runs/ +} + +/** + * Options for generating a dataset + */ +export interface GenerateDatasetOptions { + commits: string[]; // Commit hashes to evaluate + repoPath: string; + outputPath: string; + depthMode?: 'fast' | 'normal' | 'deep'; + concurrency?: number; // Number of commits to evaluate in parallel (default: 1) +} + +/** + * Saved benchmark run metadata (for listing) + */ +export interface BenchmarkRunInfo { + name: string; + timestamp: string; + model: string; + provider: string; + commitCount: number; + overallNmae: number; + overallR2: number; + filePath: string; +} diff --git a/src/constants/provider-models.ts b/src/constants/provider-models.ts index 2831939..244fc3d 100644 --- a/src/constants/provider-models.ts +++ b/src/constants/provider-models.ts @@ -156,6 +156,22 @@ export const PROVIDER_MODELS = { }, ], openai: [ + { + name: 'gpt-5.2 - [High Intelligence, High Cost, Standard Speed, Reasoning]', + value: 'gpt-5.2', + pricing: { + input: '0.00000175', + output: '0.000014', + }, + }, + { + name: 'gpt-5.2-chat-latest - [Medium-High Intelligence, High Cost, Fast Speed, No Reasoning]', + value: 'gpt-5.2-chat-latest', + pricing: { + input: '0.00000175', + output: '0.000014', + }, + }, { name: 'gpt-4 - [Medium Intelligence, High Cost, Standard Speed]', value: 'gpt-4', @@ -172,14 +188,6 @@ export const PROVIDER_MODELS = { output: '0.0000015', }, }, - { - name: 'gpt-5.1-codex-mini - [Medium-High Intelligence, Medium Cost, Fast Speed]', - value: 'gpt-5.1-codex-mini', - pricing: { - input: '0.00000025', - output: '0.000002', - }, - }, { name: 'gpt-5.1 - [High Intelligence, High Cost, Standard Speed]', value: 'gpt-5.1', diff --git a/src/llm/llm-service.ts b/src/llm/llm-service.ts index f75d429..6e366c7 100644 --- a/src/llm/llm-service.ts +++ b/src/llm/llm-service.ts @@ -62,8 +62,8 @@ export class LLMService { case 'openai': return new ChatOpenAI({ - openAIApiKey: apiKey, - temperature, + apiKey, + // temperature, maxTokens, modelName: model, }); diff --git a/src/services/metrics-calculation.service.ts b/src/services/metrics-calculation.service.ts index 8c833b6..b58ce28 100644 --- a/src/services/metrics-calculation.service.ts +++ b/src/services/metrics-calculation.service.ts @@ -71,8 +71,10 @@ export class MetricsCalculationService { if (contributors.length > 0) { const weightedValue = calculateWeightedAverage(contributors, metricName); - // Determine decimal places based on metric - if (metricName.includes('Hours') || metricName.includes('Time')) { + // Handle null when all agents returned null for this metric + if (weightedValue === null) { + // Skip setting this metric - leave it undefined + } else if (metricName.includes('Hours') || metricName.includes('Time')) { averagedMetrics[metricName] = Number(weightedValue.toFixed(2)) as any; } else { averagedMetrics[metricName] = Number(weightedValue.toFixed(1)) as any;