diff --git a/src/llama_dyno/cli.py b/src/llama_dyno/cli.py index 32a92bf..ead627b 100644 --- a/src/llama_dyno/cli.py +++ b/src/llama_dyno/cli.py @@ -570,7 +570,7 @@ def submit( False, "--quick", help="Quick tune mode before submission" ), ): - """Submit results — creates a Gist with your benchmark.""" + """Submit results — opens a PR to the community repo, else a Gist, else saves locally.""" if not validate_model(model): console.print(f"[red]ERROR:[/] '{model}' is not a valid .gguf file.") raise typer.Exit(1) diff --git a/src/llama_dyno/submit.py b/src/llama_dyno/submit.py index e07ed47..cfaf289 100644 --- a/src/llama_dyno/submit.py +++ b/src/llama_dyno/submit.py @@ -1,13 +1,17 @@ -"""Submit benchmark results to GitHub Gist.""" +"""Submit benchmark results: PR to the community repo, else Gist, else local.""" from __future__ import annotations import os import subprocess +import tempfile from typing import Any from .report import format_json +# Community results repo that `dyno submit` opens PRs against. +COMMUNITY_REPO = "Lachytonner/llama-dyno-results" + def _gh_installed() -> bool: """Check if gh CLI is available.""" @@ -69,6 +73,69 @@ def _result_json_content(report: dict[str, Any]) -> str: return format_json(report) +def _run(cmd: list[str], cwd: str | None = None) -> subprocess.CompletedProcess: + """Run a subprocess with captured output and a generous timeout.""" + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=120) + + +def submit_via_pr(report: dict[str, Any]) -> str | None: + """Open a PR against the community results repo with this benchmark. + + Forks + clones COMMUNITY_REPO, adds the result JSON under + results//, pushes to the fork, and opens a PR via gh. + Returns the PR URL, or None if gh is unavailable or any step fails (the + caller then falls back to a Gist, then a local save). + """ + if not _gh_installed() or not _gh_auth_status(): + return None + + filename = _result_filename(report) + gpu = _gpu_key(report.get("hardware", {})) + short_hash = (report.get("model", {}).get("sha256") or "result")[:12] + branch = f"dyno-{gpu}-{short_hash}" + title = ( + f"Add result: {report.get('hardware', {}).get('gpu_name', 'GPU')}" + f" + {report.get('model', {}).get('name', 'model')}" + ) + content = _result_json_content(report) + + try: + with tempfile.TemporaryDirectory() as tmp: + if _run(["gh", "repo", "fork", COMMUNITY_REPO, "--clone"], cwd=tmp).returncode != 0: + return None + repo_dir = os.path.join(tmp, COMMUNITY_REPO.split("/")[-1]) + if not os.path.isdir(repo_dir): + return None + + if _run(["git", "checkout", "-b", branch], cwd=repo_dir).returncode != 0: + return None + + rel = f"results/{gpu}/{filename}" + dest = os.path.join(repo_dir, "results", gpu, filename) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "w") as f: + f.write(content) + + for step in (["git", "add", rel], ["git", "commit", "-m", title], + ["git", "push", "-u", "origin", branch]): + if _run(step, cwd=repo_dir).returncode != 0: + return None + + # Disambiguate the head ref for a cross-fork PR. + who = _run(["gh", "api", "user", "-q", ".login"]) + head = f"{who.stdout.strip()}:{branch}" if who.returncode == 0 and who.stdout.strip() else branch + + pr = _run([ + "gh", "pr", "create", "--repo", COMMUNITY_REPO, "--head", head, + "--title", title, "--body", "Automated benchmark submission via `dyno submit`.", + ], cwd=repo_dir) + if pr.returncode == 0 and pr.stdout.strip(): + return pr.stdout.strip() + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + return None + return None + + def submit_via_gist(report: dict[str, Any]) -> str | None: """Submit results as a GitHub Gist. @@ -109,16 +176,20 @@ def submit_via_gist(report: dict[str, Any]) -> str | None: def submit_report(report: dict[str, Any]) -> str: - """Submit report, trying Gist first then saving locally. + """Submit report: PR to the community repo, else Gist, else save locally. - Returns the Gist URL on success. - Raises RuntimeError if submission fails (with a message including the local path). + Returns the PR or Gist URL on success. + Raises RuntimeError if all remote paths fail (message includes the local path). """ + url = submit_via_pr(report) + if url: + return url + url = submit_via_gist(report) if url: return url - # Gist failed — save locally + # Both remote paths failed — save locally result_dir = os.path.join(os.getcwd(), "dyno-results") os.makedirs(result_dir, exist_ok=True) filename = _result_filename(report) @@ -127,8 +198,8 @@ def submit_report(report: dict[str, Any]) -> str: f.write(_result_json_content(report)) raise RuntimeError( - "Could not submit via Gist. " - "Install gh: gh auth login\n\n" + "Could not submit via PR or Gist (is gh installed and authenticated? " + "run 'gh auth login').\n\n" f"Result saved locally to: {path}\n" - "You can manually submit it at https://gist.github.com" + "You can open a PR manually at https://github.com/" + COMMUNITY_REPO ) diff --git a/src/llama_dyno/tune.py b/src/llama_dyno/tune.py index 807e812..81fd0aa 100644 --- a/src/llama_dyno/tune.py +++ b/src/llama_dyno/tune.py @@ -13,7 +13,7 @@ find_bench_binary, run_bench, ) -from .types import BenchParams, TrialResult, TuneResult +from .types import BenchParams, TrialResult, TuneResult, score_trial console = Console() @@ -37,18 +37,8 @@ def thorough(cls) -> TuneConfig: def _score_trial(t: TrialResult, pp_weight: float = 0.3, tg_weight: float = 0.7) -> float: - """Combined score weighting pp and tg throughput. - - Args: - t: Trial result. - pp_weight: Weight for prompt-processing throughput (default 0.3). - tg_weight: Weight for text-generation throughput (default 0.7). - """ - if t.oom or t.error: - return -1.0 - pp = t.pp_tokens_s or 0 - tg = t.tg_tokens_s or 0 - return pp * pp_weight + tg * tg_weight + """Score a trial via the canonical scorer in types.score_trial.""" + return score_trial(t, pp_weight, tg_weight) def build_progress_table(trials: list[TrialResult]) -> Table: @@ -406,6 +396,31 @@ def _hill_climb( return best_params +def _detect_vram_mib() -> int: + """Detect total GPU VRAM in MiB (pynvml, then nvidia-smi). 0 if unknown. + + A single monkeypatchable seam for the tuner's VRAM heuristic. + """ + try: + import pynvml + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + vram = pynvml.nvmlDeviceGetMemoryInfo(handle).total // (1024 * 1024) + pynvml.nvmlShutdown() + return int(vram) + except Exception: + pass + import subprocess as sp + try: + out = sp.run( + ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5, + ) + return int(out.stdout.strip()) if out.stdout.strip() else 0 + except Exception: + return 0 + + def _estimate_model_size(model_path: str) -> int: """Estimate model file size in MiB.""" import os @@ -475,28 +490,7 @@ def run_tune( console.print(" Model type: Dense (skipping MoE-specific tuning)") console.print() - # Detect GPU VRAM - vram_total = 0 - try: - import pynvml - pynvml.nvmlInit() - handle = pynvml.nvmlDeviceGetHandleByIndex(0) - vram_total = pynvml.nvmlDeviceGetMemoryInfo(handle).total // (1024 * 1024) - pynvml.nvmlShutdown() - except Exception: - pass - # Fallback nvidia-smi - if vram_total == 0: - import subprocess as sp - try: - out = sp.run( - ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], - capture_output=True, text=True, timeout=5, - ) - vram_total = int(out.stdout.strip()) if out.stdout.strip() else 0 - except Exception: - pass - + vram_total = _detect_vram_mib() model_size_mib = _estimate_model_size(model_path) console.print(f"[bold]Dyno Tuning[/] - mode: {mode}") diff --git a/src/llama_dyno/types.py b/src/llama_dyno/types.py index 49c9d92..5520d86 100644 --- a/src/llama_dyno/types.py +++ b/src/llama_dyno/types.py @@ -102,13 +102,24 @@ class TrialResult: @property def score(self) -> float: - """Combined score weighting both pp and tg throughput.""" - if self.oom or self.error: - return -1.0 - pp = self.pp_tokens_s or 0 - tg = self.tg_tokens_s or 0 - # Weight prompt processing less than generation (typically 1:1 or 1:2) - return pp * 0.3 + tg * 0.7 + """Combined score with default weights. + + Convenience wrapper; use score_trial() directly to pass custom weights. + """ + return score_trial(self) + + +def score_trial( + trial: TrialResult, pp_weight: float = 0.3, tg_weight: float = 0.7 +) -> float: + """Combined pp/tg throughput score. OOM/error trials score -1. + + Single source of truth for trial scoring — both TrialResult.score and the + tuner's scoring call through here so custom weights are always honored. + """ + if trial.oom or trial.error: + return -1.0 + return (trial.pp_tokens_s or 0) * pp_weight + (trial.tg_tokens_s or 0) * tg_weight @dataclass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a76977d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,62 @@ +"""Shared test fixtures and mocks for llama-dyno.""" + +from __future__ import annotations + +from llama_dyno.types import BenchParams, TrialResult + + +class MockBenchRunner: + """Simulates llama-bench runs so the search algorithm can be tested offline. + + Speeds vary by parameter so the tuner has a deterministic optimum to find: + OOM above ngl 60, flash attention helps, KV quant helps slightly, ngl scales + up to 50, batch peaks in 1024-2048, and thread=auto is best. + + The signature mirrors bench.run_bench so it can be monkeypatched in as a + drop-in replacement (positional or keyword args, plus binary/timeout/warmup). + """ + + def __init__(self): + self.calls: list[BenchParams] = [] + + def run( + self, + model_path: str, + params: BenchParams | None = None, + binary: str | None = None, + timeout: int = 300, + warmup: bool = True, + **kwargs, + ) -> TrialResult: + params = params or BenchParams() + self.calls.append(params) + + # Simulate OOM at very high ngl + if params.ngl > 60: + return TrialResult(params=params, oom=True, error="CUDA OOM (mocked)") + + fa_bonus = 1.2 if params.flash_attn else 1.0 + kv_factor = {"f16": 1.0, "q8_0": 1.05, "q4_0": 1.1}.get(params.ct_k, 1.0) + ngl_factor = min(1.0, params.ngl / 50) * 0.5 + 0.5 + + batch_penalty = 1.0 + if params.batch_size < 256: + batch_penalty = 0.7 + elif params.batch_size < 1024: + batch_penalty = 0.9 + elif params.batch_size > 2048: + batch_penalty = 0.85 + + thread_factor = 1.0 + if 0 < params.threads < 4: + thread_factor = 0.8 + + base_pp = 500.0 + base_tg = 30.0 + mult = ngl_factor * fa_bonus * kv_factor * batch_penalty * thread_factor + return TrialResult( + params=params, + pp_tokens_s=base_pp * mult, + tg_tokens_s=base_tg * mult, + oom=False, + ) diff --git a/tests/test_search.py b/tests/test_search.py index 72af47e..487b38e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -2,70 +2,11 @@ from __future__ import annotations -import json -from dataclasses import dataclass -from typing import Callable - import pytest -from llama_dyno.types import BenchParams, TrialResult, TuneResult, model_info_from_path - - -class MockBenchRunner: - """Mock that simulates bench runs for testing the search algorithm. - - The mock returns speeds that vary by parameters, allowing us to verify - the tuner picks the optimal config. - """ +from conftest import MockBenchRunner - def __init__(self): - self.calls: list[BenchParams] = [] - - def run(self, model_path: str, params: BenchParams) -> TrialResult: - self.calls.append(params) - - # Simulate OOM at very high ngl for "small VRAM" scenarios - if params.ngl > 60: - return TrialResult(params=params, oom=True, error="CUDA OOM (mocked)") - - # Simulate better performance with flash attention - fa_bonus = 1.2 if params.flash_attn else 1.0 - - # Simulate KV cache quant effects - kv_factor = {"f16": 1.0, "q8_0": 1.05, "q4_0": 1.1}.get(params.ct_k, 1.0) - - # Simulate ngl scaling (more layers = faster up to a point) - ngl_factor = min(1.0, params.ngl / 50) * 0.5 + 0.5 - - # Simulate batch size sweet spot at 1024 - batch_penalty = 1.0 - if params.batch_size < 256: - batch_penalty = 0.7 - elif params.batch_size < 1024: - batch_penalty = 0.9 # sub-optimal - elif params.batch_size > 2048: - batch_penalty = 0.85 - - # Simulate thread scaling - thread_factor = 1.0 - if params.threads == 0: - thread_factor = 1.0 # auto = good - elif params.threads < 4: - thread_factor = 0.8 - - # Base speed - base_pp = 500.0 - base_tg = 30.0 - - pp = base_pp * ngl_factor * fa_bonus * kv_factor * batch_penalty * thread_factor - tg = base_tg * ngl_factor * fa_bonus * kv_factor * batch_penalty * thread_factor - - return TrialResult( - params=params, - pp_tokens_s=pp, - tg_tokens_s=tg, - oom=False, - ) +from llama_dyno.types import BenchParams, TrialResult, model_info_from_path def test_bench_params_to_flag_list(): diff --git a/tests/test_submit.py b/tests/test_submit.py new file mode 100644 index 0000000..024a271 --- /dev/null +++ b/tests/test_submit.py @@ -0,0 +1,87 @@ +"""Tests for the submit fallback chain (PR → Gist → local), with gh/git mocked.""" + +from __future__ import annotations + +import os +import subprocess + +import pytest + +from llama_dyno import submit + + +@pytest.fixture +def report(): + return { + "hardware": {"gpu_name": "NVIDIA GeForce RTX 4070", "backend": "llama.cpp"}, + "model": {"name": "test-model.gguf", "sha256": "abc123def4567890", "quantization": "Q4_K_M"}, + "dyno_version": "1.0.2", + } + + +def _cp(args, rc=0, out="", err=""): + return subprocess.CompletedProcess(args, rc, stdout=out, stderr=err) + + +def test_submit_report_prefers_pr(report, monkeypatch): + monkeypatch.setattr(submit, "submit_via_pr", lambda r: "https://github.com/x/pull/1") + monkeypatch.setattr(submit, "submit_via_gist", lambda r: "https://gist.github.com/x") + assert submit.submit_report(report) == "https://github.com/x/pull/1" + + +def test_submit_report_falls_back_to_gist(report, monkeypatch): + monkeypatch.setattr(submit, "submit_via_pr", lambda r: None) + monkeypatch.setattr(submit, "submit_via_gist", lambda r: "https://gist.github.com/x") + assert submit.submit_report(report) == "https://gist.github.com/x" + + +def test_submit_report_falls_back_to_local(report, monkeypatch, tmp_path): + monkeypatch.setattr(submit, "submit_via_pr", lambda r: None) + monkeypatch.setattr(submit, "submit_via_gist", lambda r: None) + monkeypatch.chdir(tmp_path) + + with pytest.raises(RuntimeError, match="saved locally"): + submit.submit_report(report) + + saved = list((tmp_path / "dyno-results").glob("*.json")) + assert len(saved) == 1 + + +def test_submit_via_pr_returns_none_without_gh(report, monkeypatch): + monkeypatch.setattr(submit, "_gh_installed", lambda: False) + assert submit.submit_via_pr(report) is None + + +def test_submit_via_pr_success(report, monkeypatch): + monkeypatch.setattr(submit, "_gh_installed", lambda: True) + monkeypatch.setattr(submit, "_gh_auth_status", lambda: True) + + def fake_run(cmd, cwd=None): + if cmd[:3] == ["gh", "repo", "fork"]: + os.makedirs(os.path.join(cwd, "llama-dyno-results"), exist_ok=True) + return _cp(cmd, 0) + if cmd[:3] == ["gh", "api", "user"]: + return _cp(cmd, 0, out="octocat") + if cmd[:3] == ["gh", "pr", "create"]: + return _cp(cmd, 0, out="https://github.com/Lachytonner/llama-dyno-results/pull/7") + return _cp(cmd, 0) # git checkout/add/commit/push + + monkeypatch.setattr(submit, "_run", fake_run) + url = submit.submit_via_pr(report) + assert url == "https://github.com/Lachytonner/llama-dyno-results/pull/7" + + +def test_submit_via_pr_returns_none_when_push_fails(report, monkeypatch): + monkeypatch.setattr(submit, "_gh_installed", lambda: True) + monkeypatch.setattr(submit, "_gh_auth_status", lambda: True) + + def fake_run(cmd, cwd=None): + if cmd[:3] == ["gh", "repo", "fork"]: + os.makedirs(os.path.join(cwd, "llama-dyno-results"), exist_ok=True) + return _cp(cmd, 0) + if cmd[:2] == ["git", "push"]: + return _cp(cmd, 1, err="push rejected") + return _cp(cmd, 0) + + monkeypatch.setattr(submit, "_run", fake_run) + assert submit.submit_via_pr(report) is None diff --git a/tests/test_tune.py b/tests/test_tune.py new file mode 100644 index 0000000..c94791e --- /dev/null +++ b/tests/test_tune.py @@ -0,0 +1,93 @@ +"""Integration tests for the tuning search, run fully offline. + +These exercise run_tune and its phases (coarse sweep, hill climb, convergence, +pruning, MoE toggles) by monkeypatching the module's external calls — the mock +bench runner stands in for real llama-bench subprocesses. +""" + +from __future__ import annotations + +from conftest import MockBenchRunner + +import llama_dyno.tune as tune +from llama_dyno.tune import TuneConfig, run_tune +from llama_dyno.types import TrialResult + + +def _patch(monkeypatch, runner, *, vram=10_000, size=8_000, metadata=None): + """Wire run_tune's external calls to offline stand-ins. + + vram/size chosen so vram_ratio lands in the 0.8-1.5 band → the coarse sweep + tries ngl [99, 50, 25]; the mock OOMs above 60, so ngl=50 is the real peak. + """ + monkeypatch.setattr(tune, "run_bench", runner) + monkeypatch.setattr(tune, "find_bench_binary", lambda: "fake-bench") + monkeypatch.setattr(tune, "extract_model_metadata", lambda _p: metadata or {}) + monkeypatch.setattr(tune, "_detect_vram_mib", lambda: vram) + monkeypatch.setattr(tune, "_estimate_model_size", lambda _p: size) + + +def test_run_tune_finds_best_config_and_rejects_oom(monkeypatch): + mock = MockBenchRunner() + _patch(monkeypatch, mock.run) + + result = run_tune("dummy.gguf", mode="quick") + wp = result.winning_params + + # Mock optimum: ngl=50 (99 OOMs, 25 is slower), FA on, batch sweet spot 1024. + assert wp.ngl == 50 + assert wp.flash_attn is True + assert wp.batch_size == 1024 + assert wp.threads == 0 # auto beats forced thread counts in the mock + + # An OOM config was tried but never selected as the winner. + assert any(t.oom for t in result.trials), "ngl=99 should have OOM'd" + winner_trial = next(t for t in result.trials if t.params == wp) + assert not winner_trial.oom + + +def test_run_tune_respects_trial_budget(monkeypatch): + mock = MockBenchRunner() + _patch(monkeypatch, mock.run) + + result = run_tune("dummy.gguf", mode="thorough") + assert len(result.trials) <= TuneConfig.thorough().max_trials + + +def test_run_tune_does_not_rebench_duplicate_params(monkeypatch): + mock = MockBenchRunner() + _patch(monkeypatch, mock.run) + + run_tune("dummy.gguf", mode="thorough") + + seen = [tuple(sorted(p.to_dict().items())) for p in mock.calls] + assert len(seen) == len(set(seen)), "identical configs were benched twice" + + +def test_run_tune_early_convergence_skips_hill_climb(monkeypatch): + # Constant, never-OOM runner → Phase-1 scores are identical → converged. + def const_runner(model_path, params=None, binary=None, timeout=300, warmup=True, **kw): + return TrialResult(params=params, pp_tokens_s=100.0, tg_tokens_s=20.0) + + _patch(monkeypatch, const_runner) + result = run_tune("dummy.gguf", mode="quick") + + # Hill climb is the only phase that varies batch size; if it ran, some trial + # would have batch != 512. Convergence must have skipped it. + assert all(t.params.batch_size == 512 for t in result.trials) + + +def test_run_tune_moe_enables_ik_flags(monkeypatch): + mock = MockBenchRunner() + _patch(monkeypatch, mock.run, metadata={"is_moe": True}) + + result = run_tune("dummy.gguf", mode="quick", is_ik=True) + assert any(t.params.fmoe for t in result.trials), "MoE + ik should try -fmoe" + + +def test_run_tune_dense_skips_ik_flags(monkeypatch): + mock = MockBenchRunner() + _patch(monkeypatch, mock.run, metadata={"is_moe": False}) + + result = run_tune("dummy.gguf", mode="quick", is_ik=True) + assert not any(t.params.fmoe for t in result.trials), "dense model should skip -fmoe"