Search test coverage, submit PR flow, scoring consolidation

src/llama_dyno/cli.py

@@ -570,7 +570,7 @@ def submit(
         False, "--quick", help="Quick tune mode before submission"
     ),
 ):
-    """Submit results — creates a Gist with your benchmark."""
+    """Submit results — opens a PR to the community repo, else a Gist, else saves locally."""
     if not validate_model(model):
         console.print(f"[red]ERROR:[/] '{model}' is not a valid .gguf file.")
         raise typer.Exit(1)

src/llama_dyno/submit.py

@@ -1,13 +1,17 @@
-"""Submit benchmark results to GitHub Gist."""
+"""Submit benchmark results: PR to the community repo, else Gist, else local."""
 
 from __future__ import annotations
 
 import os
 import subprocess
+import tempfile
 from typing import Any
 
 from .report import format_json
 
+# Community results repo that `dyno submit` opens PRs against.
+COMMUNITY_REPO = "Lachytonner/llama-dyno-results"
+
 
 def _gh_installed() -> bool:
     """Check if gh CLI is available."""
@@ -69,6 +73,69 @@ def _result_json_content(report: dict[str, Any]) -> str:
     return format_json(report)
 
 
+def _run(cmd: list[str], cwd: str | None = None) -> subprocess.CompletedProcess:
+    """Run a subprocess with captured output and a generous timeout."""
+    return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=120)
+
+
+def submit_via_pr(report: dict[str, Any]) -> str | None:
+    """Open a PR against the community results repo with this benchmark.
+
+    Forks + clones COMMUNITY_REPO, adds the result JSON under
+    results/<gpu>/<filename>, pushes to the fork, and opens a PR via gh.
+    Returns the PR URL, or None if gh is unavailable or any step fails (the
+    caller then falls back to a Gist, then a local save).
+    """
+    if not _gh_installed() or not _gh_auth_status():
+        return None
+
+    filename = _result_filename(report)
+    gpu = _gpu_key(report.get("hardware", {}))
+    short_hash = (report.get("model", {}).get("sha256") or "result")[:12]
+    branch = f"dyno-{gpu}-{short_hash}"
+    title = (
+        f"Add result: {report.get('hardware', {}).get('gpu_name', 'GPU')}"
+        f" + {report.get('model', {}).get('name', 'model')}"
+    )
+    content = _result_json_content(report)
+
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            if _run(["gh", "repo", "fork", COMMUNITY_REPO, "--clone"], cwd=tmp).returncode != 0:
+                return None
+            repo_dir = os.path.join(tmp, COMMUNITY_REPO.split("/")[-1])
+            if not os.path.isdir(repo_dir):
+                return None
+
+            if _run(["git", "checkout", "-b", branch], cwd=repo_dir).returncode != 0:
+                return None
+
+            rel = f"results/{gpu}/{filename}"
+            dest = os.path.join(repo_dir, "results", gpu, filename)
+            os.makedirs(os.path.dirname(dest), exist_ok=True)
+            with open(dest, "w") as f:
+                f.write(content)
+
+            for step in (["git", "add", rel], ["git", "commit", "-m", title],
+                         ["git", "push", "-u", "origin", branch]):
+                if _run(step, cwd=repo_dir).returncode != 0:
+                    return None
+
+            # Disambiguate the head ref for a cross-fork PR.
+            who = _run(["gh", "api", "user", "-q", ".login"])
+            head = f"{who.stdout.strip()}:{branch}" if who.returncode == 0 and who.stdout.strip() else branch
+
+            pr = _run([
+                "gh", "pr", "create", "--repo", COMMUNITY_REPO, "--head", head,
+                "--title", title, "--body", "Automated benchmark submission via `dyno submit`.",
+            ], cwd=repo_dir)
+            if pr.returncode == 0 and pr.stdout.strip():
+                return pr.stdout.strip()
+    except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
+        return None
+    return None
+
+
 def submit_via_gist(report: dict[str, Any]) -> str | None:
     """Submit results as a GitHub Gist.
 
@@ -109,16 +176,20 @@ def submit_via_gist(report: dict[str, Any]) -> str | None:
 
 
 def submit_report(report: dict[str, Any]) -> str:
-    """Submit report, trying Gist first then saving locally.
+    """Submit report: PR to the community repo, else Gist, else save locally.
 
-    Returns the Gist URL on success.
-    Raises RuntimeError if submission fails (with a message including the local path).
+    Returns the PR or Gist URL on success.
+    Raises RuntimeError if all remote paths fail (message includes the local path).
     """
+    url = submit_via_pr(report)
+    if url:
+        return url
+
     url = submit_via_gist(report)
     if url:
         return url
 
-    # Gist failed — save locally
+    # Both remote paths failed — save locally
     result_dir = os.path.join(os.getcwd(), "dyno-results")
     os.makedirs(result_dir, exist_ok=True)
     filename = _result_filename(report)
@@ -127,8 +198,8 @@ def submit_report(report: dict[str, Any]) -> str:
         f.write(_result_json_content(report))
 
     raise RuntimeError(
-        "Could not submit via Gist. "
-        "Install gh: gh auth login\n\n"
+        "Could not submit via PR or Gist (is gh installed and authenticated? "
+        "run 'gh auth login').\n\n"
         f"Result saved locally to: {path}\n"
-        "You can manually submit it at https://gist.github.com"
+        "You can open a PR manually at https://github.com/" + COMMUNITY_REPO
     )

src/llama_dyno/tune.py

@@ -13,7 +13,7 @@
     find_bench_binary,
     run_bench,
 )
-from .types import BenchParams, TrialResult, TuneResult
+from .types import BenchParams, TrialResult, TuneResult, score_trial
 
 console = Console()
 
@@ -37,18 +37,8 @@ def thorough(cls) -> TuneConfig:
 
 
 def _score_trial(t: TrialResult, pp_weight: float = 0.3, tg_weight: float = 0.7) -> float:
-    """Combined score weighting pp and tg throughput.
-
-    Args:
-        t: Trial result.
-        pp_weight: Weight for prompt-processing throughput (default 0.3).
-        tg_weight: Weight for text-generation throughput (default 0.7).
-    """
-    if t.oom or t.error:
-        return -1.0
-    pp = t.pp_tokens_s or 0
-    tg = t.tg_tokens_s or 0
-    return pp * pp_weight + tg * tg_weight
+    """Score a trial via the canonical scorer in types.score_trial."""
+    return score_trial(t, pp_weight, tg_weight)
 
 
 def build_progress_table(trials: list[TrialResult]) -> Table:
@@ -406,6 +396,31 @@ def _hill_climb(
     return best_params
 
 
+def _detect_vram_mib() -> int:
+    """Detect total GPU VRAM in MiB (pynvml, then nvidia-smi). 0 if unknown.
+
+    A single monkeypatchable seam for the tuner's VRAM heuristic.
+    """
+    try:
+        import pynvml
+        pynvml.nvmlInit()
+        handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+        vram = pynvml.nvmlDeviceGetMemoryInfo(handle).total // (1024 * 1024)
+        pynvml.nvmlShutdown()
+        return int(vram)
+    except Exception:
+        pass
+    import subprocess as sp
+    try:
+        out = sp.run(
+            ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
+            capture_output=True, text=True, timeout=5,
+        )
+        return int(out.stdout.strip()) if out.stdout.strip() else 0
+    except Exception:
+        return 0
+
+
 def _estimate_model_size(model_path: str) -> int:
     """Estimate model file size in MiB."""
     import os
@@ -475,28 +490,7 @@ def run_tune(
         console.print("  Model type: Dense (skipping MoE-specific tuning)")
     console.print()
 
-    # Detect GPU VRAM
-    vram_total = 0
-    try:
-        import pynvml
-        pynvml.nvmlInit()
-        handle = pynvml.nvmlDeviceGetHandleByIndex(0)
-        vram_total = pynvml.nvmlDeviceGetMemoryInfo(handle).total // (1024 * 1024)
-        pynvml.nvmlShutdown()
-    except Exception:
-        pass
-    # Fallback nvidia-smi
-    if vram_total == 0:
-        import subprocess as sp
-        try:
-            out = sp.run(
-                ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
-                capture_output=True, text=True, timeout=5,
-            )
-            vram_total = int(out.stdout.strip()) if out.stdout.strip() else 0
-        except Exception:
-            pass
-
+    vram_total = _detect_vram_mib()
     model_size_mib = _estimate_model_size(model_path)
 
     console.print(f"[bold]Dyno Tuning[/] - mode: {mode}")

src/llama_dyno/types.py

@@ -102,13 +102,24 @@ class TrialResult:
 
     @property
     def score(self) -> float:
-        """Combined score weighting both pp and tg throughput."""
-        if self.oom or self.error:
-            return -1.0
-        pp = self.pp_tokens_s or 0
-        tg = self.tg_tokens_s or 0
-        # Weight prompt processing less than generation (typically 1:1 or 1:2)
-        return pp * 0.3 + tg * 0.7
+        """Combined score with default weights.
+
+        Convenience wrapper; use score_trial() directly to pass custom weights.
+        """
+        return score_trial(self)
+
+
+def score_trial(
+    trial: TrialResult, pp_weight: float = 0.3, tg_weight: float = 0.7
+) -> float:
+    """Combined pp/tg throughput score. OOM/error trials score -1.
+
+    Single source of truth for trial scoring — both TrialResult.score and the
+    tuner's scoring call through here so custom weights are always honored.
+    """
+    if trial.oom or trial.error:
+        return -1.0
+    return (trial.pp_tokens_s or 0) * pp_weight + (trial.tg_tokens_s or 0) * tg_weight
 
 
 @dataclass

tests/conftest.py

@@ -0,0 +1,62 @@
+"""Shared test fixtures and mocks for llama-dyno."""
+
+from __future__ import annotations
+
+from llama_dyno.types import BenchParams, TrialResult
+
+
+class MockBenchRunner:
+    """Simulates llama-bench runs so the search algorithm can be tested offline.
+
+    Speeds vary by parameter so the tuner has a deterministic optimum to find:
+    OOM above ngl 60, flash attention helps, KV quant helps slightly, ngl scales
+    up to 50, batch peaks in 1024-2048, and thread=auto is best.
+
+    The signature mirrors bench.run_bench so it can be monkeypatched in as a
+    drop-in replacement (positional or keyword args, plus binary/timeout/warmup).
+    """
+
+    def __init__(self):
+        self.calls: list[BenchParams] = []
+
+    def run(
+        self,
+        model_path: str,
+        params: BenchParams | None = None,
+        binary: str | None = None,
+        timeout: int = 300,
+        warmup: bool = True,
+        **kwargs,
+    ) -> TrialResult:
+        params = params or BenchParams()
+        self.calls.append(params)
+
+        # Simulate OOM at very high ngl
+        if params.ngl > 60:
+            return TrialResult(params=params, oom=True, error="CUDA OOM (mocked)")
+
+        fa_bonus = 1.2 if params.flash_attn else 1.0
+        kv_factor = {"f16": 1.0, "q8_0": 1.05, "q4_0": 1.1}.get(params.ct_k, 1.0)
+        ngl_factor = min(1.0, params.ngl / 50) * 0.5 + 0.5
+
+        batch_penalty = 1.0
+        if params.batch_size < 256:
+            batch_penalty = 0.7
+        elif params.batch_size < 1024:
+            batch_penalty = 0.9
+        elif params.batch_size > 2048:
+            batch_penalty = 0.85
+
+        thread_factor = 1.0
+        if 0 < params.threads < 4:
+            thread_factor = 0.8
+
+        base_pp = 500.0
+        base_tg = 30.0
+        mult = ngl_factor * fa_bonus * kv_factor * batch_penalty * thread_factor
+        return TrialResult(
+            params=params,
+            pp_tokens_s=base_pp * mult,
+            tg_tokens_s=base_tg * mult,
+            oom=False,
+        )

tests/test_search.py

@@ -2,70 +2,11 @@
 
 from __future__ import annotations
 
-import json
-from dataclasses import dataclass
-from typing import Callable
-
 import pytest
 
-from llama_dyno.types import BenchParams, TrialResult, TuneResult, model_info_from_path
-
-
-class MockBenchRunner:
-    """Mock that simulates bench runs for testing the search algorithm.
-
-    The mock returns speeds that vary by parameters, allowing us to verify
-    the tuner picks the optimal config.
-    """
+from conftest import MockBenchRunner
 
-    def __init__(self):
-        self.calls: list[BenchParams] = []
-
-    def run(self, model_path: str, params: BenchParams) -> TrialResult:
-        self.calls.append(params)
-
-        # Simulate OOM at very high ngl for "small VRAM" scenarios
-        if params.ngl > 60:
-            return TrialResult(params=params, oom=True, error="CUDA OOM (mocked)")
-
-        # Simulate better performance with flash attention
-        fa_bonus = 1.2 if params.flash_attn else 1.0
-
-        # Simulate KV cache quant effects
-        kv_factor = {"f16": 1.0, "q8_0": 1.05, "q4_0": 1.1}.get(params.ct_k, 1.0)
-
-        # Simulate ngl scaling (more layers = faster up to a point)
-        ngl_factor = min(1.0, params.ngl / 50) * 0.5 + 0.5
-
-        # Simulate batch size sweet spot at 1024
-        batch_penalty = 1.0
-        if params.batch_size < 256:
-            batch_penalty = 0.7
-        elif params.batch_size < 1024:
-            batch_penalty = 0.9  # sub-optimal
-        elif params.batch_size > 2048:
-            batch_penalty = 0.85
-
-        # Simulate thread scaling
-        thread_factor = 1.0
-        if params.threads == 0:
-            thread_factor = 1.0  # auto = good
-        elif params.threads < 4:
-            thread_factor = 0.8
-
-        # Base speed
-        base_pp = 500.0
-        base_tg = 30.0
-
-        pp = base_pp * ngl_factor * fa_bonus * kv_factor * batch_penalty * thread_factor
-        tg = base_tg * ngl_factor * fa_bonus * kv_factor * batch_penalty * thread_factor
-
-        return TrialResult(
-            params=params,
-            pp_tokens_s=pp,
-            tg_tokens_s=tg,
-            oom=False,
-        )
+from llama_dyno.types import BenchParams, TrialResult, model_info_from_path
 
 
 def test_bench_params_to_flag_list():