diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43fc6d8..318d662 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python-version: ["3.11", "3.12", "3.13"] fail-fast: false diff --git a/README.md b/README.md index dcfd8c5..a00e924 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 🏎️ Dyno — llama.cpp Auto-Tuner & Benchmark -**Dyno** is an open-source CLI that auto-tunes and benchmarks [llama.cpp](https://github.com/ggml-org/llama.cpp) / [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) inference on NVIDIA GPUs, producing reproducible, shareable results. +**Dyno** is an open-source CLI that auto-tunes and benchmarks [llama.cpp](https://github.com/ggml-org/llama.cpp) / [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) inference on NVIDIA and Apple Silicon GPUs, producing reproducible, shareable results. ```bash pipx install llama-dyno @@ -34,8 +34,8 @@ dyno report ~/Downloads/my-model.q4_k_m.gguf ## Prerequisites - **Python 3.11+** -- **NVIDIA GPU** with drivers + CUDA -- **llama-bench** binary from [llama.cpp](https://github.com/ggml-org/llama.cpp) or [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) +- **A GPU**: NVIDIA (drivers + CUDA) or Apple Silicon (M-series, uses Metal + unified memory) +- **llama-bench** binary from [llama.cpp](https://github.com/ggml-org/llama.cpp) or [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) — built with the matching backend (CUDA or Metal) Install llama.cpp: @@ -182,7 +182,7 @@ pytest - GUI - Multi-GPU -- Non-NVIDIA backends (AMD, Intel, Apple Silicon) +- AMD (ROCm) / Intel backends (planned) - Server hosting for results ## License diff --git a/pyproject.toml b/pyproject.toml index 308393b..6e44998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,14 +5,14 @@ build-backend = "hatchling.build" [project] name = "llama-dyno" version = "1.1.0" -description = "Auto-tune and benchmark llama.cpp / ik_llama.cpp inference on NVIDIA GPUs" +description = "Auto-tune and benchmark llama.cpp / ik_llama.cpp inference on NVIDIA and Apple Silicon GPUs" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" authors = [ { name = "Flowdesk Systems", email = "lachy@flowdesk.systems" }, ] -keywords = ["llama.cpp", "benchmark", "tuning", "gpu", "nvidia", "inference"] +keywords = ["llama.cpp", "benchmark", "tuning", "gpu", "nvidia", "apple-silicon", "metal", "inference"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", diff --git a/src/llama_dyno/cli.py b/src/llama_dyno/cli.py index ead627b..320c751 100644 --- a/src/llama_dyno/cli.py +++ b/src/llama_dyno/cli.py @@ -206,7 +206,8 @@ def detect(): table.add_column("Value") table.add_row("GPU", hw.gpu_name) - table.add_row("VRAM", f"{hw.vram_total_mib} MiB") + mem_label = "Unified memory" if hw.gpu_name.startswith("Apple") else "VRAM" + table.add_row(mem_label, f"{hw.vram_total_mib} MiB") table.add_row("Driver", hw.driver_version) table.add_row("CUDA", hw.cuda_version or "N/A") table.add_row("CPU", hw.cpu_name) diff --git a/src/llama_dyno/detect.py b/src/llama_dyno/detect.py index 0c436a5..8a93291 100644 --- a/src/llama_dyno/detect.py +++ b/src/llama_dyno/detect.py @@ -151,6 +151,28 @@ def _detect_ram() -> int: return 0 +def _apple_silicon_gpu() -> tuple[str, int] | None: + """Detect an Apple Silicon GPU as (name, usable memory MiB), else None. + + Apple GPUs share unified memory with the CPU — there is no discrete VRAM, so + we report total RAM as the pool the GPU can draw from (macOS lets Metal use + most of it). + # ponytail: unified mem ≈ total RAM; refine via Metal recommendedMaxWorkingSetSize if OOM heuristics misfire + """ + if platform.system() != "Darwin" or platform.machine() != "arm64": + return None + chip = "" + try: + out = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], + capture_output=True, text=True, timeout=5, + ) + chip = out.stdout.strip() + except Exception: + pass + return chip or "Apple Silicon GPU", _detect_ram() + + def _find_binary(name: str) -> str | None: """Find a binary in PATH. Returns path or None.""" # WSL detection: prefer Linux binaries; Windows binaries under /mnt/c/ @@ -245,6 +267,13 @@ def detect_ik_features(binary_path: str | None = None) -> IkFeatures: def detect_hardware() -> HardwareFingerprint: """Fingerprint the current hardware and detect llama.cpp backend.""" gpu_name, vram, driver, cuda_ver = _detect_gpu() + # No NVIDIA GPU found — fall back to Apple Silicon (unified memory) if present. + if gpu_name == "Unknown" or vram == 0: + apple = _apple_silicon_gpu() + if apple: + gpu_name, vram = apple + driver = f"macOS {platform.mac_ver()[0]}" + cuda_ver = None cpu_name, cpu_cores = _detect_cpu() ram = _detect_ram() diff --git a/src/llama_dyno/tune.py b/src/llama_dyno/tune.py index 81fd0aa..b413614 100644 --- a/src/llama_dyno/tune.py +++ b/src/llama_dyno/tune.py @@ -397,9 +397,9 @@ def _hill_climb( def _detect_vram_mib() -> int: - """Detect total GPU VRAM in MiB (pynvml, then nvidia-smi). 0 if unknown. + """Detect GPU memory in MiB (NVIDIA VRAM, then Apple unified memory). 0 if unknown. - A single monkeypatchable seam for the tuner's VRAM heuristic. + A single monkeypatchable seam for the tuner's memory heuristic. """ try: import pynvml @@ -416,9 +416,13 @@ def _detect_vram_mib() -> int: ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, ) - return int(out.stdout.strip()) if out.stdout.strip() else 0 + if out.stdout.strip(): + return int(out.stdout.strip()) except Exception: - return 0 + pass + from .detect import _apple_silicon_gpu + apple = _apple_silicon_gpu() + return apple[1] if apple else 0 def _estimate_model_size(model_path: str) -> int: diff --git a/tests/test_detect.py b/tests/test_detect.py new file mode 100644 index 0000000..3f3d296 --- /dev/null +++ b/tests/test_detect.py @@ -0,0 +1,62 @@ +"""Tests for hardware detection, focused on the Apple Silicon path (mocked).""" + +from __future__ import annotations + +import subprocess + +from llama_dyno import detect + + +def _fake_sysctl(out: str): + def run(cmd, capture_output=True, text=True, timeout=5, **kw): + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + return run + + +def test_apple_silicon_gpu_on_arm_mac(monkeypatch): + monkeypatch.setattr(detect.platform, "system", lambda: "Darwin") + monkeypatch.setattr(detect.platform, "machine", lambda: "arm64") + monkeypatch.setattr(detect, "_detect_ram", lambda: 16384) + monkeypatch.setattr(detect.subprocess, "run", _fake_sysctl("Apple M2 Max\n")) + + assert detect._apple_silicon_gpu() == ("Apple M2 Max", 16384) + + +def test_apple_silicon_gpu_none_on_linux(monkeypatch): + monkeypatch.setattr(detect.platform, "system", lambda: "Linux") + monkeypatch.setattr(detect.platform, "machine", lambda: "x86_64") + assert detect._apple_silicon_gpu() is None + + +def test_apple_silicon_gpu_none_on_intel_mac(monkeypatch): + monkeypatch.setattr(detect.platform, "system", lambda: "Darwin") + monkeypatch.setattr(detect.platform, "machine", lambda: "x86_64") + assert detect._apple_silicon_gpu() is None + + +def test_detect_hardware_uses_apple_fallback(monkeypatch): + monkeypatch.setattr(detect, "_detect_gpu", lambda: ("Unknown", 0, "unknown", None)) + monkeypatch.setattr(detect, "_apple_silicon_gpu", lambda: ("Apple M2 Max", 32768)) + monkeypatch.setattr(detect.platform, "mac_ver", lambda: ("14.5", ("", "", ""), "arm64")) + monkeypatch.setattr(detect, "_find_binary", lambda name: None) + + hw = detect.detect_hardware() + assert hw.gpu_name == "Apple M2 Max" + assert hw.vram_total_mib == 32768 + assert hw.cuda_version is None + assert hw.driver_version.startswith("macOS") + + +def test_detect_hardware_keeps_nvidia(monkeypatch): + monkeypatch.setattr(detect, "_detect_gpu", lambda: ("NVIDIA RTX 4070", 12282, "550.0", "12.4")) + + def _boom(): + raise AssertionError("Apple fallback must not run when NVIDIA is present") + + monkeypatch.setattr(detect, "_apple_silicon_gpu", _boom) + monkeypatch.setattr(detect, "_find_binary", lambda name: None) + + hw = detect.detect_hardware() + assert hw.gpu_name == "NVIDIA RTX 4070" + assert hw.vram_total_mib == 12282 + assert hw.cuda_version == "12.4"