From c811478cfbc77aae671d6b4d2ee2fa5b728bd98e Mon Sep 17 00:00:00 2001 From: "Christian M. Todie" Date: Sat, 25 Jul 2026 19:51:41 +0200 Subject: [PATCH] feat(local-models): first-class MLX providers + move local servers to 1143x ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire mlx_lm.server (`mlx`) and `rapid-mlx` as first-class OpenAI-compatible providers across all four harness presets, and relocate every local model server off the crowded dev/test ports into a contiguous 1143x block beside ollama, so serving a local model never collides with a web dev server: ollama 11434 · llama.cpp 11435 · mlx 11436 · rapid-mlx 11437 · vLLM 11438 - add `mlx` provider (:11436, 6 local MLX models incl. gpt-oss-120b) and staged `rapid-mlx` (:11437, for the brew formula) to pi/opencode/hermes/omp - add packages/local-models/serve/serve-mlx.sh (mirrors serve-llamacpp.sh) - move llama.cpp 8080->11435 and vLLM 8000->11438 in presets, serve scripts (--port), and READMEs; historical docs/eval/* intentionally left at old ports - MLX is the fastest local backend on Apple silicon — worth first-class wiring Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Gni5WQNg9NwTZDszfhYSnf --- packages/create/README.md | 2 +- packages/harness-hermes/README.md | 2 +- .../config/cli-config.local-endpoint.yaml | 31 ++++++++++++++-- packages/harness-omp/README.md | 2 +- packages/harness-omp/config/models.yml | 35 +++++++++++++++++-- .../harness-opencode/config/opencode.json | 29 +++++++++++++-- packages/harness-pi/README.md | 2 +- packages/harness-pi/config/models.json | 25 +++++++++++-- packages/local-models/README.md | 6 ++-- packages/local-models/serve/serve-llamacpp.sh | 4 +-- packages/local-models/serve/serve-mlx.sh | 20 +++++++++++ packages/local-models/serve/serve-vllm.sh | 4 +-- 12 files changed, 143 insertions(+), 19 deletions(-) create mode 100755 packages/local-models/serve/serve-mlx.sh diff --git a/packages/create/README.md b/packages/create/README.md index b4c4590..5383f0e 100644 --- a/packages/create/README.md +++ b/packages/create/README.md @@ -101,7 +101,7 @@ with the "unsigned" color theme and OSC 133 shell integration. Model gateway config presets for all four agent harnesses, pointing at the unsigned model gateway (`llm.unsigned.gg/v1`) and local endpoints -(ollama `:11434`, llama.cpp `:8080`, vLLM `:8000`). Copy-if-absent — never +(ollama `:11434`, llama.cpp `:11435`, vLLM `:11438`). Copy-if-absent — never clobbers live config. Requires `UNSIGNED_LLM_API_KEY` for gateway access. ### `skills` — agentskills.io symlink installer diff --git a/packages/harness-hermes/README.md b/packages/harness-hermes/README.md index c369310..7ba0773 100644 --- a/packages/harness-hermes/README.md +++ b/packages/harness-hermes/README.md @@ -24,7 +24,7 @@ the upstream script resolves to, without executing unreviewed remote code. - **`config/cli-config.local-endpoint.yaml`** → seeded to `~/.hermes/cli-config.yaml` if absent: points hermes at a local - OpenAI-compatible endpoint (vLLM :8000 default; ollama/llama.cpp are a + OpenAI-compatible endpoint (vLLM :11438 default; ollama/llama.cpp are a one-line `openai_base_url` change). **Config-key caveat:** validate keys against `hermes config list` on your installed version — upstream moves fast (0.18.0 "The Judgment Release" is 2026-07-01); the preset is diff --git a/packages/harness-hermes/config/cli-config.local-endpoint.yaml b/packages/harness-hermes/config/cli-config.local-endpoint.yaml index a1fdaba..d209cda 100644 --- a/packages/harness-hermes/config/cli-config.local-endpoint.yaml +++ b/packages/harness-hermes/config/cli-config.local-endpoint.yaml @@ -88,9 +88,36 @@ providers: - qwen3.6-35b-fast - yutori/n1.5 + # mlx — Apple-silicon local (mlx_lm.server). First-class local provider on + # :11436 (off llama.cpp's :11435 so both co-tenant). Serve: `mlx-serve `. + mlx: + name: MLX (mlx_lm.server, local) + api: http://localhost:11436/v1 + api_key: none + default_model: mlx-community/gpt-oss-120b-MXFP4-Q8 + models: + - mlx-community/gpt-oss-120b-MXFP4-Q8 + - mlx-community/Qwen3.6-35B-A3B-8bit + - mlx-community/Devstral-Small-2-24B-Instruct-2512-4bit + - mlx-community/Qwen3-Coder-30B-A3B-Instruct-8bit + - mlx-community/gemma-4-31B-it-OptiQ-4bit + - mlx-community/Qwen3-VL-32B-Instruct-8bit + + # rapid-mlx — alternate OpenAI-compatible MLX server (brew: rapid-mlx) on :11437. + # Pending install/eval; staged so the endpoint resolves once served. + rapid-mlx: + name: rapid-mlx (local) + api: http://localhost:11437/v1 + api_key: none + default_model: mlx-community/gpt-oss-120b-MXFP4-Q8 + models: + - mlx-community/gpt-oss-120b-MXFP4-Q8 + # Alternate local endpoints (swap `api`): -# llama.cpp: http://localhost:8080/v1 (serve/serve-llamacpp.sh) -# vLLM: http://localhost:8000/v1 (serve/serve-vllm.sh) +# mlx (mlx_lm.server): http://localhost:11436/v1 (mlx-serve ) +# rapid-mlx: http://localhost:11437/v1 (brew: rapid-mlx) +# llama.cpp: http://localhost:11435/v1 (serve/serve-llamacpp.sh) +# vLLM: http://localhost:11438/v1 (serve/serve-vllm.sh) # Memory/skill-loop boundary vs Engram (OPS-480, 2026-07-08; see README Verdict # section for the full reasoning). hermes's built-in memory store (two flat diff --git a/packages/harness-omp/README.md b/packages/harness-omp/README.md index e7f46b9..86196f2 100644 --- a/packages/harness-omp/README.md +++ b/packages/harness-omp/README.md @@ -37,7 +37,7 @@ are maintained separately. the same endpoints as harness-pi's `models.json` (the `llm` unsigned model gateway at llm.unsigned.gg — Crusoe/OpenRouter/Anthropic/MorphLLM upstreams, key via `UNSIGNED_LLM_API_KEY`, 1P item `unsigned-llm` — plus ollama :11434, - llama.cpp :8080, vLLM :8000) in omp's provider schema. **Rename note + llama.cpp :11435, vLLM :11438) in omp's provider schema. **Rename note (2026-07-07):** the gateway provider was `neuralwatt`; presets are copy-if-absent, so an existing `~/.omp/agent/models.yml` keeps the old name — hand-merge the `llm` block (the old hostname keeps working through diff --git a/packages/harness-omp/config/models.yml b/packages/harness-omp/config/models.yml index 3f43082..577d42a 100644 --- a/packages/harness-omp/config/models.yml +++ b/packages/harness-omp/config/models.yml @@ -160,7 +160,7 @@ providers: contextWindow: 65536 reasoning: true llamacpp: - baseUrl: http://localhost:8080/v1 + baseUrl: http://localhost:11435/v1 api: openai-completions auth: none discovery: @@ -168,8 +168,39 @@ providers: models: - id: local vllm: - baseUrl: http://localhost:8000/v1 + baseUrl: http://localhost:11438/v1 api: openai-completions auth: none models: - id: served-model + mlx: + # Apple-silicon local (mlx_lm.server), first-class provider on :11436 — + # off llama.cpp's :11435 to co-tenant. Serve via `mlx-serve `. + baseUrl: http://localhost:11436/v1 + api: openai-completions + auth: none + models: + - id: mlx-community/gpt-oss-120b-MXFP4-Q8 + contextWindow: 131072 + reasoning: true + - id: mlx-community/Qwen3.6-35B-A3B-8bit + contextWindow: 131072 + reasoning: true + - id: mlx-community/Devstral-Small-2-24B-Instruct-2512-4bit + contextWindow: 131072 + - id: mlx-community/Qwen3-Coder-30B-A3B-Instruct-8bit + contextWindow: 262144 + - id: mlx-community/gemma-4-31B-it-OptiQ-4bit + contextWindow: 131072 + - id: mlx-community/Qwen3-VL-32B-Instruct-8bit + contextWindow: 131072 + rapid-mlx: + # Alternate OpenAI-compatible MLX server (brew: rapid-mlx) on :11437. + # Pending install/eval; staged so the endpoint resolves once served. + baseUrl: http://localhost:11437/v1 + api: openai-completions + auth: none + models: + - id: mlx-community/gpt-oss-120b-MXFP4-Q8 + contextWindow: 131072 + reasoning: true diff --git a/packages/harness-opencode/config/opencode.json b/packages/harness-opencode/config/opencode.json index 86e5512..728ee24 100644 --- a/packages/harness-opencode/config/opencode.json +++ b/packages/harness-opencode/config/opencode.json @@ -163,7 +163,7 @@ "npm": "@ai-sdk/openai-compatible", "name": "llama.cpp server (local)", "options": { - "baseURL": "http://localhost:8080/v1" + "baseURL": "http://localhost:11435/v1" }, "models": { "local": { @@ -175,13 +175,38 @@ "npm": "@ai-sdk/openai-compatible", "name": "vLLM (local)", "options": { - "baseURL": "http://localhost:8000/v1" + "baseURL": "http://localhost:11438/v1" }, "models": { "served-model": { "name": "vLLM served model" } } + }, + "mlx": { + "npm": "@ai-sdk/openai-compatible", + "name": "MLX (mlx_lm.server, local :11436)", + "options": { + "baseURL": "http://localhost:11436/v1" + }, + "models": { + "mlx-community/gpt-oss-120b-MXFP4-Q8": { "name": "gpt-oss-120b (MLX flagship)" }, + "mlx-community/Qwen3.6-35B-A3B-8bit": { "name": "Qwen3.6-35B-A3B (MLX fast)" }, + "mlx-community/Devstral-Small-2-24B-Instruct-2512-4bit": { "name": "Devstral-2 24B (MLX code)" }, + "mlx-community/Qwen3-Coder-30B-A3B-Instruct-8bit": { "name": "Qwen3-Coder-30B (MLX)" }, + "mlx-community/gemma-4-31B-it-OptiQ-4bit": { "name": "Gemma-4 31B (MLX vision)" }, + "mlx-community/Qwen3-VL-32B-Instruct-8bit": { "name": "Qwen3-VL 32B (MLX)" } + } + }, + "rapid-mlx": { + "npm": "@ai-sdk/openai-compatible", + "name": "rapid-mlx (local :11437)", + "options": { + "baseURL": "http://localhost:11437/v1" + }, + "models": { + "mlx-community/gpt-oss-120b-MXFP4-Q8": { "name": "gpt-oss-120b (rapid-mlx)" } + } } } } diff --git a/packages/harness-pi/README.md b/packages/harness-pi/README.md index 7719155..eef1cba 100644 --- a/packages/harness-pi/README.md +++ b/packages/harness-pi/README.md @@ -17,7 +17,7 @@ consumes: local-models/README.md (endpoint wiring) - **`config/models.json`** → `~/.pi/agent/models.json` (only if absent — live config is never clobbered): three OpenAI-compatible local providers — - `ollama` (:11434), `llamacpp` (:8080), `vllm` (:8000). Model ids are + `ollama` (:11434), `llamacpp` (:11435), `vllm` (:11438). Model ids are placeholders; align them with what you actually serve (`packages/local-models/MODELS.md`). - **`extensions/local-model-switch.ts`** → symlinked into diff --git a/packages/harness-pi/config/models.json b/packages/harness-pi/config/models.json index f8454df..03da1b1 100644 --- a/packages/harness-pi/config/models.json +++ b/packages/harness-pi/config/models.json @@ -224,7 +224,7 @@ ] }, "llamacpp": { - "baseUrl": "http://localhost:8080/v1", + "baseUrl": "http://localhost:11435/v1", "api": "openai-completions", "apiKey": "none", "models": [ @@ -234,7 +234,7 @@ ] }, "vllm": { - "baseUrl": "http://localhost:8000/v1", + "baseUrl": "http://localhost:11438/v1", "api": "openai-completions", "apiKey": "none", "models": [ @@ -242,6 +242,27 @@ "id": "served-model" } ] + }, + "mlx": { + "baseUrl": "http://localhost:11436/v1", + "api": "openai-completions", + "apiKey": "none", + "models": [ + { "id": "mlx-community/gpt-oss-120b-MXFP4-Q8", "contextWindow": 131072, "reasoning": true }, + { "id": "mlx-community/Qwen3.6-35B-A3B-8bit", "contextWindow": 131072, "reasoning": true }, + { "id": "mlx-community/Devstral-Small-2-24B-Instruct-2512-4bit", "contextWindow": 131072 }, + { "id": "mlx-community/Qwen3-Coder-30B-A3B-Instruct-8bit", "contextWindow": 262144 }, + { "id": "mlx-community/gemma-4-31B-it-OptiQ-4bit", "contextWindow": 131072 }, + { "id": "mlx-community/Qwen3-VL-32B-Instruct-8bit", "contextWindow": 131072 } + ] + }, + "rapid-mlx": { + "baseUrl": "http://localhost:11437/v1", + "api": "openai-completions", + "apiKey": "none", + "models": [ + { "id": "mlx-community/gpt-oss-120b-MXFP4-Q8", "contextWindow": 131072, "reasoning": true } + ] } } } diff --git a/packages/local-models/README.md b/packages/local-models/README.md index 1a67582..c98999e 100644 --- a/packages/local-models/README.md +++ b/packages/local-models/README.md @@ -12,14 +12,14 @@ Getting a **local model worth using** behind every harness in this repo. ./probe.sh # hardware → tier # read MODELS.md § your tier, pick a model, then ONE of: ollama pull # zero-friction -serve/serve-llamacpp.sh model.gguf # GGUF, full control, :8080 -serve/serve-vllm.sh # production path, :8000 +serve/serve-llamacpp.sh model.gguf # GGUF, full control, :11435 +serve/serve-vllm.sh # production path, :11438 ``` ## The contract Everything serves **OpenAI-compatible** on fixed local ports — ollama -`:11434`, llama.cpp `:8080`, vLLM `:8000` — and every harness preset in this +`:11434`, llama.cpp `:11435`, vLLM `:11438` — and every harness preset in this repo already points at all three: | Harness | Where the endpoint is wired | diff --git a/packages/local-models/serve/serve-llamacpp.sh b/packages/local-models/serve/serve-llamacpp.sh index 7eec6dc..88391b3 100755 --- a/packages/local-models/serve/serve-llamacpp.sh +++ b/packages/local-models/serve/serve-llamacpp.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Serve a GGUF model via llama.cpp's OpenAI-compatible server on :8080 — +# Serve a GGUF model via llama.cpp's OpenAI-compatible server on :11435 — # agent-ready flags (large context + native tool-call template). # Usage: serve-llamacpp.sh /path/to/model.gguf [ctx-size] set -euo pipefail @@ -20,4 +20,4 @@ exec llama-server \ --ctx-size "$CTX" \ --jinja \ --host 127.0.0.1 \ - --port 8080 + --port 11435 diff --git a/packages/local-models/serve/serve-mlx.sh b/packages/local-models/serve/serve-mlx.sh new file mode 100755 index 0000000..6870733 --- /dev/null +++ b/packages/local-models/serve/serve-mlx.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Serve an MLX model via mlx_lm.server's OpenAI-compatible API on :11436 — +# Apple-silicon-native (fastest local backend on M-series). Deliberately :11436, +# off llama.cpp's :11435, so both can co-tenant (matches the `mlx` harness provider). +# Usage: serve-mlx.sh (default: gpt-oss-120b flagship) +set -euo pipefail + +MODEL="${1:-mlx-community/gpt-oss-120b-MXFP4-Q8}" +PORT="${PORT:-11436}" + +# Prefer the dotfiles launcher (presets: flagship/fast/code/vision/vlm/…). +if command -v mlx-serve >/dev/null 2>&1; then + exec env PORT="$PORT" mlx-serve "$MODEL" +fi + +command -v mlx_lm.server >/dev/null 2>&1 || { + echo "mlx_lm.server not found. Install: uv tool install --python 3.12 mlx-lm" >&2 + exit 1 +} +exec mlx_lm.server --model "$MODEL" --host 127.0.0.1 --port "$PORT" --trust-remote-code diff --git a/packages/local-models/serve/serve-vllm.sh b/packages/local-models/serve/serve-vllm.sh index 89183e6..fe45153 100755 --- a/packages/local-models/serve/serve-vllm.sh +++ b/packages/local-models/serve/serve-vllm.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Serve a HF model via vLLM's OpenAI-compatible server on :8000 — agent-ready +# Serve a HF model via vLLM's OpenAI-compatible server on :11438 — agent-ready # flags (auto tool choice + parser). vLLM is the production-grade path # (throughput, concurrent harnesses); llama.cpp/ollama are the lighter paths. # Usage: serve-vllm.sh [max-model-len] @@ -20,4 +20,4 @@ exec vllm serve "$MODEL" \ --enable-auto-tool-choice \ --tool-call-parser "$PARSER" \ --host 127.0.0.1 \ - --port 8000 + --port 11438