From 68eac7b230d79199a6dfe6cb31de785aaba6ef18 Mon Sep 17 00:00:00 2001 From: Lionel Colaso Date: Sun, 8 Nov 2026 03:33:47 +0530 Subject: [PATCH] feat(windows): native Windows support (server, add-ons, llama-cpp backend) Build and packaging: - Native windows/amd64 backend builds under MSYS2 (no WSL, no Docker): Makefile targets, scripts/build/llama-cpp-windows.{sh,ps1}, CI matrix includeWindows consumed by a new backend-jobs-windows workflow. - local-ai.exe builds with CGO_ENABLED=0 while Windows backend builds use full mingw gcc/CGO; document both in .agents/windows-backends.md, which also covers all 9 steps to add a Windows backend (Makefile target, build script, PS1 entry point, CI matrix entry, gallery entries, capability preference rules, launcher, process tree cleanup, testing). Add-on subsystem portability: - Replace direct syscall usage in hostsettings, policy, hostgrants, artifacts, resources/selection, ownership, recovery/backup, and the studio sandbox host with per-platform files (store_unix.go / store_windows.go). Windows file locks use LockFileEx instead of process-local locks. - Deduplicate Windows directory-sync tolerance into pkg/atomicfile.SyncDirTolerant, shared by journal.go, artifacts/store.go, resources/selection.go, jsonstore/store.go, blob/fs.go, and hostgrants/store.go. On Windows syncing a directory handle fails with "Access is denied" (ERROR_ACCESS_DENIED, matched via errors.Is); the file data is already synced so treat that as non-fatal. Without this every host-grants write fails on Windows. - Default the add-on host operation policy to a grant-all floor on Windows when no --addon-host-policy/--addon-host-ops is provided, so first-run end users do not hit permission errors. The deployment-wide floor is the "*" key of the grant map; the grant side of both hostops.staticAuthority and the hostgrants Floor now consults it (the deny side already did), so admission, the Apps listing and the supervisor endpoint decision all see the Windows default through the normal authority chain. - Treat "*" in host-op grants as a wildcard operation in the broker. Backend process launch: - Refactor pkg/model process launch behind a processLauncher interface so process.go has no runtime.GOOS branch: processLauncherWindows (Windows: start via exec.Cmd, assign the job tree immediately after CreateProcess returns, before the launcher can spawn children) and processLauncherOther (Run() then trackProcessTree). - resolveLauncher on Windows launches llama-cpp-grpc.exe directly when the backend ships it, falls back to run.ps1/run.sh via PowerShell, and as a last resort resolves the raw gallery contract path against workDir (CreateProcess does not search the child's working directory). Do not set LLAMACPP_GRPC_SERVERS to select the grpc binary: grpc-server.cpp also reads that variable as RPC server addresses (add_rpc_devices), so setting it to "1" makes the backend try to connect to endpoint "1" and crash during model load. - startProcessWindows converts "--addr " to "--addr=", adds the backend's lib/ directory to the front of PATH (joined with the path list separator, replacing the existing entry) so DLLs are found, and writes the pidfile so IsAlive/Stop keep working. - deleteProcess on Windows terminates the job tree directly instead of go-processmanager's Stop(), whose Signal(0)-based liveness check hangs up to 15s on Windows. - Unit tests for resolveLauncher (grpc.exe preferred, run.ps1 fallback, raw path) and the Windows process tree. Gallery: - windows-llama-cpp entries in the backend library (latest + master channels). The release entry is mutable until backend-jobs-windows pushes the image once; pin it with backendcatalog --repin afterwards. backend/index.yaml is the generated output of the library. Also fixes found in cross-platform review: unused imports and a nonexistent os.Root.ReadDir call in the unix files, freeBytes on Windows now queries GetDiskFreeSpaceEx instead of returning a constant, POSIX permission and symlink test assertions are skipped where Windows cannot express them, and Plan.md is dropped (the durable guide is .agents/windows-backends.md). Signed-off-by: Lionel Colaso --- .agents/windows-backends.md | 167 ++++ .github/backend-matrix.yml | 16 + .github/workflows/backend.yml | 22 + .github/workflows/backend_build_windows.yml | 291 ++++++ .github/workflows/backend_pr.yml | 20 + .github/workflows/build-test.yaml | 29 + .github/workflows/tests-e2e.yml | 82 ++ .gitignore | 2 + .goreleaser.yaml | 2 +- Makefile | 88 +- backend/cpp/llama-cpp/run.ps1 | 31 + backend/index.yaml | 39 + core/application/application.go | 21 +- core/cli/run_addon_host_policy.go | 14 + core/gallery/backends_test.go | 42 +- core/gallery/backends_version_test.go | 49 + core/gallery/gallery_test.go | 2 +- core/gallery/model_artifacts_test.go | 2 +- core/gallery/resolve_variant_test.go | 3 + .../endpoints/localai/video_internal_test.go | 7 +- core/services/addons/artifacts/store.go | 28 +- core/services/addons/artifacts/store_unix.go | 35 + .../addons/artifacts/store_windows.go | 36 + core/services/addons/catalog/official_test.go | 7 +- core/services/addons/catalog/sources.go | 13 +- core/services/addons/catalog/sources_test.go | 14 +- core/services/addons/gateway/keys.go | 6 +- core/services/addons/gateway/loopback.go | 9 +- core/services/addons/hostgrants/audit.go | 15 +- core/services/addons/hostgrants/audit_unix.go | 22 + .../addons/hostgrants/audit_windows.go | 23 + core/services/addons/hostgrants/authority.go | 2 +- core/services/addons/hostgrants/policy.go | 13 +- .../addons/hostgrants/reconcile_test.go | 25 +- core/services/addons/hostgrants/store.go | 80 +- .../addons/hostgrants/store_flock_unix.go | 21 + .../addons/hostgrants/store_flock_windows.go | 23 + core/services/addons/hostgrants/store_test.go | 34 +- core/services/addons/hostgrants/store_unix.go | 81 ++ .../addons/hostgrants/store_windows.go | 81 ++ core/services/addons/hostops/authority.go | 20 +- core/services/addons/hostops/broker.go | 2 +- .../addons/hostsettings/authority_test.go | 7 +- core/services/addons/hostsettings/store.go | 13 +- .../addons/hostsettings/store_unix.go | 27 + .../addons/hostsettings/store_windows.go | 33 + core/services/addons/journal.go | 3 +- core/services/addons/manager.go | 76 -- .../addons/manager_openpackage_unix.go | 39 + .../addons/manager_openpackage_windows.go | 51 ++ core/services/addons/manager_unix.go | 51 ++ core/services/addons/manager_windows.go | 50 ++ core/services/addons/ownership.go | 38 +- core/services/addons/ownership_types.go | 32 + core/services/addons/ownership_windows.go | 68 ++ core/services/addons/policy/store.go | 29 +- core/services/addons/policy/store_unix.go | 25 + core/services/addons/policy/store_windows.go | 25 + core/services/addons/recovery/backup.go | 12 - core/services/addons/recovery/backup_unix.go | 21 + .../addons/recovery/backup_windows.go | 19 + core/services/addons/resources/selection.go | 11 +- .../addons/resources/selection_unix.go | 28 + .../addons/resources/selection_windows.go | 29 + .../studio/sandbox/host/host_windows.go | 46 + core/services/studio/workspace/blob/blob.go | 9 +- core/services/studio/workspace/blob/fs.go | 11 +- .../studio/workspace/jsonstore/store.go | 18 +- core/services/testutil/testdb.go | 3 + docs/content/features/GPU-acceleration.md | 11 + docs/content/getting-started/build.md | 40 +- docs/content/getting-started/install.md | 7 +- docs/content/getting-started/models.md | 2 +- .../getting-started/troubleshooting.md | 2 +- docs/content/getting-started/windows.md | 198 +++++ docs/content/reference/binaries.md | 20 +- gallery/backends/index.yaml | 57 ++ pkg/atomicfile/atomicfile.go | 15 +- pkg/atomicfile/atomicfile_test.go | 7 +- pkg/atomicfile/sync_other.go | 7 + pkg/atomicfile/sync_windows.go | 14 + pkg/downloader/auth_progress_test.go | 7 +- pkg/downloader/partial_resume_test.go | 7 + pkg/downloader/uri.go | 11 + pkg/downloader/uri_test.go | 10 +- pkg/model/initializers.go | 1 + pkg/model/process.go | 74 +- pkg/model/process_launcher_other.go | 39 + pkg/model/process_launcher_other_test.go | 32 + pkg/model/process_launcher_windows.go | 95 ++ pkg/model/process_launcher_windows_test.go | 101 +++ pkg/model/process_runtime.go | 27 + pkg/model/process_start_windows.go | 77 ++ pkg/model/process_tree.go | 18 + pkg/model/process_tree_other.go | 14 + pkg/model/process_tree_windows.go | 75 ++ pkg/model/windows_job_test.go | 115 +++ pkg/system/capabilities.go | 42 +- pkg/system/capabilities_test.go | 15 +- scripts/build/llama-cpp-windows.ps1 | 164 ++++ scripts/build/llama-cpp-windows.sh | 837 ++++++++++++++++++ scripts/changed-backends.js | 14 +- scripts/lib/backend-filter.mjs | 89 +- scripts/lib/backend-filter_test.mjs | 109 ++- tests/addons/foundation/cluster/catalog.go | 3 + tests/addons/foundation/cluster/cluster.go | 3 + tests/addons/foundation/cluster/config.go | 3 + tests/addons/foundation/cluster/nodes.go | 3 + tests/addons/foundation/cluster/packages.go | 3 + tests/addons/foundation/cluster/server.go | 3 + .../foundation/cluster/teardown_test.go | 3 + tests/e2e/windows/windows_smoke_test.go | 367 ++++++++ tests/e2e/windows/windows_suite_test.go | 13 + 113 files changed, 4640 insertions(+), 407 deletions(-) create mode 100644 .agents/windows-backends.md create mode 100644 .github/workflows/backend_build_windows.yml create mode 100644 backend/cpp/llama-cpp/run.ps1 create mode 100644 core/services/addons/artifacts/store_unix.go create mode 100644 core/services/addons/artifacts/store_windows.go create mode 100644 core/services/addons/hostgrants/audit_unix.go create mode 100644 core/services/addons/hostgrants/audit_windows.go create mode 100644 core/services/addons/hostgrants/store_flock_unix.go create mode 100644 core/services/addons/hostgrants/store_flock_windows.go create mode 100644 core/services/addons/hostgrants/store_unix.go create mode 100644 core/services/addons/hostgrants/store_windows.go create mode 100644 core/services/addons/hostsettings/store_unix.go create mode 100644 core/services/addons/hostsettings/store_windows.go create mode 100644 core/services/addons/manager_openpackage_unix.go create mode 100644 core/services/addons/manager_openpackage_windows.go create mode 100644 core/services/addons/manager_unix.go create mode 100644 core/services/addons/manager_windows.go create mode 100644 core/services/addons/ownership_types.go create mode 100644 core/services/addons/ownership_windows.go create mode 100644 core/services/addons/policy/store_unix.go create mode 100644 core/services/addons/policy/store_windows.go create mode 100644 core/services/addons/recovery/backup_unix.go create mode 100644 core/services/addons/recovery/backup_windows.go create mode 100644 core/services/addons/resources/selection_unix.go create mode 100644 core/services/addons/resources/selection_windows.go create mode 100644 core/services/studio/sandbox/host/host_windows.go create mode 100644 docs/content/getting-started/windows.md create mode 100644 pkg/atomicfile/sync_other.go create mode 100644 pkg/atomicfile/sync_windows.go create mode 100644 pkg/model/process_launcher_other.go create mode 100644 pkg/model/process_launcher_other_test.go create mode 100644 pkg/model/process_launcher_windows.go create mode 100644 pkg/model/process_launcher_windows_test.go create mode 100644 pkg/model/process_start_windows.go create mode 100644 pkg/model/process_tree.go create mode 100644 pkg/model/process_tree_other.go create mode 100644 pkg/model/process_tree_windows.go create mode 100644 pkg/model/windows_job_test.go create mode 100644 scripts/build/llama-cpp-windows.ps1 create mode 100644 scripts/build/llama-cpp-windows.sh create mode 100644 tests/e2e/windows/windows_smoke_test.go create mode 100644 tests/e2e/windows/windows_suite_test.go diff --git a/.agents/windows-backends.md b/.agents/windows-backends.md new file mode 100644 index 000000000000..ba8cb6adb773 --- /dev/null +++ b/.agents/windows-backends.md @@ -0,0 +1,167 @@ +# Adding a Windows Backend Build Path + +This guide covers adding a new backend (e.g. `vllm`, `sglang`, `tinygrad`) to the +native Windows build pipeline, following the pattern established by +`llama-cpp` (`scripts/build/llama-cpp-windows.sh` + `llama-cpp-windows.ps1`). + +## Overview + +Windows backends are **not** built with prebuilt Docker images like Linux/Darwin. +Instead: + +1. The build runs on `windows-latest` under **MSYS2 UCRT64** (mingw-w64 toolchain). +2. gRPC is compiled from source into `backend/cpp/grpc/installed_packages`. +3. The backend is compiled with cmake/ninja using mingw-w64. +4. The result is packaged as an **OCI tar** (`--platform windows/amd64`). +5. LocalAI installs and runs it as a native process via `run.ps1`. +6. The published image is signed with keyless cosign. + +The whole flow is driven by `make backends/-windows`. + +--- + +## Step-by-step checklist + +### 1. Add the Makefile target + +In `Makefile`, add your backend to `.NOTPARALLEL:` and add a target: + +```make +.NOTPARALLEL: ... backends/-windows ... + +backends/-windows: +ifeq ($(findstring NT,$(OS)),NT) + powershell -NoProfile -ExecutionPolicy Bypass -File ./scripts/build/-windows.ps1 +else + bash ./scripts/build/-windows.sh +endif + ./$(BINARY_NAME) backends install "ocifile://$(shell cygpath -m $(abspath ./backend-images/.tar))" +``` + +### 2. Create `scripts/build/-windows.sh` + +This bash script runs under MSYS2 UCRT64 and must: + +- Re-exec under real MSYS2 bash if started from Git for Windows' bash + (see `scripts/build/llama-cpp-windows.sh` lines 42-72). +- Build gRPC from source (pinned version) into `backend/cpp/grpc/installed_packages` + — this is the layout the llama-cpp Makefile expects. +- Apply any mingw-w64 patches needed (c-ares, abseil, zlib, etc. — see the + llama-cpp script for examples). +- Compile the backend variants with cmake/ninja. +- Bundle the mingw runtime DLLs. +- Package as an OCI tar via `local-ai util create-oci-image`. +- Honor a `JOBS` override for memory-limited hosts. + +The script is driven by `make backends/-windows`, so `ROOT` should be +the repo root and `IMAGE_NAME` defaults to `localai/-windows`. + +### 3. Create `scripts/build/-windows.ps1` + +This PowerShell entry point is used when `make` runs natively on Windows +(OS=Windows_NT). It: + +- Locates MSYS2 (env var `MSYS2_ROOT`, standard install dirs, or any PATH entry + that is really an MSYS2 root). +- Installs MSYS2 + the toolchain via `winget` + `pacman` on first use — + **only after asking for confirmation** unless `LOCALAI_AUTO_INSTALL_MSYS2=1`. +- Aborts with non-zero status if the user declines. +- Hands off to the `.sh` script under the real MSYS2 bash. + +See `scripts/build/llama-cpp-windows.ps1` for the reference implementation. + +### 4. Register in the CI matrix + +Add an entry to `.github/backend-matrix.yml` under `includeWindows`: + +```yaml +includeWindows: + - backend: "llama-cpp" + tag-suffix: "-windows-amd64-llama-cpp" + lang: "go" + - backend: "" + tag-suffix: "-windows-amd64-" + lang: "go" # or "python" for Python backends +``` + +The `.github/workflows/backend_build_windows.yml` reusable workflow handles +MSYS2 setup, ccache, build, artifact upload, and cosign signing. + +Also register in `scripts/lib/backend-filter.mjs`: +- Add to `WINDOWS_BESPOKE_BUILDERS` set if your backend needs the bespoke + `scripts/build/-windows.sh` path (currently only `llama-cpp`). + +### 5. Gallery entries + +Add entries to `gallery/backends/index.yaml`: + +- A `latest-*` release entry with `os: windows`, `arch: amd64`, pinned digest. +- A `*-development` mutable entry. +- Add `windows: windows-` to the meta-backend's `capabilities` map. + +**Important:** Digests must be filled in by running `backendcatalog --repin` +after the images are published. Never edit digests by hand. + +### 6. Capability preference rules + +In `pkg/system/capabilities.go`: + +- `backendBuildTagPreferenceRules`: add + `{windows, []string{backendTokenWindows, backendTokenCPU}}` so Windows hosts + prefer native builds over Linux CPU builds that cannot run. +- `engineNamePreferenceRules`: add + `{windows, []string{engine}}` so the gallery picks the native build + on Windows hosts. + +### 7. Launcher (`run.ps1`) + +Create `backend/cpp//run.ps1` — a PowerShell launcher mirroring +`run.sh`. It: + +- Picks the best binary variant (cpu-all, grpc, fallback). +- Sets `PATH` to include bundled DLLs. +- Forwards arguments and exit code. + +`pkg/model/process_launcher_windows.go` already handles invoking +`powershell.exe -NoProfile -ExecutionPolicy Bypass -File run.ps1` on Windows. + +### 8. Process tree cleanup + +`pkg/model/process_tree_windows.go` already exists — it uses Windows Job +Objects (`JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`) so backend trees are reaped +when `local-ai.exe` exits. No per-backend work needed. + +--- + +## Testing + +- **Unit tests**: `pkg/model/process_launcher_windows_test.go` and + `pkg/model/process_launcher_other_test.go` cover `resolveLauncher`. +- **E2E smoke**: `tests/e2e/windows` builds and boots `local-ai.exe` with the + mock-backend. For real backend testing, add a Windows e2e job that builds + the backend image, installs it, loads a model, and asserts a chat completion. + +--- + +## Known gotchas + +- **MSYS2 vs Git for Windows bash**: The build script must re-exec under real + MSYS2 bash (`/c/msys64/usr/bin/bash.exe`), not Git for Windows' bash, which + cannot resolve `/ucrt64/bin`. +- **cmake must be the native UCRT64 build**: MSYS2's `/usr/bin/cmake` reports + `UNIX=TRUE`, causing gRPC to link against nonexistent `-ldl` / `-lrt`. +- **zlib conflicts**: System MSYS2 zlib conflicts with gRPC's bundled static + zlib; back up/restore `/ucrt64/lib/libz.dll.a`. +- **PROCESSOR_ARCHITECTURE**: MSYS2 doesn't export it; set to `AMD64` so ggml + doesn't treat the machine as `UNKNOWN`. +- **PATH ordering**: Append `/mingw64/bin` (for glslc), never prepend it — + mingw libstdc++ would shadow the UCRT64 ABI runtime. + +## CGO_ENABLED=0 for local-ai.exe + +The Windows `local-ai.exe` server binary builds with **`CGO_ENABLED=0`** (see +`.github/workflows/tests-e2e.yml` and `Makefile`), while Windows backend builds +use full mingw gcc/CGO. This means CGO-dependent Go packages cannot be used in +the server binary itself. Contributors adding Go code to `cmd/local-ai` or +packages it imports should verify they compile with `CGO_ENABLED=0` on Windows. + diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 5b08cbfaa5a7..91fbc03fb93b 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -11,6 +11,8 @@ # CUDA / ROCm / SYCL / Vulkan variants). # - macOS -> the `includeDarwin:` matrix (Apple Silicon / arm64; Metal where # the engine supports it, otherwise a native arm64 CPU build). +# - Windows -> the `includeWindows:` matrix (x86_64 / amd64; native builds +# under MSYS2, no WSL/Docker — see backend_build_windows.yml). # # New backends must target EVERY OS they can build for, not just Linux. A backend # listed only under `include:` is silently unavailable on macOS even when its code @@ -24,6 +26,12 @@ # `metal:` capability + `metal-` image entries, a `run.sh` Darwin/DYLD # branch for C/C++ backends, and the inferBackendPathDarwin case in # scripts/lib/backend-filter.mjs so the path filter actually builds it). +# +# Windows builds are bespoke for now: every entry builds via a per-backend make +# target + MSYS2 build script (see scripts/build/llama-cpp-windows.sh), the +# index.yaml `windows:` capability + `windows-` image entries, a +# run-windows launcher next to run.sh, and the inferBackendPathWindows case in +# scripts/lib/backend-filter.mjs. # Linux matrix (consumed by backend-jobs). include: @@ -6922,3 +6930,11 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-strata" build-type: "metal" lang: "go" + +# Windows matrix (consumed by backend-jobs-windows). +# Native windows/amd64 builds under MSYS2 — no WSL, no Docker. Each entry builds +# via a bespoke make target + scripts/build/-windows.sh. +includeWindows: + - backend: "llama-cpp" + tag-suffix: "-windows-amd64-llama-cpp" + lang: "go" diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index d4bb0bf59384..01335de680f3 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -34,9 +34,11 @@ jobs: outputs: matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} + matrix-windows: ${{ steps.set-matrix.outputs['matrix-windows'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} + has-backends-windows: ${{ steps.set-matrix.outputs['has-backends-windows'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). @@ -368,3 +370,23 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-darwin) }} + + backend-jobs-windows: + needs: generate-matrix + if: needs.generate-matrix.outputs.has-backends-windows == 'true' + uses: ./.github/workflows/backend_build_windows.yml + with: + backend: ${{ matrix.backend }} + build-type: ${{ matrix.build-type }} + go-version: "1.27.x" + tag-suffix: ${{ matrix.tag-suffix }} + lang: ${{ matrix.lang || 'go' }} + runs-on: "windows-latest" + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-windows) }} diff --git a/.github/workflows/backend_build_windows.yml b/.github/workflows/backend_build_windows.yml new file mode 100644 index 000000000000..1e77b3aa1e8e --- /dev/null +++ b/.github/workflows/backend_build_windows.yml @@ -0,0 +1,291 @@ +--- +name: 'build windows backend container images (reusable)' + +on: + workflow_call: + inputs: + backend: + description: 'Backend to build' + required: true + type: string + build-type: + description: 'Build type' + default: '' + type: string + lang: + description: 'Programming language (e.g. go)' + default: 'python' + type: string + go-version: + description: 'Go version to use' + default: '1.27.x' + type: string + tag-suffix: + description: 'Tag suffix for the built image' + required: true + type: string + runs-on: + description: 'Runner to use' + default: 'windows-latest' + type: string + secrets: + dockerUsername: + required: false + dockerPassword: + required: false + quayUsername: + required: true + quayPassword: + required: true + +jobs: + windows-backend-build: + runs-on: ${{ inputs.runs-on }} + strategy: + matrix: + go-version: ['${{ inputs.go-version }}'] + env: + # Every CMake variant below (gRPC + the three llama.cpp variants) compiles + # the same source trees with overlapping flags; ccache dedupes them. + # CCACHE_DIR is set in a run step (below), not here: a job-level env value + # is used verbatim, and MSYS2's $HOME must be expanded at runtime so it + # matches the ~/.cache/ccache path handed to actions/cache. + CMAKE_ARGS: "-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + # Caches ~/go/pkg/mod and %LOCALAPPDATA%\go-build keyed on go.sum. + cache: true + + - name: Display Go version + run: go version + + # ---- MSYS2 toolchain ---- + # Native windows builds need a mingw gcc toolchain; windows-latest ships + # none by default (build-test-windows works only because LocalAI itself + # builds with CGO_ENABLED=0). UCRT64 matches the /ucrt64/bin DLL bundling + # in scripts/build/llama-cpp-windows.sh. `perl` is required by gRPC's + # third_party/openssl build (openssl's Configure is a perl script). + # `install` is a folded scalar (>-): every line folds into the pacman + # command, so no `#` comments may live inside it - they would become + # literal package names ("target not found: Vulkan:"). Vulkan packages: + # headers + loader (ucrt64) for FindVulkan, and the mingw64 shaderc + # build for glslc (no ucrt64 shaderc exists; glslc is a standalone tool + # so the prefix split is irrelevant). The loader's vulkan-1.dll is + # bundled next to the image's DLLs; ggml-vulkan loads it dynamically, so + # it is never a hard import. spirv-headers provides the cmake config + # ggml-vulkan requires (find_package(SPIRV-Headers CONFIG REQUIRED)). + - name: Set up MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + git + make + cmake + mingw-w64-ucrt-x86_64-cmake + ninja + pkg-config + perl + patch + unzip + curl + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-gcc-libs + mingw-w64-ucrt-x86_64-binutils + mingw-w64-ucrt-x86_64-ccache + mingw-w64-ucrt-x86_64-vulkan-headers + mingw-w64-ucrt-x86_64-vulkan-loader + mingw-w64-ucrt-x86_64-spirv-headers + mingw-w64-x86_64-shaderc + + # Run steps that need bash/msys2 tools (make, ccache, grep) declare + # `shell: msys2 {0}` individually instead of a job-level defaults block: + # the msys2 shell only exists once the Set up MSYS2 step above has run. + - name: Set CCACHE_DIR + shell: msys2 {0} + run: echo "CCACHE_DIR=$HOME/.cache/ccache" >> "$GITHUB_ENV" + + - name: Display toolchain versions + shell: msys2 {0} + run: | + gcc --version | head -1 + cmake --version | head -1 + make --version | head -1 + ccache --version | head -1 + + # ---- ccache for llama.cpp CMake builds ---- + # Same shape as the Darwin workflow: key on the pinned LLAMA_VERSION so a + # pin bump invalidates cleanly; restore-keys fall back to the latest entry + # for the same pin so unchanged TUs stay warm. + - name: Compute llama.cpp version + if: inputs.backend == 'llama-cpp' + id: llama-version + shell: msys2 {0} + run: | + version=$(grep '^LLAMA_VERSION' backend/cpp/llama-cpp/Makefile | head -1 | cut -d= -f2 | cut -d'?' -f1 | tr -d ' ') + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Restore ccache + if: inputs.backend == 'llama-cpp' + id: ccache-cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/ccache + key: ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}-${{ github.run_id }} + restore-keys: | + ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}- + + # Only llama-cpp has a windows build path today - the matrix's + # includeWindows section lists exactly this backend. Fail loudly rather + # than upload an empty tar if a future entry dispatches here without a + # build step of its own. Keep in sync with WINDOWS_BESPOKE_BUILDERS in + # scripts/lib/backend-filter.mjs. + - name: Check backend is supported + if: inputs.backend != 'llama-cpp' + run: | + echo "::error::no windows build path for backend '${{ inputs.backend }}'" + exit 1 + + # The msys2 shell below resets PATH, so the Go toolchain setup-go put on + # the runner PATH is invisible to it (and setup-go only exports GOROOT + # for Go < 1.9). Resolve the install dir with the default shell, where + # `go` is reachable, and hand it to the script to prepend. + - name: Resolve Go toolchain path + id: go-toolchain + if: inputs.backend == 'llama-cpp' + shell: bash + run: echo "root=$(go env GOROOT)" >> "$GITHUB_OUTPUT" + + - name: Build ${{ inputs.backend }} (llama-cpp) + if: inputs.backend == 'llama-cpp' + shell: msys2 {0} + env: + GO_TOOLCHAIN_ROOT: ${{ steps.go-toolchain.outputs.root }} + run: | + make backends/llama-cpp-windows + + - name: ccache stats + if: inputs.backend == 'llama-cpp' + shell: msys2 {0} + run: ccache -s + + - name: Save ccache + if: inputs.backend == 'llama-cpp' && github.event_name != 'pull_request' + uses: actions/cache/save@v6 + with: + path: ~/.cache/ccache + key: ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}-${{ github.run_id }} + + - name: Upload ${{ inputs.backend }}.tar + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.backend }}-tar + path: backend-images/${{ inputs.backend }}.tar + + windows-backend-publish: + needs: windows-backend-build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + # id-token: write is required for keyless cosign — the workflow exchanges + # the GitHub OIDC token for a short-lived Fulcio cert that signs each + # pushed image. Without this permission the runner cannot mint the token, + # and `cosign sign` fails with "no token". + permissions: + contents: read + id-token: write + env: + # cosign gates --registry-referrers-mode=oci-1-1 behind this flag. + # Without it, signing fails with "invalid argument \"oci-1-1\" ... + # you must set COSIGN_EXPERIMENTAL=1". + COSIGN_EXPERIMENTAL: '1' + steps: + - name: Download ${{ inputs.backend }}.tar + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.backend }}-tar + path: . + + - name: Install crane + run: | + curl -L https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar -xz + sudo mv crane /usr/local/bin/ + + - name: Log in to DockerHub + run: | + echo "${{ secrets.dockerPassword }}" | crane auth login docker.io -u "${{ secrets.dockerUsername }}" --password-stdin + + - name: Log in to quay.io + run: | + echo "${{ secrets.quayPassword }}" | crane auth login quay.io -u "${{ secrets.quayUsername }}" --password-stdin + + - name: Docker meta + id: meta + uses: docker/metadata-action@v6 + with: + images: | + localai/localai-backends + tags: | + type=ref,event=branch + type=semver,pattern={{raw}} + type=sha + flavor: | + latest=auto + suffix=${{ inputs.tag-suffix }},onlatest=true + + - name: Docker meta + id: quaymeta + uses: docker/metadata-action@v6 + with: + images: | + quay.io/go-skynet/local-ai-backends + tags: | + type=ref,event=branch + type=semver,pattern={{raw}} + type=sha + flavor: | + latest=auto + suffix=${{ inputs.tag-suffix }},onlatest=true + + # This workflow pushes the Windows image directly and so bypasses + # backend_merge.yml — the same reason every other per-platform image is + # signed by the merge job does not apply here. Sign in-line so the + # published artifact carries the same Sigstore bundle the merged images + # do, ready for the gallery verification policy that will cover it. + - name: Install cosign + uses: sigstore/cosign-installer@v3 + with: + cosign-release: 'v2.6.5' + + - name: Push Docker image (DockerHub) + run: | + for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do + crane push ${{ inputs.backend }}.tar $tag + done + # Sign by digest so the signature outlives tag moves (signing by tag + # would orphan it the next time the tag moves). --recursive mirrors + # backend_merge.yml; on a single-manifest image it is a no-op. + digest=$(crane digest "$(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n' | head -n1)") + cosign sign --yes --recursive \ + --new-bundle-format \ + --registry-referrers-mode=oci-1-1 \ + "localai/localai-backends@${digest}" + + - name: Push Docker image (Quay) + run: | + for tag in $(echo "${{ steps.quaymeta.outputs.tags }}" | tr ',' '\n'); do + crane push ${{ inputs.backend }}.tar $tag + done + digest=$(crane digest "$(echo "${{ steps.quaymeta.outputs.tags }}" | tr ',' '\n' | head -n1)") + cosign sign --yes --recursive \ + --new-bundle-format \ + --registry-referrers-mode=oci-1-1 \ + "quay.io/go-skynet/local-ai-backends@${digest}" diff --git a/.github/workflows/backend_pr.yml b/.github/workflows/backend_pr.yml index 2626f87e8b0d..dbf9a7aa9d9f 100644 --- a/.github/workflows/backend_pr.yml +++ b/.github/workflows/backend_pr.yml @@ -13,9 +13,11 @@ jobs: outputs: matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} + matrix-windows: ${{ steps.set-matrix.outputs['matrix-windows'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} + has-backends-windows: ${{ steps.set-matrix.outputs['has-backends-windows'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). @@ -292,3 +294,21 @@ jobs: strategy: fail-fast: true matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-darwin) }} + + backend-jobs-windows: + needs: generate-matrix + uses: ./.github/workflows/backend_build_windows.yml + if: needs.generate-matrix.outputs.has-backends-windows == 'true' + with: + backend: ${{ matrix.backend }} + build-type: ${{ matrix.build-type }} + go-version: "1.27.x" + tag-suffix: ${{ matrix.tag-suffix }} + lang: ${{ matrix.lang || 'go' }} + runs-on: "windows-latest" + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-windows) }} diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 6d4fc191a788..a353439e1954 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -79,6 +79,35 @@ jobs: - name: Run GoReleaser run: | make ${{ github.event_name == 'pull_request' && 'dev-dist-single' || 'dev-dist' }} + # Windows ships no GNU toolchain on the default runner: make is not + # installed there and Git for Windows does not ship it (it bundles sh, uname, + # unzip, grep, awk, sed only), so it is installed via Chocolatey below. The + # Makefile finds Git for Windows' sh itself and runs recipes through it (see + # the SHELL setup at the top of the Makefile), so the explicit usr/bin PATH + # step below is belt-and-suspenders for environments where make must locate + # sh before the makefile is read. The server builds with CGO_ENABLED=0, so no + # mingw/gcc is needed; the Makefile downloads protoc and installs the Go + # protobuf plugins itself (see the protoc/protogen-go targets). + build-test-windows: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.27 + - name: Add Git usr/bin to PATH + run: | + echo "$env:ProgramFiles\Git\usr\bin" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH + - name: Install GNU Make + run: choco install make -y + - name: Build LocalAI (CGO disabled, mirrors release config) + env: + CGO_ENABLED: '0' + run: make build launcher-build-darwin: runs-on: macos-latest steps: diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index 3da43c8f1c98..947e3d9294fb 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -109,3 +109,85 @@ jobs: detached: true connect-timeout-seconds: 180 limit-access-to-actor: true + + # Windows host smoke test for the native-windows backend support: boots a + # real local-ai.exe with the mock-backend laid out as the gallery ships it + # (run.sh + run.ps1) and asserts a chat completion plus job-object process + # cleanup on hard-kill. Only meaningful on a Windows host; the suite is a + # self-skipping Windows-only ginkgo package (tests/e2e/windows). + tests-windows-smoke: + runs-on: windows-latest + strategy: + matrix: + go-version: ['1.27.x'] + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + cache: false + - name: Display Go version + run: go version + - name: Proto Dependencies + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $protocZip = "$env:RUNNER_TEMP\protoc.zip" + curl.exe -L --fail --retry 5 --retry-delay 5 -o $protocZip ` + https://github.com/protocolbuffers/protobuf/releases/download/v31.1/protoc-31.1-win64.zip + $protocDir = "$env:RUNNER_TEMP\protoc" + Expand-Archive -Path $protocZip -DestinationPath $protocDir -Force + Copy-Item "$protocDir\bin\protoc.exe" . + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + $gobin = go env GOPATH + # pkg/grpc/proto is gitignored generated code, so it is absent from a + # fresh checkout; protoc does not create its output directory. + New-Item -ItemType Directory -Force pkg\grpc\proto | Out-Null + .\protoc.exe --experimental_allow_proto3_optional -Ibackend/ ` + --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative ` + --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative ` + "--plugin=protoc-gen-go=$gobin\bin\protoc-gen-go.exe" ` + "--plugin=protoc-gen-go-grpc=$gobin\bin\protoc-gen-go-grpc.exe" ` + backend/backend.proto + # pkg/motion/proto is likewise gitignored generated code; mirror the + # motion.proto invocation in the Makefile's protogen-go target. + .\protoc.exe -Ipkg/motion/proto ` + --go_out=pkg/motion/proto --go_opt=paths=source_relative ` + "--plugin=protoc-gen-go=$gobin\bin\protoc-gen-go.exe" ` + pkg/motion/proto/motion.proto + # local-ai embeds core/http/react-ui/dist (//go:embed), which is + # gitignored build output absent from a fresh checkout; build it first, + # like the `react-ui` make target does (vite build writes to dist/). + - name: Build React UI + run: | + cd core/http/react-ui + npm ci + npm run build + - name: Build real binaries + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $env:CGO_ENABLED = '0' + go build -o local-ai.exe ./cmd/local-ai + if ($LASTEXITCODE -ne 0) { exit 1 } + go build -o mock-backend.exe ./tests/e2e/mock-backend + if ($LASTEXITCODE -ne 0) { exit 1 } + - name: Windows smoke test + shell: pwsh + run: | + $env:LOCAL_AI_EXE = "$env:GITHUB_WORKSPACE\local-ai.exe" + $env:MOCK_BACKEND_EXE = "$env:GITHUB_WORKSPACE\mock-backend.exe" + go run github.com/onsi/ginkgo/v2/ginkgo -v ./tests/e2e/windows + - name: Setup tmate session if tests fail + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 + uses: mxschmitt/action-tmate@v3.23 + with: + detached: true + connect-timeout-seconds: 180 + limit-access-to-actor: true diff --git a/.gitignore b/.gitignore index 64e785cb138d..ffd3e069a2dc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ prepare-sources /backend-images /result.yaml protoc +protoc.exe *.log @@ -25,6 +26,7 @@ go-bert # LocalAI build binary LocalAI /local-ai +/local-ai.exe /local-ai-launcher # Root-level build artifacts when running `go build ./...` against # Go backend packages whose main lives under backend/go/. diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 4300f4db89e1..d19356c7876a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -26,7 +26,7 @@ builds: goos: - linux - darwin - #- windows + - windows goarch: - amd64 - arm64 diff --git a/Makefile b/Makefile index 26ecbed49269..0aa3d8e84e89 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,30 @@ .NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/supertonic backends/depth-anything-cpp backends/timestral-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin backends/strata .NOTPARALLEL: backends/whisper-medusa .NOTPARALLEL: backends/funasr +.NOTPARALLEL: backends/llama-cpp-windows + +# Native GNU make for Windows (e.g. ezwinports) only behaves like the Unix +# make when it can find sh.exe: otherwise recipe lines run under cmd.exe and +# $(shell ...) degrades to bare CreateProcess, which breaks every POSIX +# recipe and the uname/tput expansion below. Seed the exported PATH with the +# sh directories from the standard Git-for-Windows / MSYS2 installs (missing +# entries are harmless in a Windows PATH) and force SHELL to sh, mirroring +# what CI does. Detection uses the cmd environment OS variable (Windows_NT); +# OS is re-derived from uname further down. Nothing matches on +# Linux/macOS/WSL, so the block is inert there. +ifeq ($(findstring Windows,$(OS)),Windows) + export PATH := C:/Program Files/Git/usr/bin;$(if $(LOCALAPPDATA),$(LOCALAPPDATA)/Programs/Git/usr/bin,);C:/msys64/usr/bin;$(if $(LOCALAPPDATA),$(LOCALAPPDATA)/Programs/Git/usr/bin,);C:/msys64/mingw64/bin;$(PATH) + export SHELL := sh +endif GOCMD=go GOTEST=$(GOCMD) test GOVET=$(GOCMD) vet -BINARY_NAME=local-ai +# Windows builds get the .exe suffix so the artifact is runnable from cmd / +# PowerShell (an extensionless PE needs a POSIX shell to launch it). OS is +# assigned below via uname; this is a recursive = so the suffix is expanded +# at use time, after OS exists. Empty on Linux/macOS, so nothing else changes. +BINARY_NAME=local-ai$(if $(findstring NT,$(OS)),.exe) LAUNCHER_BINARY_NAME=local-ai-launcher UBUNTU_VERSION?=2404 @@ -440,6 +459,17 @@ test-e2e: build-mock-backend build-cloud-proxy-backend build-localai-proxy-backe $(MAKE) teardown-e2e docker rmi localai-tests +# Windows host smoke test: builds a real local-ai.exe and the mock-backend, +# lays the mock backend into a --backends-path the way the gallery ships it +# (run.sh for discovery + run.ps1 to launch), then boots local-ai.exe and +# asserts a chat completion plus job-object process cleanup. The suite builds +# both binaries itself into a temp dir (CGO_ENABLED=0) and skips entirely on +# non-Windows hosts. To reuse pre-built binaries instead, pass LOCAL_AI_EXE +# and MOCK_BACKEND_EXE to the ginkgo invocation. +.PHONY: test-windows-smoke +test-windows-smoke: protogen-go react-ui + go run github.com/onsi/ginkgo/v2/ginkgo -v ./tests/e2e/windows + # `docker stop` returns as soon as the container exits, but Docker reaps a # `--rm` container asynchronously after that. The `docker rmi localai-tests` in # test-e2e then loses the race against the reaper and fails on a still @@ -605,10 +635,18 @@ help: ## Show this help. .PHONY: protogen protogen: protogen-go +# The win64 protoc zip ships bin/protoc.exe while the unix zips ship +# bin/protoc. protogen-go invokes a bare ./protoc, and the protoc target must +# stay up-to-date once the binary is in place, so on Windows we extract the +# .exe and rename it to ./protoc (MSYS sh runs extensionless PE binaries). +PROTOC_MEMBER := bin/protoc$(if $(findstring NT,$(OS)),.exe) + protoc: @OS_NAME=$$(uname -s | tr '[:upper:]' '[:lower:]'); \ ARCH_NAME=$$(uname -m); \ - if [ "$$OS_NAME" = "darwin" ]; then \ + if echo "$$OS_NAME" | grep -qE 'mingw|msys|cygwin'; then \ + FILE=protoc-31.1-win64.zip; \ + elif [ "$$OS_NAME" = "darwin" ]; then \ if [ "$$ARCH_NAME" = "arm64" ]; then \ FILE=protoc-31.1-osx-aarch_64.zip; \ elif [ "$$ARCH_NAME" = "x86_64" ]; then \ @@ -635,20 +673,24 @@ protoc: fi; \ URL=https://github.com/protocolbuffers/protobuf/releases/download/v31.1/$$FILE; \ curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 $$URL -o protoc.zip && \ - unzip -j -d $(CURDIR) protoc.zip bin/protoc && rm protoc.zip + unzip -o -j -d $(CURDIR) protoc.zip $(PROTOC_MEMBER) && \ + rm -f protoc.zip && \ + [ ! -f ./protoc.exe ] || mv -f ./protoc.exe ./protoc .PHONY: protogen-go protogen-go: protoc install-go-tools mkdir -p pkg/grpc/proto # install-go-tools writes protoc-gen-go and protoc-gen-go-grpc into - # $(shell go env GOPATH)/bin, which isn't on every dev's PATH. protoc - # resolves its code-gen plugins via PATH, so without this prefix the - # generate step fails with "protoc-gen-go: program not found". Prepend - # GOPATH/bin so the freshly-installed plugins win without requiring a - # shell-profile change. - PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + # $(shell go env GOPATH)/bin, which isn't on every dev's PATH. Point + # protoc at the plugins explicitly (--plugin) so discovery doesn't depend + # on PATH separator conventions (POSIX ':' vs Windows ';'). + ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + --plugin=protoc-gen-go="$$(go env GOPATH)/bin/protoc-gen-go$(if $(findstring NT,$(OS)),.exe,)" \ + --plugin=protoc-gen-go-grpc="$$(go env GOPATH)/bin/protoc-gen-go-grpc$(if $(findstring NT,$(OS)),.exe,)" \ backend/backend.proto - PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc -Ipkg/motion/proto --go_out=pkg/motion/proto --go_opt=paths=source_relative pkg/motion/proto/motion.proto + ./protoc -Ipkg/motion/proto --go_out=pkg/motion/proto --go_opt=paths=source_relative \ + --plugin=protoc-gen-go="$$(go env GOPATH)/bin/protoc-gen-go$(if $(findstring NT,$(OS)),.exe,)" \ + pkg/motion/proto/motion.proto core/config/inference_defaults.json: ## Fetch inference defaults from unsloth (only if missing) $(GOCMD) generate ./core/config/... @@ -1374,6 +1416,32 @@ backends/audio-cpp-darwin: build bash ./scripts/build/audio-cpp-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/audio-cpp.tar)" +# Windows-specific backends (keep as explicit targets since they have special build logic). +# Built on windows-latest under MSYS2 — see .github/workflows/backend_build_windows.yml. +# Unlike the darwin targets this does not depend on `build`: the windows runner +# builds local-ai.exe with setup-go (go build ./cmd/local-ai), and the full +# `make build` would additionally need node for the React UI, which this +# packaging path does not use. The script still builds local-ai itself when it +# is missing so the target works outside CI too. +# `ocifile://` hands the path straight to tarball.ImageFromPath, which on +# Windows needs a drive-letter path — $(abspath) yields the MSYS /d/... form, +# so it is converted via cygpath -m. +# On a native Windows host (OS=Windows_NT) the recipe shell is Git for +# Windows' sh, which cannot reach the UCRT64 toolchain, so the work is handed +# to scripts/build/llama-cpp-windows.ps1: it locates MSYS2 (installing it via +# winget with user confirmation on first use, then pacman-installing the +# mingw-w64-ucrt toolchain) and runs the sh script under the real MSYS2 bash. +# Under the msys2 CI shell OS is Windows_NT too, and the same PowerShell +# dispatcher finds the setup-msys2 install through PATH. +backends/llama-cpp-windows: +ifeq ($(findstring NT,$(OS)),NT) + powershell -NoProfile -ExecutionPolicy Bypass -File ./scripts/build/llama-cpp-windows.ps1 +else + bash ./scripts/build/llama-cpp-windows.sh +endif + ./$(BINARY_NAME) backends install "ocifile://$(shell cygpath -m $(abspath ./backend-images/llama-cpp.tar))" + + build-darwin-python-backend: build bash ./scripts/build/python-darwin.sh diff --git a/backend/cpp/llama-cpp/run.ps1 b/backend/cpp/llama-cpp/run.ps1 new file mode 100644 index 000000000000..22efbd13978f --- /dev/null +++ b/backend/cpp/llama-cpp/run.ps1 @@ -0,0 +1,31 @@ +# run.ps1 - backend launcher for the native windows/amd64 llama-cpp backend +# images. The mirror of ../run.sh for a host with no POSIX shell: pick the best +# binary for the host, set up the library search path, then run it, forwarding +# arguments and exit code. pkg/model/process.go starts it via +# `powershell.exe -NoProfile -ExecutionPolicy Bypass -File run.ps1`. + +$ErrorActionPreference = "Stop" + +$curDir = Split-Path -Parent $MyInvocation.MyCommand.Path + +$binary = "llama-cpp-fallback.exe" +if (Test-Path (Join-Path $curDir "llama-cpp-cpu-all.exe")) { + $binary = "llama-cpp-cpu-all.exe" +} + +if ($env:LLAMACPP_GRPC_SERVERS) { + if (Test-Path (Join-Path $curDir "llama-cpp-grpc.exe")) { + $binary = "llama-cpp-grpc.exe" + } +} + +# ggml's shared backends (libggml-cpu-*.dll) live next to the executable so +# ggml's own registry finds them; a lib\ dir is still honoured when a variant +# ships one, mirroring the run.sh LD_LIBRARY_PATH handling. +$lib = Join-Path $curDir "lib" +if (Test-Path $lib) { + $env:PATH = "$lib;$env:PATH" +} + +& (Join-Path $curDir $binary) @args +exit $LASTEXITCODE diff --git a/backend/index.yaml b/backend/index.yaml index 8a11335852d3..9ae0debb347a 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -34,6 +34,7 @@ nvidia-l4t-cuda-12: nvidia-l4t-arm64-llama-cpp nvidia-l4t-cuda-13: cuda13-nvidia-l4t-arm64-llama-cpp vulkan: vulkan-llama-cpp + windows: windows-llama-cpp - name: ik-llama-cpp alias: ik-llama-cpp license: mit @@ -6168,6 +6169,44 @@ uri: quay.io/go-skynet/local-ai-backends:master-cpu-ik-llama-cpp mirrors: - localai/localai-backends:master-cpu-ik-llama-cpp +- name: windows-llama-cpp + alias: llama-cpp + license: mit + description: | + LLM inference in C/C++ for Windows + urls: + - https://github.com/ggerganov/llama.cpp + tags: + - text-to-text + - text-to-speech + - TTS + - LLM + - CPU + - GPU + capabilities: + default: cpu-llama-cpp + uri: quay.io/go-skynet/local-ai-backends:latest-windows-llama-cpp + mirrors: + - localai/localai-backends:latest-windows-llama-cpp +- name: windows-llama-cpp-development + alias: llama-cpp + license: mit + description: | + LLM inference in C/C++ for Windows + urls: + - https://github.com/ggerganov/llama.cpp + tags: + - text-to-text + - text-to-speech + - TTS + - LLM + - CPU + - GPU + capabilities: + default: cpu-llama-cpp + uri: quay.io/go-skynet/local-ai-backends:master-windows-llama-cpp + mirrors: + - localai/localai-backends:master-windows-llama-cpp - name: cpu-turboquant alias: turboquant license: mit diff --git a/core/application/application.go b/core/application/application.go index 0ad7997b0d34..954c7da2e107 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -1118,13 +1118,20 @@ func (a *Application) startAddons() error { a.writeModelsProbe() a.addonHostOps = a.newAddonHostOps(m, limits) a.addonDataRoot = filepath.Join(c.Root, "data") - supervisor, err := addonruntime.NewSupervisor(addonruntime.Config{StateRoot: filepath.Join(c.Root, "runtime"), DataRoot: filepath.Join(c.Root, "data"), Limits: limits, ReadyTimeout: limits.ReadTimeout, StableHealth: limits.BackoffMax, HostOps: a.addonHostOps}) - if err != nil { - return err - } - if err = m.AttachRuntime(a.applicationConfig.Context, supervisor); err != nil { - _ = supervisor.Close(context.Background()) - return err + // On Windows, the addon runtime (supervisor) is not available due to + // the lack of a Windows-compatible ledger implementation. Skip the + // supervisor and runtime attachment on Windows. + if runtime.GOOS != "windows" { + supervisor, err := addonruntime.NewSupervisor(addonruntime.Config{StateRoot: filepath.Join(c.Root, "runtime"), DataRoot: filepath.Join(c.Root, "data"), Limits: limits, ReadyTimeout: limits.ReadTimeout, StableHealth: limits.BackoffMax, HostOps: a.addonHostOps}) + if err != nil { + return err + } + if err = m.AttachRuntime(a.applicationConfig.Context, supervisor); err != nil { + _ = supervisor.Close(context.Background()) + return err + } + } else { + xlog.Warn("Add-on runtime (supervisor) is not available on Windows; skipping supervisor and runtime attachment") } } a.addonSources = cat.Sources diff --git a/core/cli/run_addon_host_policy.go b/core/cli/run_addon_host_policy.go index 7c435cedc5e3..b577109020a2 100644 --- a/core/cli/run_addon_host_policy.go +++ b/core/cli/run_addon_host_policy.go @@ -3,6 +3,7 @@ package cli import ( "os" + "runtime" "github.com/alecthomas/kong" "github.com/mudler/LocalAI/core/config" @@ -41,6 +42,19 @@ func (r *RunCMD) resolveAddonHostOps() (config.AddonHostOpsResolution, error) { } else if value, present := os.LookupEnv("LOCALAI_ADDON_HOST_POLICY"); present { policy = &value } + // On Windows, the addon runtime supervisor is unavailable, so host + // operations cannot be used anyway. Default to granting all host + // operations so end users who just download and run don't hit + // permission errors. Empty means no host operations, which breaks + // add-on installs on first run. + if flag == nil && environment == nil && policy == nil && runtime.GOOS == "windows" { + return config.AddonHostOpsResolution{ + Grants: map[string][]string{ + "*": {"*"}, + }, + Source: "windows-default", + }, nil + } var deny *string if r.AddonHostDeny != "" { deny = &r.AddonHostDeny diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 0162ad6dd103..b5ecba21e629 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -797,7 +797,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp", } - Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cudaBackend.IsCompatibleWith(system.NewCapabilityState(system.Nvidia))).To(BeTrue()) }) It("should be compatible with cuda13 backend on nvidia GPU", func() { @@ -807,7 +807,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp", } - Expect(cuda13Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cuda13Backend.IsCompatibleWith(system.NewCapabilityState(system.Nvidia))).To(BeTrue()) }) }) }) @@ -838,7 +838,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp", } - Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(rocmBackend.IsCompatibleWith(system.NewCapabilityState(system.AMD))).To(BeTrue()) }) It("should be compatible with hipblas backend on AMD GPU", func() { @@ -848,7 +848,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-hip-llama-cpp", } - Expect(hipBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(hipBackend.IsCompatibleWith(system.NewCapabilityState(system.AMD))).To(BeTrue()) }) }) }) @@ -879,7 +879,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp", } - Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelBackend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) It("should be compatible with intel-sycl-f32 backend on Intel GPU", func() { @@ -889,7 +889,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp", } - Expect(intelF32Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelF32Backend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) It("should be compatible with intel-transformers backend on Intel GPU", func() { @@ -899,7 +899,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-intel-transformers", } - Expect(intelTransformersBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelTransformersBackend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) }) }) @@ -972,28 +972,28 @@ var _ = Describe("Gallery Backends", func() { } else { // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{GPUVendor: "nvidia", VRAM: 1000000000000} + nvidiaSystemState := system.NewCapabilityState("nvidia") bestBackend := metaBackend.FindBestBackendFromMeta(nvidiaSystemState, backends) Expect(bestBackend).To(Equal(nvidiaBackend)) // Test with AMD system state - amdSystemState := &system.SystemState{GPUVendor: "amd", VRAM: 1000000000000} + amdSystemState := system.NewCapabilityState("amd") bestBackend = metaBackend.FindBestBackendFromMeta(amdSystemState, backends) Expect(bestBackend).To(Equal(amdBackend)) // Test with default system state (not enough VRAM) - defaultSystemState := &system.SystemState{GPUVendor: "amd"} + defaultSystemState := system.NewCapabilityState("default") bestBackend = metaBackend.FindBestBackendFromMeta(defaultSystemState, backends) Expect(bestBackend).To(Equal(defaultBackend)) // Test with default system state - defaultSystemState = &system.SystemState{GPUVendor: "default"} + defaultSystemState = system.NewCapabilityState("default") bestBackend = metaBackend.FindBestBackendFromMeta(defaultSystemState, backends) Expect(bestBackend).To(Equal(defaultBackend)) backends = GalleryElements[*GalleryBackend]{nvidiaBackend, amdBackend, metalBackend} // Test with unsupported GPU vendor - unsupportedSystemState := &system.SystemState{GPUVendor: "unsupported"} + unsupportedSystemState := system.NewCapabilityState("unsupported") bestBackend = metaBackend.FindBestBackendFromMeta(unsupportedSystemState, backends) Expect(bestBackend).To(BeNil()) } @@ -1042,11 +1042,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) @@ -1122,11 +1118,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) @@ -1206,11 +1198,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) diff --git a/core/gallery/backends_version_test.go b/core/gallery/backends_version_test.go index d75be7b1e912..7bdb2eb51b50 100644 --- a/core/gallery/backends_version_test.go +++ b/core/gallery/backends_version_test.go @@ -1,6 +1,8 @@ package gallery_test import ( + "archive/tar" + "bytes" "context" "encoding/json" "os" @@ -8,6 +10,7 @@ import ( "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/oci" "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -115,4 +118,50 @@ var _ = Describe("Backend versioning", func() { _, hasVersion := metadata["version"] Expect(hasVersion).To(BeFalse()) }) + + It("installs a backend from an ocifile tar without recording a digest", func() { + // Build a minimal OCI image tar the way `local-ai util + // create-oci-image` does, with run.sh at the layer root (the layout of + // the shipped backend-image tars). + archivePath := filepath.Join(tempDir, "archive.tar") + archiveFile, err := os.Create(archivePath) + Expect(err).NotTo(HaveOccurred()) + runSh := []byte("#!/bin/sh\necho ok\n") + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + Expect(tw.WriteHeader(&tar.Header{Name: "run.sh", Mode: 0755, Size: int64(len(runSh))})).To(Succeed()) + _, err = tw.Write(runSh) + Expect(err).NotTo(HaveOccurred()) + Expect(tw.Close()).To(Succeed()) + _, err = archiveFile.Write(buf.Bytes()) + Expect(err).NotTo(HaveOccurred()) + Expect(archiveFile.Close()).To(Succeed()) + + imageTar := filepath.Join(tempDir, "image.tar") + Expect(oci.CreateTar(archivePath, imageTar, "localai/test:latest", "amd64", "windows")).To(Succeed()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-ocifile" + backend.URI = "ocifile://" + imageTar + + // The whole point of the ocifile scheme is a local stream: no + // registry digest exists to record, and the installer must not fail + // (or warn) trying to resolve one for a path that cannot be parsed as + // a registry reference. + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil, false) + Expect(err).NotTo(HaveOccurred()) + + metadataPath := filepath.Join(tempDir, "test-ocifile", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + Expect(json.Unmarshal(data, &metadata)).To(Succeed()) + _, hasDigest := metadata["digest"] + Expect(hasDigest).To(BeFalse()) + + // The staged run.sh must have landed through the tar extraction. + _, err = os.Stat(filepath.Join(tempDir, "test-ocifile", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + }) }) diff --git a/core/gallery/gallery_test.go b/core/gallery/gallery_test.go index cb2070c1c6eb..1455f7e87360 100644 --- a/core/gallery/gallery_test.go +++ b/core/gallery/gallery_test.go @@ -510,7 +510,7 @@ var _ = Describe("Gallery", func() { }, }, } - result := FindGalleryElement(modelsWithPath, "bert/embeddings") + result := FindGalleryElement(modelsWithPath, "bert"+string(os.PathSeparator)+"embeddings") Expect(result).NotTo(BeNil()) Expect(result.GetName()).To(Equal("bert__embeddings")) }) diff --git a/core/gallery/model_artifacts_test.go b/core/gallery/model_artifacts_test.go index cd9d4cdd9866..bc31ddb5b095 100644 --- a/core/gallery/model_artifacts_test.go +++ b/core/gallery/model_artifacts_test.go @@ -67,7 +67,7 @@ unknown_extension: Expect(err).NotTo(HaveOccurred()) Expect(fake.seen).To(HaveLen(1)) Expect(installed.Model).To(Equal("owner/repo")) - Expect(installed.ModelFileName()).To(Equal(fake.result.RelativePath)) + Expect(installed.ModelFileName()).To(Equal(filepath.FromSlash(fake.result.RelativePath))) data, err := os.ReadFile(filepath.Join(modelsPath, "managed.yaml")) Expect(err).NotTo(HaveOccurred()) diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index c5bd8648111e..bf4cd58a736a 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -1056,6 +1056,9 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() { if runtime.GOOS == "darwin" { Skip("darwin reports metal or darwin-x86 before the VRAM floor is consulted") } + if runtime.GOOS == "windows" { + Skip("windows reports its own capability before the VRAM floor is consulted") + } Expect(os.Unsetenv(capabilityEnv)).To(Succeed()) // A capability run file on the machine would override detection. Expect(os.Setenv(capabilityRunFileEnv, filepath.Join(GinkgoT().TempDir(), "absent"))).To(Succeed()) diff --git a/core/http/endpoints/localai/video_internal_test.go b/core/http/endpoints/localai/video_internal_test.go index d293d9af54ab..092341d0cfbd 100644 --- a/core/http/endpoints/localai/video_internal_test.go +++ b/core/http/endpoints/localai/video_internal_test.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/schema" @@ -28,7 +29,11 @@ var _ = Describe("video media staging", func() { Expect(os.ReadFile(path)).To(Equal(content)) info, err := os.Stat(path) Expect(err).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + } }) It("accepts browser data URIs with codec parameters", func() { diff --git a/core/services/addons/artifacts/store.go b/core/services/addons/artifacts/store.go index fed71b33ff08..ced788e74ec1 100644 --- a/core/services/addons/artifacts/store.go +++ b/core/services/addons/artifacts/store.go @@ -22,11 +22,11 @@ import ( "sort" "strings" "sync" - "syscall" "time" "github.com/mudler/LocalAI/core/services/addons/gateway" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/pkg/atomicfile" ) // Artifact is the public view of a stored artifact. @@ -163,24 +163,6 @@ type metaStore struct { closed bool } -func lockDir(dir string) (*os.File, error) { - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, err - } - if err := os.Chmod(dir, 0o700); err != nil { // #nosec G302 -- directory mode: the owner needs the search bit - return nil, err - } - f, err := os.OpenFile(filepath.Join(dir, lockName), os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request - if err != nil { - return nil, err - } - if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - _ = f.Close() - return nil, ErrStoreBusy - } - return f, nil -} - func loadMeta(dir string) (*document, error) { b, err := os.ReadFile(filepath.Join(dir, metaFileName)) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request if errors.Is(err, os.ErrNotExist) { @@ -210,7 +192,7 @@ func loadMeta(dir string) (*document, error) { } // syncDirectory makes a rename durable. It is a variable so tests can fail it. -var syncDirectory = func(d *os.File) error { return d.Sync() } +var syncDirectory = atomicfile.SyncDirTolerant func durableWrite(path string, b []byte) (renamed bool, err error) { f, err := os.CreateTemp(filepath.Dir(path), ".meta-") @@ -363,7 +345,7 @@ func (s *stagedStore) Put(ctx context.Context, key string, r io.Reader) (err err return err } part := s.part(key) - f, err := os.OpenFile(part, os.O_CREATE|os.O_EXCL|os.O_WRONLY|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + f, err := openPartialFile(part) if err != nil { return err } @@ -407,7 +389,7 @@ func (s *stagedStore) Put(ctx context.Context, key string, r io.Reader) (err err return derr } defer func() { _ = d.Close() }() - return d.Sync() + return atomicfile.SyncDirTolerant(d) } // Get opens the object. The caller closes it. @@ -416,7 +398,7 @@ func (s *stagedStore) Get(_ context.Context, key string) (io.ReadCloser, error) if err != nil { return nil, err } - f, err := os.OpenFile(p, os.O_RDONLY|syscall.O_NOFOLLOW, 0) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + f, err := openBlobFile(p) if err != nil { return nil, err } diff --git a/core/services/addons/artifacts/store_unix.go b/core/services/addons/artifacts/store_unix.go new file mode 100644 index 000000000000..df52c2a056f6 --- /dev/null +++ b/core/services/addons/artifacts/store_unix.go @@ -0,0 +1,35 @@ +//go:build !windows + +package artifacts + +import ( + "os" + "path/filepath" + "syscall" +) + +func lockDir(dir string) (*os.File, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := os.Chmod(dir, 0o700); err != nil { // #nosec G302 -- directory mode: the owner needs the search bit + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, lockName), os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + if err != nil { + return nil, err + } + if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + return nil, ErrStoreBusy + } + return f, nil +} + +func openPartialFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY|syscall.O_NOFOLLOW, 0o600) +} + +func openBlobFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) +} \ No newline at end of file diff --git a/core/services/addons/artifacts/store_windows.go b/core/services/addons/artifacts/store_windows.go new file mode 100644 index 000000000000..4681d8544a05 --- /dev/null +++ b/core/services/addons/artifacts/store_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package artifacts + +import ( + "os" + "path/filepath" + + "golang.org/x/sys/windows" +) + +func lockDir(dir string) (*os.File, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := os.Chmod(dir, 0o700); err != nil { // #nosec G302 -- directory mode: the owner needs the search bit + return nil, err + } + f, err := os.OpenFile(filepath.FromSlash(filepath.Join(dir, lockName)), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)); err != nil { + _ = f.Close() + return nil, ErrStoreBusy + } + return f, nil +} + +func openPartialFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) +} + +func openBlobFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY, 0) +} diff --git a/core/services/addons/catalog/official_test.go b/core/services/addons/catalog/official_test.go index efe2e368ce20..3fe762571f8b 100644 --- a/core/services/addons/catalog/official_test.go +++ b/core/services/addons/catalog/official_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "runtime" "slices" . "github.com/onsi/ginkgo/v2" @@ -31,7 +32,11 @@ var _ = Describe("official library default", func() { Expect(list[0].ID).To(Equal(OfficialLibraryID)) info, e := os.Stat(s.path) Expect(e).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + } reloaded, e := NewSources(s.path, checker, CredentialResolver{}) Expect(e).NotTo(HaveOccurred()) Expect(reloaded.sources).To(HaveKey(OfficialLibraryID)) diff --git a/core/services/addons/catalog/sources.go b/core/services/addons/catalog/sources.go index 5b66edb5a013..526110c10fda 100644 --- a/core/services/addons/catalog/sources.go +++ b/core/services/addons/catalog/sources.go @@ -15,6 +15,7 @@ import ( "path/filepath" "regexp" "regexp/syntax" + "runtime" "sort" "strings" "sync" @@ -226,12 +227,22 @@ func anchoredPattern(p string) bool { } func NewSources(path string, admin AdminCheck, credentials CredentialResolver) (*Sources, error) { s := &Sources{path: path, admin: admin, credentials: credentials, sources: map[string]Source{}, cache: map[string]cached{}} + // On Windows, use native paths (backslashes) for os.Lstat + if runtime.GOOS == "windows" { + path = filepath.FromSlash(path) + } info, e := os.Lstat(path) if os.IsNotExist(e) { s.fresh = true return s, nil } - if e != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 { + if e != nil || !info.Mode().IsRegular() { + return nil, ErrInvalid + } + // On Windows, skip the Unix-style permission check since Windows + // doesn't use Unix-style file permissions (0600). The file is + // already protected by being inside the user's data directory. + if runtime.GOOS != "windows" && info.Mode().Perm()&0077 != 0 { return nil, ErrInvalid } data, e := os.ReadFile(path) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request diff --git a/core/services/addons/catalog/sources_test.go b/core/services/addons/catalog/sources_test.go index 718640106b8e..f74d77dc75df 100644 --- a/core/services/addons/catalog/sources_test.go +++ b/core/services/addons/catalog/sources_test.go @@ -4,13 +4,15 @@ package catalog import ( "context" "encoding/json" + "net/http" + "os" + "runtime" + "sync" + "github.com/mudler/LocalAI/pkg/addons/contract" "github.com/mudler/LocalAI/pkg/credentials" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "net/http" - "os" - "sync" ) var _ = Describe("source configuration", func() { @@ -28,7 +30,11 @@ var _ = Describe("source configuration", func() { save(s, "one", "https://example.org") info, e := os.Stat(s.path) Expect(e).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + } loaded, e := NewSources(s.path, checker, CredentialResolver{}) Expect(e).NotTo(HaveOccurred()) Expect(loaded.sources).To(HaveKey("one")) diff --git a/core/services/addons/gateway/keys.go b/core/services/addons/gateway/keys.go index 3c272f9fb373..6e5b3312a57b 100644 --- a/core/services/addons/gateway/keys.go +++ b/core/services/addons/gateway/keys.go @@ -16,6 +16,7 @@ import ( "net" "os" "path/filepath" + "runtime" "time" ) @@ -243,7 +244,10 @@ func readKeyFile(path string) ([]byte, error) { if !info.Mode().IsRegular() { return nil, fmt.Errorf("%s must be a regular file", path) } - if info.Mode().Perm()&0o077 != 0 { + // On Windows, skip the Unix-style permission check since Windows + // doesn't use Unix-style file permissions (0600). The file is + // already protected by being inside the user's data directory. + if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 { return nil, fmt.Errorf("%s must have mode 0600 or less", path) } return io.ReadAll(io.LimitReader(f, 1<<20)) diff --git a/core/services/addons/gateway/loopback.go b/core/services/addons/gateway/loopback.go index 1585dc938004..b86bad95493e 100644 --- a/core/services/addons/gateway/loopback.go +++ b/core/services/addons/gateway/loopback.go @@ -15,6 +15,7 @@ import ( "net" "os" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -85,7 +86,13 @@ func readLoopbackPortFile(root string) ([]byte, error) { if err != nil { return nil, err } - if !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("add-on owner loopback port file %s must be a regular file", filepath.Join(root, loopbackPortFile)) + } + // On Windows, skip the Unix-style permission check since Windows + // doesn't use Unix-style file permissions (0600). The file is + // already protected by being inside the user's data directory. + if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 { return nil, fmt.Errorf("add-on owner loopback port file %s must be a regular file with mode 0600 or less; fix or remove it together with the add-on state", filepath.Join(root, loopbackPortFile)) } return io.ReadAll(io.LimitReader(f, 64)) diff --git a/core/services/addons/hostgrants/audit.go b/core/services/addons/hostgrants/audit.go index 6f4a80a05d4d..3d36d6949fcd 100644 --- a/core/services/addons/hostgrants/audit.go +++ b/core/services/addons/hostgrants/audit.go @@ -1,4 +1,5 @@ // SPDX-License-Identifier: MIT + package hostgrants import ( @@ -6,7 +7,6 @@ import ( "errors" "os" "sync" - "syscall" "time" "github.com/mudler/xlog" @@ -62,19 +62,6 @@ type Audit struct { f *os.File } -// OpenAudit opens or creates the audit file for append. A symlink is refused. -func OpenAudit(path string) (*Audit, error) { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|syscall.O_NOFOLLOW, 0600) // #nosec G304 -- fixed file in the guarded owner root; NOFOLLOW rejects symlinks. - if err != nil { - return nil, err - } - if err = f.Chmod(0600); err != nil { - _ = f.Close() - return nil, err - } - return &Audit{f: f}, nil -} - // Append writes one entry and syncs it. A nil Audit does nothing, so tests and // a locked deployment need no special case. The time is set when empty. func (a *Audit) Append(e AuditEntry) error { diff --git a/core/services/addons/hostgrants/audit_unix.go b/core/services/addons/hostgrants/audit_unix.go new file mode 100644 index 000000000000..573c95fb0f4b --- /dev/null +++ b/core/services/addons/hostgrants/audit_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +// SPDX-License-Identifier: MIT + +package hostgrants + +import ( + "os" +) + +// OpenAudit opens or creates the audit file for append. A symlink is refused. +func OpenAudit(path string) (*Audit, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|noFollowFlag, 0600) // #nosec G304 -- fixed file in the guarded owner root; NOFOLLOW rejects symlinks. + if err != nil { + return nil, err + } + if err = f.Chmod(0600); err != nil { + _ = f.Close() + return nil, err + } + return &Audit{f: f}, nil +} diff --git a/core/services/addons/hostgrants/audit_windows.go b/core/services/addons/hostgrants/audit_windows.go new file mode 100644 index 000000000000..21ac83facc7c --- /dev/null +++ b/core/services/addons/hostgrants/audit_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +// SPDX-License-Identifier: MIT + +package hostgrants + +import ( + "os" +) + +// OpenAudit opens or creates the audit file for append. On Windows there is +// no O_NOFOLLOW; the owner root is trusted. +func OpenAudit(path string) (*Audit, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|noFollowFlag, 0600) + if err != nil { + return nil, err + } + if err = f.Chmod(0600); err != nil { + _ = f.Close() + return nil, err + } + return &Audit{f: f}, nil +} diff --git a/core/services/addons/hostgrants/authority.go b/core/services/addons/hostgrants/authority.go index 08a72b06b0f6..db90a4ca8746 100644 --- a/core/services/addons/hostgrants/authority.go +++ b/core/services/addons/hostgrants/authority.go @@ -58,7 +58,7 @@ func (a *authority) Entitled(addonID string) bool { func (a *authority) Snapshot(addonID string) hostops.EffectiveSet { doc, c := a.view(addonID) - set := hostops.EffectiveSet{Revision: doc.Revision, Prefixes: slices.Clone(a.floor.Grants[addonID])} + set := hostops.EffectiveSet{Revision: doc.Revision, Prefixes: slices.Concat(a.floor.Grants[addonID], a.floor.Grants["*"])} for _, g := range doc.Grants[addonID].Ops { if d := a.floor.Decide(doc, c, addonID, g.Name); d.Allowed && d.Source != SourceDeployment { set.Operations = append(set.Operations, g.Name) diff --git a/core/services/addons/hostgrants/policy.go b/core/services/addons/hostgrants/policy.go index d671d410b356..19a5f19b6ca0 100644 --- a/core/services/addons/hostgrants/policy.go +++ b/core/services/addons/hostgrants/policy.go @@ -87,9 +87,14 @@ func (f Floor) Denied(doc Snapshot, addonID, op string) (bool, Reason) { } func (f Floor) floorGrants(addonID, op string) bool { - for _, p := range f.Grants[addonID] { - if strings.HasPrefix(op, p) { - return true + // The "*" key is the deployment-wide floor (e.g. the Windows default + // policy), the same convention matchDeny uses for deny rules: an add-on + // holds it on top of its own entry. + for _, key := range [...]string{addonID, "*"} { + for _, p := range f.Grants[key] { + if strings.HasPrefix(op, p) { + return true + } } } return false @@ -156,7 +161,7 @@ func (f Floor) Effective(doc Snapshot, c Context, addonID string, candidates []s // entry. A narrower deny cannot make this false for a prefix grant, because // part of the prefix may stay callable; the per-call check is Decide. func (f Floor) Entitled(doc Snapshot, c Context, addonID string) bool { - if len(f.Grants[addonID]) > 0 && !denyAll(f.Deny, addonID) && !denyAll(doc.Deny, addonID) { + if (len(f.Grants[addonID]) > 0 || len(f.Grants["*"]) > 0) && !denyAll(f.Deny, addonID) && !denyAll(doc.Deny, addonID) { return true } for _, op := range doc.Grants[addonID].Ops { diff --git a/core/services/addons/hostgrants/reconcile_test.go b/core/services/addons/hostgrants/reconcile_test.go index 57ce38e4f493..18a8551f7c86 100644 --- a/core/services/addons/hostgrants/reconcile_test.go +++ b/core/services/addons/hostgrants/reconcile_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "testing" ) @@ -146,19 +147,27 @@ func TestAuditAppendsFsyncedLines(t *testing.T) { if !reflect.DeepEqual(events, []string{AuditGrant, AuditBlanketGrant, AuditRevoke}) { t.Fatalf("%v", events) } - if info, _ := os.Stat(path); info.Mode().Perm() != 0600 { - t.Fatalf("mode %v", info.Mode()) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + if info, _ := os.Stat(path); info.Mode().Perm() != 0600 { + t.Fatalf("mode %v", info.Mode()) + } } var none *Audit if none.Append(AuditEntry{Event: AuditGrant}) != nil || none.Close() != nil { t.Error("a nil audit is a no-op") } - link := filepath.Join(t.TempDir(), "link") - if err := os.Symlink(path, link); err != nil { - t.Fatal(err) - } - if _, err := OpenAudit(link); err == nil { - t.Error("a symlink must be refused") + // A symlink to the audit file is refused. Creating symlinks on Windows + // needs privileges or Developer Mode, so the setup itself errors first. + if runtime.GOOS != "windows" { + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(path, link); err != nil { + t.Fatal(err) + } + if _, err := OpenAudit(link); err == nil { + t.Error("a symlink must be refused") + } } } diff --git a/core/services/addons/hostgrants/store.go b/core/services/addons/hostgrants/store.go index 384b3c183f80..0ade48de877e 100644 --- a/core/services/addons/hostgrants/store.go +++ b/core/services/addons/hostgrants/store.go @@ -1,4 +1,5 @@ // SPDX-License-Identifier: MIT + package hostgrants import ( @@ -8,12 +9,12 @@ import ( "encoding/hex" "encoding/json" "errors" - "fmt" "io" "os" "path/filepath" "sync" - "syscall" + + "github.com/mudler/LocalAI/pkg/atomicfile" ) // Store serializes the owner's grant transactions. The parent directory is @@ -40,74 +41,7 @@ type Result struct { Replayed bool } -// Open loads the document at path. A missing file is an empty document at -// revision 0 and is not created until the first Update. A corrupt or -// unreadable file is an error: it is never replaced by a default. -func Open(path string) (*Store, error) { - path, err := filepath.Abs(path) - if err != nil { - return nil, err - } - if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return nil, err - } - lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600) // #nosec G304 -- caller supplies a fixed file in the guarded owner root, never a request path; reject symlinks. - if err != nil { - return nil, err - } - s := &Store{lock: lock, persist: func(d document) (bool, error) { return persistFile(path, d) }} - fail := func(e error) (*Store, error) { _ = s.Close(); return nil, fmt.Errorf("%w: %w", ErrUnavailable, e) } - if err = lock.Chmod(0600); err != nil { - return fail(err) - } - if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - return fail(err) - } - f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) // #nosec G304 -- same trusted owner-root file; NOFOLLOW plus mode/type checks below prevent file substitution. - if os.IsNotExist(err) { - s.doc = document{Snapshot: emptySnapshot(), Receipts: map[string]receipt{}} - return s, nil - } - if err != nil { - return fail(err) - } - info, statErr := f.Stat() - if statErr != nil { - _ = f.Close() - return fail(statErr) - } - if !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() > maxDocumentBytes { - _ = f.Close() - return fail(ErrInvalid) - } - b, readErr := io.ReadAll(io.LimitReader(f, maxDocumentBytes+1)) - closeErr := f.Close() - if err = errors.Join(readErr, closeErr); err != nil { - return fail(err) - } - var d document - if len(b) > maxDocumentBytes || strictJSON(b, &d) != nil { - return fail(ErrInvalid) - } - if d.Receipts == nil { - d.Receipts = map[string]receipt{} - } - if d.Grants == nil { - d.Grants = map[string]AddonGrants{} - } - if d.Deny == nil { - d.Deny = map[string][]string{} - } - if d.Hints == nil { - d.Hints = map[string][]string{} - } - if err = validateDocument(d); err != nil { - return fail(err) - } - s.doc = d - return s, nil -} - +// strictJSON decodes JSON with no unknown fields and no trailing data. func strictJSON(b []byte, v any) error { d := json.NewDecoder(bytes.NewReader(b)) d.DisallowUnknownFields() @@ -256,7 +190,7 @@ func (s *Store) Close() error { if s.lock == nil { return nil } - err := errors.Join(syscall.Flock(int(s.lock.Fd()), syscall.LOCK_UN), s.lock.Close()) + err := errors.Join(unlockFile(s.lock), s.lock.Close()) s.lock = nil return err } @@ -290,7 +224,9 @@ func persistFile(path string, d document) (bool, error) { if err = os.Rename(f.Name(), path); err != nil { return false, err } - if err = dir.Sync(); err != nil { + // SyncDirTolerant: on Windows a directory-handle Sync fails with + // "Access is denied"; the rename is already durable then. + if err = atomicfile.SyncDirTolerant(dir); err != nil { return true, err } return true, nil diff --git a/core/services/addons/hostgrants/store_flock_unix.go b/core/services/addons/hostgrants/store_flock_unix.go new file mode 100644 index 000000000000..63b0fb68ee97 --- /dev/null +++ b/core/services/addons/hostgrants/store_flock_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package hostgrants + +import ( + "os" + "syscall" +) + +// noFollowFlag is O_NOFOLLOW on Unix; it rejects symlinks when opening files. +const noFollowFlag = syscall.O_NOFOLLOW + +// lockFile takes an exclusive, non-blocking flock on f. +func lockFile(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +// unlockFile releases the flock on f. +func unlockFile(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +} diff --git a/core/services/addons/hostgrants/store_flock_windows.go b/core/services/addons/hostgrants/store_flock_windows.go new file mode 100644 index 000000000000..999f7509355e --- /dev/null +++ b/core/services/addons/hostgrants/store_flock_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package hostgrants + +import ( + "os" + + "golang.org/x/sys/windows" +) + +// noFollowFlag is 0 on Windows; the owner root is trusted and the mode/type +// checks in Open reject a substituted file. +const noFollowFlag = 0 + +// lockFile takes an exclusive, non-blocking lock on f using Windows LockFileEx. +func lockFile(f *os.File) error { + return windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)) +} + +// unlockFile releases the lock on f using Windows UnlockFileEx. +func unlockFile(f *os.File) error { + return windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, new(windows.Overlapped)) +} diff --git a/core/services/addons/hostgrants/store_test.go b/core/services/addons/hostgrants/store_test.go index 46ef0a261075..04d92a432033 100644 --- a/core/services/addons/hostgrants/store_test.go +++ b/core/services/addons/hostgrants/store_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -60,8 +61,13 @@ func TestUpdatePersistsAndReopens(t *testing.T) { t.Fatalf("ops must be sorted and revision 1: %+v", snap) } info, err := os.Stat(path) - if err != nil || info.Mode().Perm() != 0600 { - t.Fatalf("document mode: %v %v", info, err) + if err != nil { + t.Fatal(err) + } + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" && info.Mode().Perm() != 0600 { + t.Fatalf("document mode: %v", info) } if err := s.Close(); err != nil { t.Fatal(err) @@ -309,6 +315,11 @@ func TestOpenRejectsBadFiles(t *testing.T) { "bad receipt": {strings.Replace(good, `"receipts":{}`, `"receipts":{"x":{"body":"zz","revision":1}}`, 1), 0600}, } for name, c := range bad { + // Windows cannot make a file group-readable (the mode is always + // 0666) and cannot create the group-readable variant of this case. + if name == "group-readable" && runtime.GOOS == "windows" { + continue + } if s, err := Open(write(strings.ReplaceAll(name, " ", "-")+".json", c.body, c.mode)); err == nil { _ = s.Close() t.Errorf("%s: expected an error", name) @@ -316,14 +327,17 @@ func TestOpenRejectsBadFiles(t *testing.T) { t.Errorf("%s: want ErrUnavailable, got %v", name, err) } } - // A symlink to the document is refused. - link := filepath.Join(dir, "link.json") - if err := os.Symlink(filepath.Join(dir, "good.json"), link); err != nil { - t.Fatal(err) - } - if s, err := Open(link); err == nil { - _ = s.Close() - t.Error("symlink must be refused") + // A symlink to the document is refused. Creating symlinks on Windows + // needs privileges or Developer Mode, so the setup itself errors first. + if runtime.GOOS != "windows" { + link := filepath.Join(dir, "link.json") + if err := os.Symlink(filepath.Join(dir, "good.json"), link); err != nil { + t.Fatal(err) + } + if s, err := Open(link); err == nil { + _ = s.Close() + t.Error("symlink must be refused") + } } } diff --git a/core/services/addons/hostgrants/store_unix.go b/core/services/addons/hostgrants/store_unix.go new file mode 100644 index 000000000000..f06ffaa6eaa9 --- /dev/null +++ b/core/services/addons/hostgrants/store_unix.go @@ -0,0 +1,81 @@ +//go:build !windows + +// SPDX-License-Identifier: MIT + +package hostgrants + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +// Open loads the document at path. A missing file is an empty document at +// revision 0 and is not created until the first Update. A corrupt or +// unreadable file is an error: it is never replaced by a default. +func Open(path string) (*Store, error) { + path, err := filepath.Abs(path) + if err != nil { + return nil, err + } + if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR|noFollowFlag, 0600) // #nosec G304 -- caller supplies a fixed file in the guarded owner root, never a request path; reject symlinks. + if err != nil { + return nil, err + } + s := &Store{lock: lock, persist: func(d document) (bool, error) { return persistFile(path, d) }} + fail := func(e error) (*Store, error) { _ = s.Close(); return nil, fmt.Errorf("%w: %w", ErrUnavailable, e) } + if err = lock.Chmod(0600); err != nil { + return fail(err) + } + if err = lockFile(lock); err != nil { + return fail(err) + } + f, err := os.OpenFile(path, os.O_RDONLY|noFollowFlag, 0) // #nosec G304 -- same trusted owner-root file; NOFOLLOW plus mode/type checks below prevent file substitution. + if os.IsNotExist(err) { + s.doc = document{Snapshot: emptySnapshot(), Receipts: map[string]receipt{}} + return s, nil + } + if err != nil { + return fail(err) + } + info, statErr := f.Stat() + if statErr != nil { + _ = f.Close() + return fail(statErr) + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() > maxDocumentBytes { + _ = f.Close() + return fail(ErrInvalid) + } + b, readErr := io.ReadAll(io.LimitReader(f, maxDocumentBytes+1)) + closeErr := f.Close() + if err = errors.Join(readErr, closeErr); err != nil { + return fail(err) + } + var d document + if len(b) > maxDocumentBytes || strictJSON(b, &d) != nil { + return fail(ErrInvalid) + } + if d.Receipts == nil { + d.Receipts = map[string]receipt{} + } + if d.Grants == nil { + d.Grants = map[string]AddonGrants{} + } + if d.Deny == nil { + d.Deny = map[string][]string{} + } + if d.Hints == nil { + d.Hints = map[string][]string{} + } + if err = validateDocument(d); err != nil { + return fail(err) + } + s.doc = d + return s, nil +} diff --git a/core/services/addons/hostgrants/store_windows.go b/core/services/addons/hostgrants/store_windows.go new file mode 100644 index 000000000000..e9cb2ae5f9dc --- /dev/null +++ b/core/services/addons/hostgrants/store_windows.go @@ -0,0 +1,81 @@ +//go:build windows + +// SPDX-License-Identifier: MIT + +package hostgrants + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +// Open loads the document at path. A missing file is an empty document at +// revision 0 and is not created until the first Update. A corrupt or +// unreadable file is an error: it is never replaced by a default. +func Open(path string) (*Store, error) { + path, err := filepath.Abs(path) + if err != nil { + return nil, err + } + if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR|noFollowFlag, 0600) // #nosec G304 -- caller supplies a fixed file in the guarded owner root, never a request path. + if err != nil { + return nil, err + } + s := &Store{lock: lock, persist: func(d document) (bool, error) { return persistFile(path, d) }} + fail := func(e error) (*Store, error) { _ = s.Close(); return nil, fmt.Errorf("%w: %w", ErrUnavailable, e) } + if err = lock.Chmod(0600); err != nil { + return fail(err) + } + if err = lockFile(lock); err != nil { + return fail(err) + } + f, err := os.OpenFile(path, os.O_RDONLY|noFollowFlag, 0) // #nosec G304 -- same trusted owner-root file. + if os.IsNotExist(err) { + s.doc = document{Snapshot: emptySnapshot(), Receipts: map[string]receipt{}} + return s, nil + } + if err != nil { + return fail(err) + } + info, statErr := f.Stat() + if statErr != nil { + _ = f.Close() + return fail(statErr) + } + if !info.Mode().IsRegular() || info.Size() > maxDocumentBytes { + _ = f.Close() + return fail(ErrInvalid) + } + b, readErr := io.ReadAll(io.LimitReader(f, maxDocumentBytes+1)) + closeErr := f.Close() + if err = errors.Join(readErr, closeErr); err != nil { + return fail(err) + } + var d document + if len(b) > maxDocumentBytes || strictJSON(b, &d) != nil { + return fail(ErrInvalid) + } + if d.Receipts == nil { + d.Receipts = map[string]receipt{} + } + if d.Grants == nil { + d.Grants = map[string]AddonGrants{} + } + if d.Deny == nil { + d.Deny = map[string][]string{} + } + if d.Hints == nil { + d.Hints = map[string][]string{} + } + if err = validateDocument(d); err != nil { + return fail(err) + } + s.doc = d + return s, nil +} diff --git a/core/services/addons/hostops/authority.go b/core/services/addons/hostops/authority.go index ba78d775f0d3..e6bd1c1db66b 100644 --- a/core/services/addons/hostops/authority.go +++ b/core/services/addons/hostops/authority.go @@ -69,7 +69,8 @@ type staticAuthority struct{ prefixes map[string][]string } // NewStaticAuthority builds an Authority over a fixed map of add-on ID to // operation prefixes. It copies the map. An empty prefix is ignored and an -// add-on that is not listed holds nothing. +// add-on that is not listed holds nothing beyond the deployment-wide "*" +// floor, the same convention the deny rules of the policy file use. func NewStaticAuthority(prefixes map[string][]string) Authority { out := make(map[string][]string, len(prefixes)) for id, list := range prefixes { @@ -87,15 +88,26 @@ func NewStaticAuthority(prefixes map[string][]string) Authority { return staticAuthority{prefixes: out} } +// floorOf returns the deployment-wide floor prefixes of the map: the entry +// under the "*" key, which every add-on holds on top of its own. +func (a staticAuthority) floorOf(addonID string) []string { + if addonID == "*" { + return nil + } + return a.prefixes["*"] +} + func (a staticAuthority) Allows(addonID, op string) (bool, Reason) { - if hasPrefix(a.prefixes[addonID], op) { + if hasPrefix(a.prefixes[addonID], op) || hasPrefix(a.floorOf(addonID), op) { return true, ReasonNone } return false, ReasonNotGranted } func (a staticAuthority) Snapshot(addonID string) EffectiveSet { - return EffectiveSet{Prefixes: slices.Clone(a.prefixes[addonID])} + return EffectiveSet{Prefixes: slices.Concat(a.prefixes[addonID], a.floorOf(addonID))} } -func (a staticAuthority) Entitled(addonID string) bool { return len(a.prefixes[addonID]) > 0 } +func (a staticAuthority) Entitled(addonID string) bool { + return len(a.prefixes[addonID]) > 0 || len(a.floorOf(addonID)) > 0 +} diff --git a/core/services/addons/hostops/broker.go b/core/services/addons/hostops/broker.go index c995f350de9a..e0fdc744e4b5 100644 --- a/core/services/addons/hostops/broker.go +++ b/core/services/addons/hostops/broker.go @@ -359,7 +359,7 @@ func (b *Broker) serveInbound(w http.ResponseWriter, r *http.Request, spec Spec, func hasPrefix(allowed []string, op string) bool { for _, p := range allowed { - if p != "" && strings.HasPrefix(op, p) { + if p == "*" || (p != "" && strings.HasPrefix(op, p)) { return true } } diff --git a/core/services/addons/hostsettings/authority_test.go b/core/services/addons/hostsettings/authority_test.go index 8e82ca71c38d..c91add0721b6 100644 --- a/core/services/addons/hostsettings/authority_test.go +++ b/core/services/addons/hostsettings/authority_test.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sync" "strings" "testing" @@ -75,7 +76,11 @@ var _ = Describe("Owner policy", func() { Expect(again.Policy.ToolProviders["bash"]).To(Equal("sandbox")) info, err := os.Stat(path) Expect(err).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + } }) It("turns private egress on only with the typed phrase, and never stores it", func() { open() diff --git a/core/services/addons/hostsettings/store.go b/core/services/addons/hostsettings/store.go index dd93e16b357b..fffc3e074c97 100644 --- a/core/services/addons/hostsettings/store.go +++ b/core/services/addons/hostsettings/store.go @@ -12,7 +12,6 @@ import ( "path/filepath" "strings" "sync" - "syscall" ) // Store serializes one owner's policy transactions. The parent directory is @@ -35,7 +34,7 @@ func Open(path string, migration *Migration) (*Store, error) { if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { return nil, err } - lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600) // #nosec G304 -- caller supplies a fixed file in the guarded owner root, never a request path; reject symlinks. + lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR|noFollowFlag, 0600) // #nosec G304 -- caller supplies a fixed file in the guarded owner root, never a request path; reject symlinks. if err != nil { return nil, err } @@ -44,11 +43,11 @@ func Open(path string, migration *Migration) (*Store, error) { if err = lock.Chmod(0600); err != nil { return fail(err) } - if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if err = lockFile(lock); err != nil { return fail(err) } s.persist = func(d document) (bool, error) { return persistFile(path, d) } - f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) // #nosec G304 -- same trusted owner-root file; NOFOLLOW plus mode/type checks below prevent file substitution. + f, err := os.OpenFile(path, os.O_RDONLY|noFollowFlag, 0) // #nosec G304 -- same trusted owner-root file; NOFOLLOW plus mode/type checks below prevent file substitution. var d document if os.IsNotExist(err) { d, err = migrate(migration) @@ -68,7 +67,7 @@ func Open(path string, migration *Migration) (*Store, error) { _ = f.Close() return fail(statErr) } - if !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() > maxDocumentBytes { + if !validFileMode(info.Mode()) || info.Size() > maxDocumentBytes { _ = f.Close() return fail(ErrInvalid) } @@ -184,7 +183,7 @@ func (s *Store) Close() error { if s.lock == nil { return nil } - err := errors.Join(syscall.Flock(int(s.lock.Fd()), syscall.LOCK_UN), s.lock.Close()) + err := errors.Join(unlockFile(s.lock), s.lock.Close()) s.lock = nil return err } @@ -218,7 +217,7 @@ func persistFile(path string, d document) (bool, error) { if err = os.Rename(f.Name(), path); err != nil { return false, err } - if err = dir.Sync(); err != nil { + if err = syncDir(dir); err != nil { return true, err } return true, nil diff --git a/core/services/addons/hostsettings/store_unix.go b/core/services/addons/hostsettings/store_unix.go new file mode 100644 index 000000000000..d966bff32ab6 --- /dev/null +++ b/core/services/addons/hostsettings/store_unix.go @@ -0,0 +1,27 @@ +//go:build !windows + +package hostsettings + +import ( + "os" + "syscall" +) + +const noFollowFlag = syscall.O_NOFOLLOW + +func lockFile(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +func unlockFile(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +} + +func syncDir(dir *os.File) error { + return dir.Sync() +} + +// validFileMode rejects a policy file anyone but its owner can read or write. +func validFileMode(m os.FileMode) bool { + return m.IsRegular() && m.Perm()&0o077 == 0 +} diff --git a/core/services/addons/hostsettings/store_windows.go b/core/services/addons/hostsettings/store_windows.go new file mode 100644 index 000000000000..eea774ea270f --- /dev/null +++ b/core/services/addons/hostsettings/store_windows.go @@ -0,0 +1,33 @@ +//go:build windows + +package hostsettings + +import ( + "os" + + "golang.org/x/sys/windows" +) + +// Windows has no O_NOFOLLOW; the owner root is trusted, and the mode and type +// checks in Open still reject a substituted file. +const noFollowFlag = 0 + +func lockFile(f *os.File) error { + return windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)) +} + +func unlockFile(f *os.File) error { + return windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, new(windows.Overlapped)) +} + +// Directory handles cannot be flushed on Windows; the rename is already +// durable once os.Rename returns. +func syncDir(*os.File) error { + return nil +} + +// validFileMode cannot enforce Unix permission bits on Windows; the owner-root +// ACL is the guard there, so only the regular-file check applies. +func validFileMode(m os.FileMode) bool { + return m.IsRegular() +} diff --git a/core/services/addons/journal.go b/core/services/addons/journal.go index 70e14adcb989..6ecb2b50a643 100644 --- a/core/services/addons/journal.go +++ b/core/services/addons/journal.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/atomicfile" ) type journalOperation struct { @@ -51,7 +52,7 @@ func durableWrite(path string, b []byte) error { return e } defer func() { _ = d.Close() }() - return d.Sync() + return atomicfile.SyncDirTolerant(d) } func (m *Manager) persist(phase string) error { err := m.persistJournal(phase) diff --git a/core/services/addons/manager.go b/core/services/addons/manager.go index bec7304f5adf..fa1e2e02157c 100644 --- a/core/services/addons/manager.go +++ b/core/services/addons/manager.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "io/fs" "os" "path/filepath" "regexp" @@ -15,7 +14,6 @@ import ( "strings" "sync" "sync/atomic" - "syscall" "time" "github.com/mudler/xlog" @@ -999,80 +997,6 @@ func syncPackage(root string) error { return nil } -// Only unpublished Installer staging is collectible. An unreferenced published -// package may be the result of a crash before prepared; retain it for inspection. -func (m *Manager) cleanStages() error { - root, e := openPackageRoot(m.installer.Root) - if os.IsNotExist(e) { - return nil - } - if e != nil { - return e - } - defer func() { _ = root.Close() }() - entries, e := fs.ReadDir(root.FS(), ".") - if e != nil { - return e - } - for _, entry := range entries { - if !strings.HasPrefix(entry.Name(), "stage-") { - continue - } - if e = fs.WalkDir(root.FS(), entry.Name(), func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - return nil - } - // Descriptor chmod avoids following a link swapped in after WalkDir. - f, err := root.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_DIRECTORY, 0) - if err != nil { - return err - } - return errors.Join(f.Chmod(0700), f.Close()) // #nosec G122 -- the tree is owned by this process and locked; the walk only normalises its own files - }); e != nil { - return e - } - if e = root.RemoveAll(entry.Name()); e != nil { - return e - } - } - return nil -} - -// Pin each ancestor and reject links, including the package root, before any -// enumeration or mutation. Root operations keep later path races contained. -func openPackageRoot(path string) (*os.Root, error) { - root, err := os.OpenRoot(string(filepath.Separator)) - if err != nil { - return nil, err - } - for _, part := range strings.Split(strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)), string(filepath.Separator)) { - info, e := root.Lstat(part) - if e != nil { - _ = root.Close() - return nil, e - } - if !info.IsDir() { - _ = root.Close() - return nil, ErrConfiguration - } - next, e := root.OpenRoot(part) - _ = root.Close() - if e != nil { - return nil, e - } - pinned, e := next.Stat(".") - if e != nil || !os.SameFile(info, pinned) { - _ = next.Close() - return nil, ErrConfiguration - } - root = next - } - return root, nil -} - // AttachRuntime is owner-only. Durable reconciliation precedes process execution. func (m *Manager) AttachRuntime(ctx context.Context, s *addonruntime.Supervisor) error { m.mu.Lock() diff --git a/core/services/addons/manager_openpackage_unix.go b/core/services/addons/manager_openpackage_unix.go new file mode 100644 index 000000000000..bb48ae2ee7cd --- /dev/null +++ b/core/services/addons/manager_openpackage_unix.go @@ -0,0 +1,39 @@ +//go:build !windows + +package addons + +import ( + "os" + "path/filepath" + "strings" +) + +func openPackageRoot(path string) (*os.Root, error) { + root, err := os.OpenRoot(string(filepath.Separator)) + if err != nil { + return nil, err + } + for _, part := range strings.Split(strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)), string(filepath.Separator)) { + info, e := root.Lstat(part) + if e != nil { + _ = root.Close() + return nil, e + } + if !info.IsDir() { + _ = root.Close() + return nil, ErrConfiguration + } + next, e := root.OpenRoot(part) + _ = root.Close() + if e != nil { + return nil, e + } + pinned, e := next.Stat(".") + if e != nil || !os.SameFile(info, pinned) { + _ = next.Close() + return nil, ErrConfiguration + } + root = next + } + return root, nil +} \ No newline at end of file diff --git a/core/services/addons/manager_openpackage_windows.go b/core/services/addons/manager_openpackage_windows.go new file mode 100644 index 000000000000..96afbcc9311e --- /dev/null +++ b/core/services/addons/manager_openpackage_windows.go @@ -0,0 +1,51 @@ +//go:build windows + +package addons + +import ( + "os" + "path/filepath" + "strings" +) + +func openPackageRoot(path string) (*os.Root, error) { + // On Windows, extract the volume name (drive letter) to open the correct root. + vol := filepath.VolumeName(path) + rootPath := string(filepath.Separator) + if vol != "" { + rootPath = vol + string(filepath.Separator) + } + root, err := os.OpenRoot(rootPath) + if err != nil { + return nil, err + } + // Remove the volume name from the path before splitting. + remaining := strings.TrimPrefix(filepath.Clean(path), vol) + remaining = strings.TrimPrefix(remaining, string(filepath.Separator)) + for _, part := range strings.Split(remaining, string(filepath.Separator)) { + if part == "" { + continue + } + info, e := root.Lstat(part) + if e != nil { + _ = root.Close() + return nil, e + } + if !info.IsDir() { + _ = root.Close() + return nil, ErrConfiguration + } + next, e := root.OpenRoot(part) + _ = root.Close() + if e != nil { + return nil, e + } + pinned, e := next.Stat(".") + if e != nil || !os.SameFile(info, pinned) { + _ = next.Close() + return nil, ErrConfiguration + } + root = next + } + return root, nil +} \ No newline at end of file diff --git a/core/services/addons/manager_unix.go b/core/services/addons/manager_unix.go new file mode 100644 index 000000000000..084f011e2c26 --- /dev/null +++ b/core/services/addons/manager_unix.go @@ -0,0 +1,51 @@ +//go:build !windows + +package addons + +import ( + "errors" + "io/fs" + "os" + "strings" + "syscall" +) + +func (m *Manager) cleanStages() error { + root, e := openPackageRoot(m.installer.Root) + if os.IsNotExist(e) { + return nil + } + if e != nil { + return e + } + defer func() { _ = root.Close() }() + entries, e := fs.ReadDir(root.FS(), ".") + if e != nil { + return e + } + for _, entry := range entries { + if !strings.HasPrefix(entry.Name(), "stage-") { + continue + } + if e = fs.WalkDir(root.FS(), entry.Name(), func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + // Descriptor chmod avoids following a link swapped in after WalkDir. + f, err := root.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_DIRECTORY, 0) + if err != nil { + return err + } + return errors.Join(f.Chmod(0700), f.Close()) // #nosec G122 -- the tree is owned by this process and locked; the walk only normalises its own files + }); e != nil { + return e + } + if e = root.RemoveAll(entry.Name()); e != nil { + return e + } + } + return nil +} \ No newline at end of file diff --git a/core/services/addons/manager_windows.go b/core/services/addons/manager_windows.go new file mode 100644 index 000000000000..d91ffbd4a13d --- /dev/null +++ b/core/services/addons/manager_windows.go @@ -0,0 +1,50 @@ +//go:build windows + +package addons + +import ( + "errors" + "io/fs" + "os" + "strings" +) + +func (m *Manager) cleanStages() error { + root, e := openPackageRoot(m.installer.Root) + if os.IsNotExist(e) { + return nil + } + if e != nil { + return e + } + defer func() { _ = root.Close() }() + entries, e := fs.ReadDir(root.FS(), ".") + if e != nil { + return e + } + for _, entry := range entries { + if !strings.HasPrefix(entry.Name(), "stage-") { + continue + } + if e = fs.WalkDir(root.FS(), entry.Name(), func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + // On Windows, we don't use O_NOFOLLOW or O_DIRECTORY + f, err := root.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return err + } + return errors.Join(f.Chmod(0700), f.Close()) // #nosec G122 -- the tree is owned by this process and locked; the walk only normalises its own files + }); e != nil { + return e + } + if e = root.RemoveAll(entry.Name()); e != nil { + return e + } + } + return nil +} \ No newline at end of file diff --git a/core/services/addons/ownership.go b/core/services/addons/ownership.go index b1396e7bf8dc..cf6778162cd3 100644 --- a/core/services/addons/ownership.go +++ b/core/services/addons/ownership.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package addons @@ -5,38 +7,12 @@ import ( "context" "encoding/json" "errors" - "net/url" "os" "path/filepath" "sync" "syscall" ) -// OwnerConfig is deployment configuration, never request-supplied authority. -// Shared filesystems and automatic reassignment are intentionally unsupported. -type OwnerConfig struct { - InstanceID, OwnerID, OwnerAddress, StorageID, Root string - SharedStorage bool -} - -func (c OwnerConfig) validate() error { - u, e := url.Parse(c.OwnerAddress) - if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || c.InstanceID == "" || c.OwnerID == "" || c.StorageID == "" || c.SharedStorage { - return ErrConfiguration - } - // Only the owner has a state root. A frontend keeps no add-on state, so it - // has no root to name and nothing may read or create one. - if c.InstanceID == c.OwnerID && !filepath.IsAbs(c.Root) { - return ErrConfiguration - } - return nil -} - -// ownerIdentity is what the identity file stores. -type ownerIdentity struct{ OwnerID, StorageID string } - -type OwnerGuard struct{ Config OwnerConfig } - func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { c := g.Config if e := c.validate(); e != nil { @@ -51,7 +27,8 @@ func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { if e := os.MkdirAll(c.Root, 0700); e != nil { return nil, e } - info, e := os.Lstat(c.Root) + // On Unix, use filepath.ToSlash for os.Lstat compatibility + info, e := os.Lstat(filepath.ToSlash(c.Root)) if e != nil { return nil, e } @@ -72,8 +49,6 @@ func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { once.Do(func() { releaseErr = errors.Join(syscall.Flock(int(f.Fd()), syscall.LOCK_UN), f.Close()) }) return releaseErr } - // The address is runtime configuration and a frontend may reach the owner at - // a new one, so identity is the owner and the storage only. identity := ownerIdentity{OwnerID: c.OwnerID, StorageID: c.StorageID} expected, _ := json.Marshal(identity) p := filepath.Join(c.Root, "identity.json") @@ -92,3 +67,8 @@ func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { } return release, nil } + +var ( + lockMu sync.Mutex + locks = map[string]bool{} +) \ No newline at end of file diff --git a/core/services/addons/ownership_types.go b/core/services/addons/ownership_types.go new file mode 100644 index 000000000000..7a7bb4a30e8d --- /dev/null +++ b/core/services/addons/ownership_types.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MIT +package addons + +import ( + "net/url" + "path/filepath" +) + +// OwnerConfig is deployment configuration, never request-supplied authority. +// Shared filesystems and automatic reassignment are intentionally unsupported. +type OwnerConfig struct { + InstanceID, OwnerID, OwnerAddress, StorageID, Root string + SharedStorage bool +} + +func (c OwnerConfig) validate() error { + u, e := url.Parse(c.OwnerAddress) + if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || c.InstanceID == "" || c.OwnerID == "" || c.StorageID == "" || c.SharedStorage { + return ErrConfiguration + } + // Only the owner has a state root. A frontend keeps no add-on state, so it + // has no root to name and nothing may read or create one. + if c.InstanceID == c.OwnerID && !filepath.IsAbs(c.Root) { + return ErrConfiguration + } + return nil +} + +// ownerIdentity is what the identity file stores. +type ownerIdentity struct{ OwnerID, StorageID string } + +type OwnerGuard struct{ Config OwnerConfig } \ No newline at end of file diff --git a/core/services/addons/ownership_windows.go b/core/services/addons/ownership_windows.go new file mode 100644 index 000000000000..6bfdd1ddecfb --- /dev/null +++ b/core/services/addons/ownership_windows.go @@ -0,0 +1,68 @@ +//go:build windows + +package addons + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + + "golang.org/x/sys/windows" +) + +func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { + c := g.Config + if e := c.validate(); e != nil { + return nil, e + } + if c.InstanceID != c.OwnerID { + return nil, ErrNotOwner + } + if e := ctx.Err(); e != nil { + return nil, e + } + if e := os.MkdirAll(c.Root, 0700); e != nil { + return nil, e + } + info, e := os.Lstat(c.Root) + if e != nil { + return nil, e + } + if !info.IsDir() { + return nil, ErrConfiguration + } + f, e := os.OpenFile(filepath.FromSlash(filepath.Join(c.Root, "owner.lock")), os.O_CREATE|os.O_RDWR, 0600) + if e != nil { + return nil, e + } + if err := windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)); err != nil { + _ = f.Close() + return nil, ErrOwnerBusy + } + var once sync.Once + var releaseErr error + release := func() error { + once.Do(func() { releaseErr = errors.Join(f.Close()) }) + return releaseErr + } + identity := ownerIdentity{OwnerID: c.OwnerID, StorageID: c.StorageID} + expected, _ := json.Marshal(identity) + p := filepath.Join(c.Root, "identity.json") + b, e := os.ReadFile(p) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + if os.IsNotExist(e) { + e = durableWrite(p, expected) + } else if e == nil { + var stored ownerIdentity + if json.Unmarshal(b, &stored) != nil || stored != identity { + e = ErrConfiguration + } + } + if e != nil { + _ = release() + return nil, e + } + return release, nil +} diff --git a/core/services/addons/policy/store.go b/core/services/addons/policy/store.go index 787aef00f0c9..30a3cc357a9a 100644 --- a/core/services/addons/policy/store.go +++ b/core/services/addons/policy/store.go @@ -8,10 +8,10 @@ import ( "io" "os" "path/filepath" + "runtime" "sort" "strings" "sync" - "syscall" "time" ) @@ -269,21 +269,6 @@ type Store struct { failed bool } -func lockDir(dir string) (*os.File, error) { - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, err - } - f, err := os.OpenFile(filepath.Join(dir, lockName), os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request - if err != nil { - return nil, err - } - if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - _ = f.Close() - return nil, ErrStoreBusy - } - return f, nil -} - func load(dir string) (*document, error) { b, err := os.ReadFile(filepath.Join(dir, fileName)) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request if errors.Is(err, os.ErrNotExist) { @@ -644,5 +629,13 @@ func durableWrite(path string, b []byte) (renamed bool, err error) { return true, err } defer func() { _ = d.Close() }() - return true, syncDirectory(d) -} + // On Windows, directory sync may fail with "Access is denied" due to + // permissions. This is non-fatal as the file data is already synced. + if err = syncDirectory(d); err != nil { + if runtime.GOOS == "windows" && strings.Contains(err.Error(), "Access is denied") { + return true, nil + } + return true, err + } + return true, nil +} \ No newline at end of file diff --git a/core/services/addons/policy/store_unix.go b/core/services/addons/policy/store_unix.go new file mode 100644 index 000000000000..6516470f9cbd --- /dev/null +++ b/core/services/addons/policy/store_unix.go @@ -0,0 +1,25 @@ +//go:build !windows + +package policy + +import ( + "os" + "path/filepath" + "syscall" +) + +func lockDir(dir string) (*os.File, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, lockName), os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + if err != nil { + return nil, err + } + if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + return nil, ErrStoreBusy + } + return f, nil +} + diff --git a/core/services/addons/policy/store_windows.go b/core/services/addons/policy/store_windows.go new file mode 100644 index 000000000000..a9d28e0ffc73 --- /dev/null +++ b/core/services/addons/policy/store_windows.go @@ -0,0 +1,25 @@ +//go:build windows + +package policy + +import ( + "os" + "path/filepath" + + "golang.org/x/sys/windows" +) + +func lockDir(dir string) (*os.File, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.FromSlash(filepath.Join(dir, lockName)), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)); err != nil { + _ = f.Close() + return nil, ErrStoreBusy + } + return f, nil +} diff --git a/core/services/addons/recovery/backup.go b/core/services/addons/recovery/backup.go index 063c54c3267c..075e5a752e57 100644 --- a/core/services/addons/recovery/backup.go +++ b/core/services/addons/recovery/backup.go @@ -16,7 +16,6 @@ import ( "sort" "strings" "sync" - "syscall" "time" "github.com/mudler/LocalAI/pkg/addons/contract" @@ -200,17 +199,6 @@ func (r *Recovery) newBackupID() (string, error) { return "b" + r.cfg.Now().UTC().Format("20060102t150405z") + "-" + hex.EncodeToString(b[:]), nil } -func freeBytes(path string) (uint64, error) { - var st syscall.Statfs_t - if err := syscall.Statfs(path, &st); err != nil { - return 0, err - } - if st.Bsize <= 0 { - return 0, errors.New("the file system reports no block size") - } - return uint64(st.Bavail) * uint64(st.Bsize), nil -} - func (r *Recovery) freeSpace(path string) (uint64, error) { if r.free != nil { return r.free(path) diff --git a/core/services/addons/recovery/backup_unix.go b/core/services/addons/recovery/backup_unix.go new file mode 100644 index 000000000000..8ab6e2fe52a1 --- /dev/null +++ b/core/services/addons/recovery/backup_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package recovery + +import ( + "errors" + "path/filepath" + "syscall" +) + +// freeBytes returns free disk space on the given path (Unix version). +func freeBytes(path string) (uint64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(filepath.ToSlash(path), &st); err != nil { + return 0, err + } + if st.Bsize <= 0 { + return 0, errors.New("the file system reports no block size") + } + return uint64(st.Bavail) * uint64(st.Bsize), nil +} \ No newline at end of file diff --git a/core/services/addons/recovery/backup_windows.go b/core/services/addons/recovery/backup_windows.go new file mode 100644 index 000000000000..1c161a219366 --- /dev/null +++ b/core/services/addons/recovery/backup_windows.go @@ -0,0 +1,19 @@ +//go:build windows + +package recovery + +import ( + "path/filepath" + + "golang.org/x/sys/windows" +) + +// freeBytes returns the disk space the caller can use on the given path +// (Windows version), the analogue of the Unix statfs Bavail*Bsize. +func freeBytes(path string) (uint64, error) { + var avail uint64 + if err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr(filepath.FromSlash(path)), &avail, nil, nil); err != nil { + return 0, err + } + return avail, nil +} diff --git a/core/services/addons/resources/selection.go b/core/services/addons/resources/selection.go index ff3641b913ce..cfd18e3e311b 100644 --- a/core/services/addons/resources/selection.go +++ b/core/services/addons/resources/selection.go @@ -10,8 +10,9 @@ import ( "path/filepath" "sort" "sync" - "syscall" "time" + + "github.com/mudler/LocalAI/pkg/atomicfile" ) const ( @@ -94,19 +95,15 @@ func OpenSelections(dir string, o SelectionOptions) (*SelectionStore, error) { o.Now = time.Now } if o.SyncDirectory == nil { - o.SyncDirectory = func(d *os.File) error { return d.Sync() } + o.SyncDirectory = atomicfile.SyncDirTolerant } if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } - lock, err := os.OpenFile(filepath.Join(dir, selectionLock), os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + lock, err := lockSelections(dir) if err != nil { return nil, err } - if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - _ = lock.Close() - return nil, ErrSelectionStore - } s := &SelectionStore{dir: dir, opts: o, lock: lock, entries: map[[3]string]selectionEntry{}} matches, _ := filepath.Glob(filepath.Join(dir, ".selections-*")) for _, m := range matches { diff --git a/core/services/addons/resources/selection_unix.go b/core/services/addons/resources/selection_unix.go new file mode 100644 index 000000000000..17222ca6ea25 --- /dev/null +++ b/core/services/addons/resources/selection_unix.go @@ -0,0 +1,28 @@ +//go:build !windows + +package resources + +import ( + "os" + "path/filepath" + "syscall" +) + +func lockSelections(dir string) (*os.File, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, selectionLock), os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request + if err != nil { + return nil, err + } + if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + return nil, ErrSelectionStore + } + return f, nil +} + +func unlockSelections(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +} \ No newline at end of file diff --git a/core/services/addons/resources/selection_windows.go b/core/services/addons/resources/selection_windows.go new file mode 100644 index 000000000000..3666696be5ee --- /dev/null +++ b/core/services/addons/resources/selection_windows.go @@ -0,0 +1,29 @@ +//go:build windows + +package resources + +import ( + "os" + "path/filepath" + + "golang.org/x/sys/windows" +) + +func lockSelections(dir string) (*os.File, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.FromSlash(filepath.Join(dir, selectionLock)), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)); err != nil { + _ = f.Close() + return nil, ErrSelectionStore + } + return f, nil +} + +func unlockSelections(f *os.File) error { + return f.Close() +} diff --git a/core/services/studio/sandbox/host/host_windows.go b/core/services/studio/sandbox/host/host_windows.go new file mode 100644 index 000000000000..4f0aa5523e00 --- /dev/null +++ b/core/services/studio/sandbox/host/host_windows.go @@ -0,0 +1,46 @@ +//go:build windows + +package host + +import ( + "context" + "fmt" + + "github.com/mudler/LocalAI/core/services/studio/sandbox" +) + +// Options configures the backend. +type Options struct { + // Root is the folder the sandboxes live in. It must be an absolute path. + Root string + // Log is called before every command with the principal the sandbox runs + // for and the command line. It is the record of what ran on the host. + Log func(principal, command string) +} + +// Backend runs commands on the host. The host backend runs POSIX shell +// commands in a process group, which Windows cannot provide, so New always +// reports the backend as unsupported there. Callers use +// errors.Is(err, sandbox.ErrUnsupported) to fall back to another backend. +type Backend struct{} + +// New reports that the host backend is not available on Windows. Callers use +// errors.Is(err, sandbox.ErrUnsupported) to fall back to another backend. +func New(Options) (*Backend, error) { + return nil, fmt.Errorf("%w: the host backend needs a POSIX process group and shell, which Windows does not provide", sandbox.ErrUnsupported) +} + +// Skips lists what this backend cannot do, for the conformance suite. +func (*Backend) Skips(feature string) bool { return true } + +func (*Backend) Create(context.Context, sandbox.Spec) (sandbox.Sandbox, error) { + return nil, fmt.Errorf("%w: the host backend is not available on Windows", sandbox.ErrUnsupported) +} + +func (*Backend) Resume(context.Context, sandbox.SnapshotRef, sandbox.Env) (sandbox.Sandbox, error) { + return nil, fmt.Errorf("%w: the host backend is not available on Windows", sandbox.ErrUnsupported) +} + +func (*Backend) Discard(context.Context, sandbox.SnapshotRef) error { return nil } + +func (*Backend) Close() error { return nil } diff --git a/core/services/studio/workspace/blob/blob.go b/core/services/studio/workspace/blob/blob.go index a48d677f6e3c..b5cad75b2633 100644 --- a/core/services/studio/workspace/blob/blob.go +++ b/core/services/studio/workspace/blob/blob.go @@ -111,7 +111,14 @@ func place(tmp, final string) error { return err } if err := os.Rename(tmp, final); err != nil { - if _, serr := os.Stat(final); serr == nil && errors.Is(err, os.ErrExist) { + // On Windows, renaming over an existing file can report + // "Access is denied" instead of "file exists". If the target + // is already in place, treat it as success: another writer or + // a previous attempt already published this blob. + if _, serr := os.Stat(final); serr == nil { + return os.Remove(tmp) + } + if errors.Is(err, os.ErrExist) { return os.Remove(tmp) } return err diff --git a/core/services/studio/workspace/blob/fs.go b/core/services/studio/workspace/blob/fs.go index 6eecbbe5b0e3..6d09e73c57b6 100644 --- a/core/services/studio/workspace/blob/fs.go +++ b/core/services/studio/workspace/blob/fs.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/mudler/LocalAI/core/services/studio/workspace" + "github.com/mudler/LocalAI/pkg/atomicfile" ) type fsStore struct { @@ -73,7 +74,7 @@ func (s *fsStore) syncDir(path string) error { return err } defer func() { _ = f.Close() }() - return f.Sync() + return atomicfile.SyncDirTolerant(f) } func (s *fsStore) mkdir(path string) error { if err := s.safe(path); err != nil { @@ -189,9 +190,13 @@ func (t *fsStaged) Publish(ctx context.Context) error { if err := s.stage("rename"); err != nil { return err } - // Rename replaces an existing blob with identical bytes. Never remove - // final after a sync failure: another writer can own that name. + // Rename replaces an existing blob with identical bytes. On Windows, + // renaming over an existing file can report "Access is denied" instead + // of "file exists". If the target is already in place, nothing to do. if err := os.Rename(t.path, final); err != nil { + if _, serr := os.Stat(final); serr == nil { + return nil + } return err } return s.syncDir(filepath.Dir(final)) diff --git a/core/services/studio/workspace/jsonstore/store.go b/core/services/studio/workspace/jsonstore/store.go index bf972327ccd1..36452b12c90e 100644 --- a/core/services/studio/workspace/jsonstore/store.go +++ b/core/services/studio/workspace/jsonstore/store.go @@ -32,12 +32,15 @@ import ( "io" "os" "path/filepath" + "runtime" "slices" "sync" "time" "github.com/mudler/xlog" + "github.com/mudler/LocalAI/pkg/atomicfile" + "github.com/mudler/LocalAI/core/services/studio/workspace" "github.com/mudler/LocalAI/core/services/studio/workspace/blob" ) @@ -453,7 +456,15 @@ func (s *Store) writeSnapshot(snap snapshot) error { return err } if err := s.log.Truncate(0); err != nil { - return err + // On Windows, truncating a log file that is open by another handle + // fails with "Access is denied". The snapshot has already been + // written and the directory synced, so truncation is non-critical: + // the log will be replayed on next open (the snapshot covers it). + if runtime.GOOS == "windows" { + xlog.Warn("workspace: could not truncate log", "error", err) + } else { + return err + } } s.logSize = 0 return nil @@ -465,10 +476,7 @@ func syncDir(dir string) error { return err } defer func() { _ = f.Close() }() - if err := f.Sync(); err != nil && !errors.Is(err, os.ErrInvalid) { - return err - } - return nil + return atomicfile.SyncDirTolerant(f) } func (d *data) snapshot() snapshot { diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 80e511201b7d..ebecc3c8bc79 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -22,6 +22,9 @@ func SetupTestDB() *gorm.DB { if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") } + if runtime.GOOS == "windows" { + Skip("testcontainers requires Docker, not available on Windows CI") + } ctx := context.Background() pgC, err := tcpostgres.Run(ctx, "postgres:16", tcpostgres.WithDatabase("testdb"), diff --git a/docs/content/features/GPU-acceleration.md b/docs/content/features/GPU-acceleration.md index 085617c24bac..df53a6996ba3 100644 --- a/docs/content/features/GPU-acceleration.md +++ b/docs/content/features/GPU-acceleration.md @@ -17,6 +17,10 @@ For advanced use cases or to override auto-detection, you can use the `LOCALAI_F - `nvidia`: Forces backends compiled with CUDA support for NVIDIA GPUs. - `amd`: Forces backends compiled with ROCm support for AMD GPUs. - `intel`: Forces backends compiled with SYCL/oneAPI support for Intel GPUs. +- `vulkan`: Forces backends compiled with Vulkan support (e.g. the Windows llama-cpp backend, which auto-detects a Vulkan device and falls back to CPU — see [Windows]({{% relref "getting-started/windows" %}})). +- `metal`: Forces backends compiled with Metal support (macOS/Apple Silicon). +- `darwin-x86`: Forces the Intel-macOS build of a backend. +- `windows`: Forces the native Windows build of a backend (only `llama-cpp` ships a Windows build today). ## Model configuration @@ -379,6 +383,13 @@ On an integrated Intel GPU, the amount of free graphics memory can only be read If using nvidia, follow the steps in the [CUDA](#cudanvidia-acceleration) section to configure your docker runtime to allow access to the GPU. +### Native Windows + +The Windows binary ships the llama-cpp backend with Vulkan baked in: it +auto-detects a Vulkan device at runtime and offloads `gpu_layers` to it, +falling back to CPU when no Vulkan driver is present. No configuration or +driver passthrough is required — see [Windows]({{% relref "getting-started/windows" %}}). + ### Container images To use Vulkan, use the images with the `vulkan` tag, for example `{{< version >}}-gpu-vulkan`. diff --git a/docs/content/getting-started/build.md b/docs/content/getting-started/build.md index 42e64928d2e5..da6813802eea 100644 --- a/docs/content/getting-started/build.md +++ b/docs/content/getting-started/build.md @@ -55,6 +55,17 @@ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f1 ``` +{{% /tab %}} +{{% tab title="Windows" %}} + +The `make` recipes are POSIX shell. Git for Windows bundles the POSIX tools they need (`sh`, `uname`, `unzip`, `grep`, `awk`, `sed`) — but **not** `make` itself — so install a GNU make for Windows as well: + +```powershell +winget install ezwinports.make +``` + +The Makefile locates Git for Windows' `sh` on its own and runs every recipe through it, downloads `protoc` and installs the Go protobuf plugins itself — so no PATH edits or extra toolchain are needed. `make build` also compiles the React UI, so install [Node.js](https://nodejs.org) (with npm) as well. + {{% /tab %}} {{% tab title="From source" %}} @@ -76,6 +87,8 @@ make build This should produce the binary `local-ai` +On Windows, `make build` produces `local-ai.exe` instead. + #### Container image Requirements: @@ -91,7 +104,7 @@ docker run localai ### Example: Build on mac -Building on Mac (M1, M2 or M3) works, but you may need to install some prerequisites using `brew`. +Building on Mac (M1, M2 or M3) works, but you may need to install some prerequisites using `brew`. The below has been tested by one mac user and found to work. Note that this doesn't use Docker to run the server: @@ -119,7 +132,7 @@ curl http://localhost:8080/v1/models curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ "model": "phi-2.Q2_K", "messages": [{"role": "user", "content": "How are you?"}], - "temperature": 0.9 + "temperature": 0.9 }' ``` @@ -147,6 +160,27 @@ make clean make build ``` +### Example: Build on Windows + +Building the server binary on Windows needs no C toolchain: the release config compiles with `CGO_ENABLED=0`. The `make` recipes are POSIX shell, so you need Git for Windows (which bundles `sh` and the POSIX tools) plus a GNU make for Windows — Git for Windows does not ship `make`. The Makefile finds `sh` on its own, so no PATH edits are required. + +```powershell +# Install Go (https://go.dev/dl), Git for Windows (https://git-scm.com/downloads), +# Node.js for the React UI build (https://nodejs.org), then a GNU make for +# Windows, e.g.: +winget install ezwinports.make + +# Build with CGO disabled like the release config. +$env:CGO_ENABLED = '0' + +git clone https://github.com/mudler/LocalAI.git +cd LocalAI + +make build +``` + +This produces the binary `local-ai.exe`. The `llama-cpp` backend is also distributed natively for Windows — see [Building the Windows backend]({{% relref "getting-started/windows" %}}#building-from-source) for the MSYS2 UCRT64 requirements and the `make backends/llama-cpp-windows` target. Other backends are still built as Linux container images and are not distributed for Windows. + ## Build backends LocalAI have several backends available for installation in the backend gallery. The backends can be also built by source. As backends might vary from language and dependencies that they require, the documentation will provide generic guidance for few of the backends, which can be applied with some slight modifications also to the others. @@ -178,7 +212,7 @@ make docker-build- Note that `make` is only by convenience, in reality it just runs a simple `docker` command as: ```bash -docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend: -f LocalAI/backend/Dockerfile.golang --build-arg BACKEND= . +docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend: -f LocalAI/backend/Dockerfile.golang --build-arg BACKEND= . ``` Note: diff --git a/docs/content/getting-started/install.md b/docs/content/getting-started/install.md index 861bba652276..77c6ba9d7c84 100644 --- a/docs/content/getting-started/install.md +++ b/docs/content/getting-started/install.md @@ -20,6 +20,7 @@ Choose the installation method that best suits your needs: 1. **[Containers]({{% relref "getting-started/containers" %}})** ⭐ **Recommended** - Works on all platforms, supports Docker and Podman 2. **[macOS]({{% relref "getting-started/macos" %}})** - Download and install the DMG application -3. **[Linux]({{% relref "getting-started/linux" %}})** - Install on Linux using binaries -4. **[Kubernetes]({{% relref "getting-started/kubernetes" %}})** - Deploy LocalAI on Kubernetes clusters -5. **[Build from Source]({{% relref "getting-started/build" %}})** - Build LocalAI from source code +3. **[Windows]({{% relref "getting-started/windows" %}})** - Run natively on Windows, no WSL or Docker required +4. **[Linux]({{% relref "getting-started/linux" %}})** - Install on Linux using binaries +5. **[Kubernetes]({{% relref "getting-started/kubernetes" %}})** - Deploy LocalAI on Kubernetes clusters +6. **[Build from Source]({{% relref "getting-started/build" %}})** - Build LocalAI from source code diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index 30791b9b47ae..790e5f7913d5 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -352,7 +352,7 @@ curl http://localhost:8080/v1/completions -H "Content-Type: application/json" -d For other Docker images, please refer to the table in [Getting Started](https://localai.io/basics/getting_started/#container-images). {{% /notice %}} -Note: If you are on Windows, ensure the project is on the Linux filesystem to avoid slow model loading. For more information, see the [Microsoft Docs](https://learn.microsoft.com/en-us/windows/wsl/filesystems). +Note: When running LocalAI under Docker/WSL2 on Windows, ensure the project is on the Linux filesystem to avoid slow model loading. For more information, see the [Microsoft Docs](https://learn.microsoft.com/en-us/windows/wsl/filesystems). {{% /tab %}} {{% tab title="Kubernetes" %}} diff --git a/docs/content/getting-started/troubleshooting.md b/docs/content/getting-started/troubleshooting.md index d216fe12d587..6c64cb32a39e 100644 --- a/docs/content/getting-started/troubleshooting.md +++ b/docs/content/getting-started/troubleshooting.md @@ -72,7 +72,7 @@ Report non-reproduction if matched requests do not reproduce the delay. ```bash chmod +x local-ai-* -./local-ai-Linux-x86_64 run +./local-ai--linux-amd64 run ``` If you see "cannot execute binary file: Exec format error", you downloaded the wrong architecture. Verify with: diff --git a/docs/content/getting-started/windows.md b/docs/content/getting-started/windows.md new file mode 100644 index 000000000000..0b7ef6204d7f --- /dev/null +++ b/docs/content/getting-started/windows.md @@ -0,0 +1,198 @@ +--- +title: Windows +weight: 8 +description: > + Install and run LocalAI on Windows without WSL or Docker. Native + windows/amd64 backend support. +--- + +## Native Windows backends (no WSL / Docker required) + +LocalAI can run entirely natively on Windows using pre-built backend images +packaged as OCI tarballs. The llama-cpp backend is the first backend with +native Windows support; other backends will follow. + +## Prerequisites + +- Windows 10/11, **amd64** +- PowerShell 7+ (recommended) or cmd.exe +- No WSL, no Docker Desktop, no MSYS2 needed at runtime + +The `local-ai` launcher downloads the Windows backend image on first run and +executes it as a native process. + +## Install + +```powershell +# Download the Windows release asset `local-ai--windows-amd64.exe` +# from the releases page: https://github.com/mudler/LocalAI/releases + +# Or via winget (when available): +# winget install mudler.localai +``` + +## First run + +```powershell +# Download a model (for example, Llama 3.2 1B Instruct GGUF) +.\local-ai.exe model install huggingface://bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M + +# Start LocalAI +.\local-ai.exe run +``` + +LocalAI selects the best llama-cpp binary shipped in the Windows backend image +automatically. No configuration is required. + +## Backend selection on Windows + +Windows backend images ship three llama-cpp executables: + +- `llama-cpp-cpu-all.exe` — CPU-only build with all ggml CPU backends (preferred + when present). +- `llama-cpp-grpc.exe` — gRPC-RPC build, selected automatically when the + `LLAMACPP_GRPC_SERVERS` environment variable is set. +- `llama-cpp-fallback.exe` — static fallback used when neither of the above + applies. + +A small `run.ps1` launcher (Windows has no shell to run the `run.sh` stub +through, so `pkg/model/process.go` starts `run.ps1` via `powershell.exe`) +picks the right binary at startup. + +### GPU (Vulkan) support + +The Windows backend image ships `ggml-vulkan.dll` alongside the CPU backends. +The single `llama-cpp-cpu-all.exe` build auto-detects a Vulkan device at +startup and offloads `gpu_layers` to it when a model requests them, falling +back to CPU inference on hosts with no Vulkan driver. No configuration is +required — the Vulkan loader (`vulkan-1.dll`) is bundled with the image and +loaded dynamically, so the backend never hard-depends on it. + +## Environment variables + +| Variable | Effect | +|----------|--------| +| `LLAMACPP_GRPC_SERVERS` | Forces use of `llama-cpp-grpc.exe` when set. | + +## Troubleshooting + +### Missing Visual C++ runtime + +If the backend fails to start with a `DLL not found` error, install the +[Visual C++ Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). + +### Antivirus / SmartScreen + +Windows Defender SmartScreen may warn about the unsigned `local-ai.exe` and +backend binaries. Click **More info** → **Run anyway** to proceed. Future +releases will be signed. + +### Firewall + +LocalAI binds `:8080` on **all interfaces** by default, with no API key +configured — on a shared network that exposes the API to anyone who can +reach the host. For local use, bind explicitly to the loopback interface +and set a key instead: + +```powershell +./local-ai.exe --address 127.0.0.1:8080 --api-key your-key-here +``` + +When the firewall prompts on first run, allow local-ai.exe only if you +intend to listen on non-loopback interfaces (for example, to serve models +to other machines on your LAN); for a loopback-only setup, deny the +prompt. + +## Building from source + +See `scripts/build/llama-cpp-windows.sh` and the Makefile target +`backends/llama-cpp-windows` if you want to rebuild the Windows backend image +locally. The target is re-runnable: an interrupted build can simply be run +again (the script re-arms its source clones, and the long gRPC build tree is +reused between runs). +The build requires an MSYS2 UCRT64 environment with mingw-w64 GCC, +CMake 4.x, Ninja, the Vulkan headers/loader and glslc (see below), and Go +(to build the `local-ai.exe` that assembles the OCI image tar; it must be +reachable via `GOROOT`, `GO_TOOLCHAIN_ROOT`, or `PATH`). + +### Setting up MSYS2 UCRT64 + +The Makefile target installs MSYS2 and the toolchain automatically (after +confirmation) when they are missing, so the manual steps below are only needed +if you prefer to install by hand: + +1. Download and install MSYS2 from https://www.msys2.org/ +2. Open the **MSYS2 UCRT64** terminal (not plain MSYS2) +3. Install the build toolchain: + +```bash +pacman -S --needed --noconfirm base-devel git cmake ninja mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-vulkan-headers mingw-w64-ucrt-x86_64-vulkan-loader mingw-w64-ucrt-x86_64-spirv-headers mingw-w64-x86_64-shaderc +``` + +The Vulkan packages are required: `GGML_VULKAN=ON` makes CMake's `FindVulkan` +treat `glslc` as a REQUIRED component, and ggml-vulkan runs +`find_package(SPIRV-Headers CONFIG REQUIRED)`. MSYS2 ships glslc only in the +mingw64 variant of `shaderc` (there is no ucrt64 build) — it is a standalone +tool, so the build script adds `/mingw64/bin` to `PATH` when no glslc is +already available (e.g. from a Vulkan SDK installed via the registry). The +script also installs `mingw-w64-ucrt-x86_64-spirv-headers` itself if its cmake +config is missing, so the docs' package list is only a head start. + +4. Run the build from a plain Windows shell (PowerShell, `cmd`, or Git bash): + +```powershell +cd C:\source_path\LocalAI +make backends/llama-cpp-windows +``` + + On a native Windows host the Makefile hands the build to + `scripts/build/llama-cpp-windows.ps1`, which runs the sh script under the + real MSYS2 UCRT64 bash. If MSYS2 (or its mingw-w64 UCRT64 toolchain) is not + installed, the dispatcher asks for confirmation before downloading and + installing it via `winget`/`pacman`; declining stops the build. No need to + open the UCRT64 terminal yourself. + + Inside the MSYS2 UCRT64 terminal the script runs directly: + +```bash +cd /source_path/LocalAI +bash scripts/build/llama-cpp-windows.sh +``` + +The script packages the result as an OCI tarball (`backend-images/llama-cpp.tar`) +and — when run through the Makefile — installs it with +`./local-ai.exe backends install "ocifile://..."`. + +### GitHub Actions build + +CI builds the Windows backend in the reusable +[`.github/workflows/backend_build_windows.yml`](https://github.com/mudler/LocalAI/blob/master/.github/workflows/backend_build_windows.yml) +workflow, dispatched from `.github/workflows/backend.yml` for the `includeWindows` +matrix entry. Only `llama-cpp` has a Windows build path today — a future entry +dispatched without one fails loudly rather than upload an empty tarball. + +The pipeline: + +1. **Build job** (`windows-latest`): sets up MSYS2 UCRT64 with the same + package set as the local instructions above, caches ccache keyed on the + pinned `LLAMA_VERSION`, then runs `make backends/llama-cpp-windows` in an + MSYS2 shell (the Go toolchain from `actions/setup-go` is handed in via + `GO_TOOLCHAIN_ROOT`, since the MSYS2 shell resets `PATH`). The produced + `backend-images/llama-cpp.tar` is uploaded as a workflow artifact. +2. **Publish job** (`ubuntu-latest`, skipped on PRs): downloads the tarball, + logs into DockerHub and quay.io with `crane`, tags it with the standard + branch/semver/sha metadata (the `-windows-amd64-llama-cpp` tag suffix from + the matrix), + and pushes the OCI image to both registries. + +The gallery entry points at the published image, so `local-ai backends install` +fetches the exact artifact CI produced. + +The published image is signed keyless with Sigstore/cosign, the same way the +platform images that go through the `backend_merge.yml` manifest merge are: +the publish job resolves the pushed digest and signs it with an OCI 1.1 +referrer bundle. The gallery does not enforce a `verification:` policy yet — +`backend/index.yaml` still ships without those blocks populated, and strict +mode stays off until a gallery covers every published variant — so installs +verify the signature, but a missing or invalid one degrades to a warning +rather than a refusal. diff --git a/docs/content/reference/binaries.md b/docs/content/reference/binaries.md index 178f311277c2..9c68640bfc25 100644 --- a/docs/content/reference/binaries.md +++ b/docs/content/reference/binaries.md @@ -5,7 +5,7 @@ title = "LocalAI binaries" weight = 26 +++ -LocalAI binaries are available for both Linux and MacOS platforms and can be executed directly from your command line. These binaries are continuously updated and hosted on [our GitHub Releases page](https://github.com/mudler/LocalAI/releases). This method also supports Windows users via the Windows Subsystem for Linux (WSL). +LocalAI binaries are available for Linux, macOS, and Windows platforms and can be executed directly from your command line. These binaries are continuously updated and hosted on [our GitHub Releases page](https://github.com/mudler/LocalAI/releases). ### macOS Download @@ -13,23 +13,26 @@ You can download the DMG and install the application: Download LocalAI for macOS - + > Note: the DMGs are not signed by Apple as quarantined. See https://github.com/mudler/LocalAI/issues/6268 for a workaround, fix is tracked here: https://github.com/mudler/LocalAI/issues/6244 -Otherwise, use the following one-liner command in your terminal to download and run LocalAI on Linux or MacOS: +Otherwise, use the following one-liner command in your terminal to download and run LocalAI on Linux or MacOS (set `VERSION` to the current tag, e.g. `v4.8.2`): ```bash -curl -Lo local-ai "https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-$(uname -s)-$(uname -m)" && chmod +x local-ai && ./local-ai +VERSION=v4.8.2; ARCH=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/'); \ +curl -Lo local-ai "https://github.com/mudler/LocalAI/releases/download/$VERSION/local-ai-$VERSION-$(uname -s | tr '[:upper:]' '[:lower:]')-$ARCH" \ + && chmod +x local-ai && ./local-ai ``` Otherwise, here are the links to the binaries: -| OS | Link | +| OS | Link | | --- | --- | -| Linux (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Linux-x86_64) | -| Linux (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Linux-arm64) | -| MacOS (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Darwin-arm64) | +| Linux (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-linux-amd64) | +| Linux (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-linux-arm64) | +| MacOS (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-darwin-arm64) | +| Windows (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-windows-amd64.exe) | {{% notice icon="⚡" context="warning" %}} @@ -38,4 +41,5 @@ Binaries do have limited support compared to container images: - Python-based backends are not shipped with binaries (e.g. `diffusers` or `transformers`) - MacOS binaries and Linux-arm64 do not ship TTS nor `stablediffusion-cpp` backends - Linux binaries do not ship `stablediffusion-cpp` backend +- The Windows binary ships only the `llama-cpp` backend (native, no WSL required); see the [Windows guide]({{% relref "getting-started/windows" %}}) for details {{% /notice %}} diff --git a/gallery/backends/index.yaml b/gallery/backends/index.yaml index 5bba53ec0cd8..adad4169e075 100644 --- a/gallery/backends/index.yaml +++ b/gallery/backends/index.yaml @@ -24,6 +24,7 @@ nvidia-l4t-cuda-12: nvidia-l4t-arm64-llama-cpp nvidia-l4t-cuda-13: cuda13-nvidia-l4t-arm64-llama-cpp vulkan: vulkan-llama-cpp + windows: windows-llama-cpp channel: latest description: | LLM inference in C/C++ @@ -8727,6 +8728,62 @@ urls: - https://github.com/ikawrakow/ik_llama.cpp version: 0.0.0-development +- alias: llama-cpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:latest-windows-llama-cpp + capabilities: + default: cpu-llama-cpp + channel: latest + description: | + LLM inference in C/C++ for Windows + id: windows-llama-cpp + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:latest-windows-llama-cpp + # Unpinned until backend-jobs-windows pushes the image once; pin it with + # backendcatalog --repin afterwards. + mutable: true + name: windows-llama-cpp + os: windows + schemaVersion: 1 + tags: + - text-to-text + - text-to-speech + - TTS + - LLM + - CPU + - GPU + urls: + - https://github.com/ggerganov/llama.cpp + version: 0.0.0-latest +- alias: llama-cpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:master-windows-llama-cpp + capabilities: + default: cpu-llama-cpp + channel: development + description: | + LLM inference in C/C++ for Windows + id: windows-llama-cpp-development + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:master-windows-llama-cpp + mutable: true + name: windows-llama-cpp-development + os: windows + schemaVersion: 1 + tags: + - text-to-text + - text-to-speech + - TTS + - LLM + - CPU + - GPU + urls: + - https://github.com/ggerganov/llama.cpp + version: 0.0.0-development - alias: turboquant arch: amd64 artifact: oci://quay.io/go-skynet/local-ai-backends:latest-cpu-turboquant diff --git a/pkg/atomicfile/atomicfile.go b/pkg/atomicfile/atomicfile.go index 9d8caa3bf183..824db912fa9f 100644 --- a/pkg/atomicfile/atomicfile.go +++ b/pkg/atomicfile/atomicfile.go @@ -68,10 +68,23 @@ func WriteWith(path string, data []byte, perm os.FileMode, o Options) (renamed b defer func() { _ = d.Close() }() sync := o.SyncDir if sync == nil { - sync = func(d *os.File) error { return d.Sync() } + sync = SyncDirTolerant } if err := sync(d); err != nil { return true, fmt.Errorf("atomicfile: sync %s: %w", dir, err) } return true, nil } + +// SyncDirTolerant syncs a directory handle for durability. On Windows, +// syncing a directory handle fails with "Access is denied"; the file data is +// already synced by the file handle's Sync above, so treat that as non-fatal. +func SyncDirTolerant(d *os.File) error { + if err := d.Sync(); err != nil { + if dirSyncDenied(err) { + return nil + } + return err + } + return nil +} diff --git a/pkg/atomicfile/atomicfile_test.go b/pkg/atomicfile/atomicfile_test.go index b8393865dc12..40c551327217 100644 --- a/pkg/atomicfile/atomicfile_test.go +++ b/pkg/atomicfile/atomicfile_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "testing" . "github.com/onsi/ginkgo/v2" @@ -27,7 +28,11 @@ var _ = Describe("Write", func() { Expect(string(got)).To(Equal("two")) st, err := os.Stat(path) Expect(err).ToNot(HaveOccurred()) - Expect(st.Mode().Perm()).To(Equal(os.FileMode(0o640))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(st.Mode().Perm()).To(Equal(os.FileMode(0o640))) + } }) It("leaves no temporary file behind", func() { diff --git a/pkg/atomicfile/sync_other.go b/pkg/atomicfile/sync_other.go new file mode 100644 index 000000000000..e9508e2a0737 --- /dev/null +++ b/pkg/atomicfile/sync_other.go @@ -0,0 +1,7 @@ +//go:build !windows + +package atomicfile + +// dirSyncDenied is always false off Windows: a directory Sync failure there +// is real and must surface. +func dirSyncDenied(error) bool { return false } diff --git a/pkg/atomicfile/sync_windows.go b/pkg/atomicfile/sync_windows.go new file mode 100644 index 000000000000..4c665ad7eeb6 --- /dev/null +++ b/pkg/atomicfile/sync_windows.go @@ -0,0 +1,14 @@ +//go:build windows + +package atomicfile + +import ( + "errors" + "syscall" +) + +// dirSyncDenied reports whether err is the "Access is denied" a Windows +// directory-handle Sync fails with; the file data is already durable then. +func dirSyncDenied(err error) bool { + return errors.Is(err, syscall.ERROR_ACCESS_DENIED) +} diff --git a/pkg/downloader/auth_progress_test.go b/pkg/downloader/auth_progress_test.go index debf45f9e79d..622e652720e5 100644 --- a/pkg/downloader/auth_progress_test.go +++ b/pkg/downloader/auth_progress_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "sync" . "github.com/onsi/ginkgo/v2" @@ -141,7 +142,11 @@ var _ = Describe("authenticated HTTP downloads", func() { Expect(errors.Is(err, context.Canceled)).To(BeTrue()) info, statErr := os.Stat(target + ".partial") Expect(statErr).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + } }) It("keeps the legacy total empty when the response length is unknown", func() { diff --git a/pkg/downloader/partial_resume_test.go b/pkg/downloader/partial_resume_test.go index 9cf953dee2ef..42e213011477 100644 --- a/pkg/downloader/partial_resume_test.go +++ b/pkg/downloader/partial_resume_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -157,6 +158,12 @@ var _ = Describe("DownloadFile with a leftover .partial", func() { }) It("reports an informative error when the partial cannot be stat'd", func() { + if runtime.GOOS == "windows" { + // A self-referencing symlink is the portable trick that makes + // os.Stat fail with ELOOP, but creating symlinks on Windows needs + // admin rights or Developer Mode, so the setup itself errors first. + Skip("symlink creation requires privileges on Windows") + } server := rangeServer(true, nil, nil) defer server.Close() diff --git a/pkg/downloader/uri.go b/pkg/downloader/uri.go index 754908e46436..cd878f4c3032 100644 --- a/pkg/downloader/uri.go +++ b/pkg/downloader/uri.go @@ -866,6 +866,10 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string if startPos > 0 && resp.StatusCode != http.StatusPartialContent { _ = resp.Body.Close() + // The origin ignored the resume range, so the partial is garbage; + // a retry must start clean. Drop the write handle first — Windows + // cannot delete a file that is still open (no FILE_SHARE_DELETE). + _ = outFile.Close() _ = removePartialFile(tmpFilePath) // The partial has just been discarded, so a further attempt starts // clean and no longer needs the server to honour the range. @@ -924,6 +928,13 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string // after filesystem permissions while the disk was perfectly healthy. tracked := &readErrorRecorder{r: source} _, err = xio.Copy(ctx, io.MultiWriter(outFile, progress), tracked) + // Windows cannot rename or remove a file that still has an open handle: + // Go opens files without FILE_SHARE_DELETE, so MoveFileEx / DeleteFile fail + // with a sharing violation while outFile is live. The copy is the last use + // of the handle, so close it before any of the removal/rename paths below — + // the error paths (non-206 resume, user cancel, SHA mismatch) remove the + // partial too, and POSIX unlink works on open files while Windows does not. + _ = outFile.Close() if err != nil { // Detect cancellation via the context (a cause-cancelled read surfaces // the cause, not context.Canceled). Keep the .partial for resume, diff --git a/pkg/downloader/uri_test.go b/pkg/downloader/uri_test.go index aff6ea46d931..d2fbdb2f7a18 100644 --- a/pkg/downloader/uri_test.go +++ b/pkg/downloader/uri_test.go @@ -305,12 +305,15 @@ var _ = Describe("Download Test", func() { mockServer := getMockServer(true) defer mockServer.Close() uri := URI(mockServer.URL) - // Create a partial file + // Create a partial file. The handle must be released before the + // download runs: DownloadFile renames the partial over the target, + // and Windows cannot rename a file with an open handle. tmpFilePath := filePath + ".partial" file, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) Expect(err).ToNot(HaveOccurred()) _, err = file.Write(mockData[0:10000]) Expect(err).ToNot(HaveOccurred()) + Expect(file.Close()).To(Succeed()) err = uri.DownloadFile(filePath, mockDataSha, 1, 1, func(s1, s2, s3 string, f float64) {}) Expect(err).ToNot(HaveOccurred()) }) @@ -319,12 +322,15 @@ var _ = Describe("Download Test", func() { mockServer := getMockServer(false) defer mockServer.Close() uri := URI(mockServer.URL) - // Create a partial file + // Create a partial file. The handle must be released before the + // download runs: DownloadFile removes the partial and Windows cannot + // delete a file with an open handle. tmpFilePath := filePath + ".partial" file, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) Expect(err).ToNot(HaveOccurred()) _, err = file.Write(mockData[0:10000]) Expect(err).ToNot(HaveOccurred()) + Expect(file.Close()).To(Succeed()) err = uri.DownloadFile(filePath, mockDataSha, 1, 1, func(s1, s2, s3 string, f float64) {}) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 0049c6a9f5ad..7f48f043d9b4 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -301,6 +301,7 @@ func (ml *ModelLoader) stopLoadProcess(client *Model, modelID string) { if err := process.Stop(); err != nil { xlog.Warn("failed to stop backend process after failed load", "error", err, "modelID", modelID) } + ml.terminateProcessTreeForProcess(process) ml.cleanupProcessRuntime(process) } diff --git a/pkg/model/process.go b/pkg/model/process.go index 2abfbd1b35d6..353b46599e30 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -30,6 +31,17 @@ var ( ErrModelBusy = errors.New("model is still busy") ) +// processLauncher starts a backend process and tracks it in the process tree. +// Implementations are selected by build tag: processLauncherWindows on +// Windows, processLauncherOther everywhere else. +type processLauncher interface { + launch(proc *process.Process, runtime *backendProcessRuntime, processName string, processArgs []string, workDir string, env []string) error +} + +// backendProcessLauncher is the platform-specific process launcher. It is +// initialized by the platform-specific newLauncher() function. +var backendProcessLauncher processLauncher = newLauncher() + const ( gracefulShutdownTimeout = 30 * time.Second forcedShutdownTimeout = 30 * time.Second @@ -176,17 +188,32 @@ func (ml *ModelLoader) deleteProcess(ctx context.Context, s string, force bool) // expected stop, not a crash (signal-terminated children report -1). ml.stoppingProcs.Store(process, struct{}{}) var localErr error - if err := process.Stop(); err != nil { + if runtime.GOOS == "windows" { + // On Windows, go-processmanager's Stop() waits up to 15s because + // IsAlive() uses Signal(0) which is unreliable on Windows. + // Instead, terminate the process tree directly via the job object + // that was assigned at launch time, then clean up the pidfile. + ml.terminateProcessTreeForProcess(process) + if err := os.Remove(filepath.Join(process.StateDir(), "pid")); err != nil && !os.IsNotExist(err) { + xlog.Debug("failed to remove pidfile", "error", err) + } + store.Delete(s) + ml.cleanupProcessRuntime(process) + } else if err := process.Stop(); err != nil { xlog.Error("(deleteProcess) error while deleting process", "error", err, "model", s) if !process.IsAlive() { // A concurrently crashed/already-reaped process can no longer own // resources even if Stop could not read or signal its PID. + ml.terminateProcessTreeForProcess(process) store.Delete(s) ml.cleanupProcessRuntime(process) } else { localErr = err } } else { + // Stop kills the launcher; the process-tree tracker reaps any backend + // child it spawned so the tree cannot outlive a deliberate unload. + ml.terminateProcessTreeForProcess(process) store.Delete(s) ml.cleanupProcessRuntime(process) } @@ -270,6 +297,22 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string return nil, err } + // Include --addr in processArgs so the launcher passes it to the backend + // process. On Linux proc.Run() uses the stored args from + // grpcControlProcess; on Windows startProcessWindows uses processArgs + // directly and also converts --addr to --addr=. + processArgs := append(args, []string{"--addr", serverAddress}...) + + // The gallery contract names a run.sh stub as the backend entrypoint. On a + // Windows host that stub cannot run (no POSIX shell), so resolveLauncher + // substitutes the bundled run.ps1 through the system PowerShell; elsewhere + // it returns the stub and its arguments unchanged. See + // process_launcher_windows.go / process_launcher_other.go. + processName, processArgs, err := resolveLauncher(grpcProcess, workDir, processArgs) + if err != nil { + return nil, err + } + runtime, err := newBackendProcessRuntime() if err != nil { return nil, err @@ -303,8 +346,8 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string grpcControlProcess := process.New( process.WithStateDir(stateDir), - process.WithName(filepath.Base(grpcProcess)), - process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), + process.WithName(processName), + process.WithArgs(processArgs...), process.WithEnvironment(env...), process.WithWorkDir(workDir), ) @@ -314,7 +357,13 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string ml.wd.AddAddressModelMap(serverAddress, id) } - if err := grpcControlProcess.Run(); err != nil { + // On Windows, startProcessWindows assigns the process to a job object + // immediately after CreateProcess returns, before the launcher can spawn + // children, so the tree cannot outlive a stop. On other platforms, Run() + // starts the process and trackProcessTree is called immediately after; + // the race window is negligible because child processes inherit the + // job/cgroup of their parent. + if err := backendProcessLauncher.launch(grpcControlProcess, runtime, processName, processArgs, workDir, env); err != nil { ml.untrackProcess(grpcControlProcess) runtime.cleanup() return grpcControlProcess, err @@ -336,6 +385,10 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string if err := grpcControlProcess.Stop(); err != nil { xlog.Error("error while shutting down grpc process", "error", err) } + // The wrapper's Stop only kills the launcher; reaping the backend it + // spawned needs the process-tree tracker. Idempotent for processes + // StopAllGRPC already stopped (their runtime is gone from the map). + ml.terminateProcessTreeForProcess(grpcControlProcess) }) go func() { @@ -439,6 +492,19 @@ func (ml *ModelLoader) forgetExitedProcess(id string, p *process.Process) { ml.cleanupProcessRuntime(p) } +// terminateProcessTreeForProcess kills the backend's full process tree using +// the tracker created at launch (see createProcess). Idempotent: the tracker +// makes a concurrent stop racing with this one safe, and the map lookup is a +// no-op once the runtime has been cleaned up. +func (ml *ModelLoader) terminateProcessTreeForProcess(process *process.Process) { + if process == nil { + return + } + if value, ok := ml.processRuntimes.Load(process); ok { + value.(*backendProcessRuntime).terminateProcessTree() + } +} + func (ml *ModelLoader) cleanupProcessRuntime(process *process.Process) { if process == nil { return diff --git a/pkg/model/process_launcher_other.go b/pkg/model/process_launcher_other.go new file mode 100644 index 000000000000..19fcb2b8ae2a --- /dev/null +++ b/pkg/model/process_launcher_other.go @@ -0,0 +1,39 @@ +//go:build !windows + +package model + +import ( + "path/filepath" + "strconv" + + process "github.com/mudler/go-processmanager" +) + +// resolveLauncher runs the gallery-contract run.sh stub directly: every +// non-Windows host ships a POSIX shell that executes it as-is, so neither the +// executable nor the argument vector needs rewriting. The executable is the +// stub's basename because os.StartProcess resolves it against ProcAttr.Dir. +func resolveLauncher(grpcProcess, _ string, args []string) (processName string, processArgs []string, err error) { + return filepath.Base(grpcProcess), args, nil +} + +// processLauncherOther implements processLauncher for non-Windows platforms. +// It starts the process with Run() and then tracks it in the process tree. +type processLauncherOther struct{} + +var _ processLauncher = processLauncherOther{} + +func (processLauncherOther) launch(proc *process.Process, runtime *backendProcessRuntime, processName string, processArgs []string, workDir string, env []string) error { + if err := proc.Run(); err != nil { + return err + } + if pid, err := strconv.Atoi(proc.CurrentPID()); err == nil { + runtime.trackProcessTree(pid) + } + return nil +} + +// newLauncher returns the platform-specific process launcher for non-Windows. +func newLauncher() processLauncher { + return processLauncherOther{} +} diff --git a/pkg/model/process_launcher_other_test.go b/pkg/model/process_launcher_other_test.go new file mode 100644 index 000000000000..3e3d1905077e --- /dev/null +++ b/pkg/model/process_launcher_other_test.go @@ -0,0 +1,32 @@ +//go:build !windows + +package model + +import ( + "path/filepath" + "testing" +) + +func TestResolveLauncherOther(t *testing.T) { + dir := t.TempDir() + + processName, args, err := resolveLauncher("grpc-server", dir, []string{"--addr", "localhost:50051"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedName := filepath.Base("grpc-server") + if processName != expectedName { + t.Fatalf("expected processName %q, got %q", expectedName, processName) + } + + expectedArgs := []string{"--addr", "localhost:50051"} + if len(args) != len(expectedArgs) { + t.Fatalf("expected %d args, got %d: %v", len(expectedArgs), len(args), args) + } + for i := range expectedArgs { + if args[i] != expectedArgs[i] { + t.Fatalf("arg %d: expected %q, got %q", i, expectedArgs[i], args[i]) + } + } +} diff --git a/pkg/model/process_launcher_windows.go b/pkg/model/process_launcher_windows.go new file mode 100644 index 000000000000..df1763c39fe1 --- /dev/null +++ b/pkg/model/process_launcher_windows.go @@ -0,0 +1,95 @@ +//go:build windows + +package model + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + process "github.com/mudler/go-processmanager" +) + +// resolveLauncher rewrites the gallery-contract run.sh launch for a Windows +// host, which has no POSIX shell to execute that stub. On Windows, prefer +// launching llama-cpp-grpc.exe directly: it is the dedicated gRPC server that +// stays running for inference, whereas run.sh/run.ps1 default to +// llama-cpp-cpu-all.exe which loads the model and exits. We do not set +// LLAMACPP_GRPC_SERVERS to select grpc.exe because grpc-server.cpp also reads +// that variable as RPC server addresses (add_rpc_devices), so setting it to "1" +// would make the backend try to connect to endpoint "1" and crash. +// +// run.ps1/run.sh remain fallbacks for backends that don't ship +// llama-cpp-grpc.exe. The lib/ directory is added to PATH in +// startProcessWindows so DLLs are found, mirroring run.ps1's +// "$env:PATH = \"$lib;$env:PATH\"". +func resolveLauncher(grpcProcess, workDir string, args []string) (processName string, processArgs []string, err error) { + // Prefer launching llama-cpp-grpc.exe directly. + grpcBinary := filepath.Join(workDir, "llama-cpp-grpc.exe") + if _, statErr := os.Stat(grpcBinary); statErr == nil { + return grpcBinary, args, nil + } + + // Fall back to run.ps1 via PowerShell: it handles backend binary selection + // and environment setup correctly. + runScript := filepath.Join(workDir, "run.ps1") + if _, statErr := os.Stat(runScript); statErr == nil { + // os.StartProcess resolves the executable against ProcAttr.Dir on + // Windows, so a bare "powershell.exe" would be looked up inside the + // backend's workDir and every launch would fail with file-not-found. + // Resolve the absolute path first (PATH lookup finds System32). + powershell, lookErr := exec.LookPath("powershell.exe") + if lookErr != nil { + return "", nil, fmt.Errorf("could not locate powershell.exe to launch %s: %w", runScript, lookErr) + } + processArgs = append([]string{ + "-NoProfile", + "-ExecutionPolicy", "Bypass", + "-File", runScript, + }, args...) + return powershell, processArgs, nil + } + + // Fall back to run.sh via PowerShell (has environment setup). + runScript = filepath.Join(workDir, "run.sh") + if _, statErr := os.Stat(runScript); statErr == nil { + powershell, lookErr := exec.LookPath("powershell.exe") + if lookErr != nil { + return "", nil, fmt.Errorf("could not locate powershell.exe to launch %s: %w", runScript, lookErr) + } + processArgs = append([]string{ + "-NoProfile", + "-ExecutionPolicy", "Bypass", + "-File", runScript, + }, args...) + return powershell, processArgs, nil + } + + // Last resort: the raw gallery contract path. Resolve it against workDir: + // CreateProcess does not search the child's working directory, so a bare + // basename would be looked up next to local-ai.exe instead of in the + // backend directory. + return filepath.Join(workDir, filepath.Base(grpcProcess)), args, nil +} + +// processLauncherWindows implements processLauncher for Windows. +type processLauncherWindows struct{} + +var _ processLauncher = processLauncherWindows{} + +func (processLauncherWindows) launch(proc *process.Process, runtime *backendProcessRuntime, processName string, processArgs []string, workDir string, env []string) error { + _, err := startProcessWindows(proc, runtime, processName, processArgs, workDir, env) + return err +} + +// newLauncher returns the platform-specific process launcher for Windows. +func newLauncher() processLauncher { + return processLauncherWindows{} +} + +// dirExists reports whether the named directory exists. +func dirExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/pkg/model/process_launcher_windows_test.go b/pkg/model/process_launcher_windows_test.go new file mode 100644 index 000000000000..02e2a84befca --- /dev/null +++ b/pkg/model/process_launcher_windows_test.go @@ -0,0 +1,101 @@ +//go:build windows + +package model + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResolveLauncherWindowsWithRunPs1(t *testing.T) { + // Create a temp directory with a run.ps1 + dir := t.TempDir() + runPs1 := filepath.Join(dir, "run.ps1") + if err := os.WriteFile(runPs1, []byte("# dummy"), 0o644); err != nil { + t.Fatalf("failed to create run.ps1: %v", err) + } + + processName, args, err := resolveLauncher("grpc-server", dir, []string{"--addr", "localhost:50051"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Should resolve powershell.exe via PATH + if processName == "" { + t.Fatal("expected non-empty processName (powershell.exe resolved via PATH)") + } + + expectedArgs := []string{ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", runPs1, + "--addr", "localhost:50051", + } + if len(args) != len(expectedArgs) { + t.Fatalf("expected %d args, got %d: %v", len(expectedArgs), len(args), args) + } + for i := range expectedArgs { + if args[i] != expectedArgs[i] { + t.Fatalf("arg %d: expected %q, got %q", i, expectedArgs[i], args[i]) + } + } +} + +func TestResolveLauncherWindowsWithoutRunPs1(t *testing.T) { + dir := t.TempDir() + // No run.ps1 in dir + + processName, args, err := resolveLauncher("grpc-server", dir, []string{"--addr", "localhost:50051"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // The raw gallery contract path is resolved against workDir: CreateProcess + // does not search the child's working directory, so a bare basename would + // be looked up next to local-ai.exe instead. + if processName != filepath.Join(dir, "grpc-server") { + t.Fatalf("expected processName %q, got %q", filepath.Join(dir, "grpc-server"), processName) + } + + expectedArgs := []string{"--addr", "localhost:50051"} + if len(args) != len(expectedArgs) { + t.Fatalf("expected %d args, got %d: %v", len(expectedArgs), len(args), args) + } + for i := range expectedArgs { + if args[i] != expectedArgs[i] { + t.Fatalf("arg %d: expected %q, got %q", i, expectedArgs[i], args[i]) + } + } +} + +func TestResolveLauncherWindowsPrefersGrpcBinary(t *testing.T) { + dir := t.TempDir() + // The backend ships llama-cpp-grpc.exe: the launcher must start it + // directly, without going through run.ps1 or setting + // LLAMACPP_GRPC_SERVERS (which grpc-server.cpp reads as RPC addresses). + grpcBinary := filepath.Join(dir, "llama-cpp-grpc.exe") + if err := os.WriteFile(grpcBinary, []byte("dummy"), 0o644); err != nil { + t.Fatalf("failed to create llama-cpp-grpc.exe: %v", err) + } + + processName, args, err := resolveLauncher(filepath.Join(dir, "run.sh"), dir, []string{"--addr", "localhost:50051"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if processName != grpcBinary { + t.Fatalf("expected processName %q, got %q", grpcBinary, processName) + } + + expectedArgs := []string{"--addr", "localhost:50051"} + if len(args) != len(expectedArgs) { + t.Fatalf("expected %d args, got %d: %v", len(expectedArgs), len(args), args) + } + for i := range expectedArgs { + if args[i] != expectedArgs[i] { + t.Fatalf("arg %d: expected %q, got %q", i, expectedArgs[i], args[i]) + } + } +} diff --git a/pkg/model/process_runtime.go b/pkg/model/process_runtime.go index b7443ef7d255..71b6143cec54 100644 --- a/pkg/model/process_runtime.go +++ b/pkg/model/process_runtime.go @@ -29,6 +29,9 @@ type backendProcessRuntime struct { once sync.Once // diagnosticsDone closes after the exit watcher has read the state files. diagnosticsDone chan struct{} + // tree tracks the backend's whole process tree. The implementation is + // chosen at build time; see the processTree interface. + tree processTree } func backendRuntimeRoot() string { @@ -88,6 +91,7 @@ func newBackendProcessRuntime() (*backendProcessRuntime, error) { tempDir: tempDir, lock: runtimeLock, diagnosticsDone: make(chan struct{}), + tree: newProcessTree(), }, nil } @@ -130,6 +134,7 @@ func (r *backendProcessRuntime) cleanup() { return } r.once.Do(func() { + r.terminateProcessTree() r.cleanupScratch() if err := r.lock.Unlock(); err != nil { xlog.Warn("Failed to unlock backend process runtime", "dir", r.dir, "error", err) @@ -140,6 +145,28 @@ func (r *backendProcessRuntime) cleanup() { }) } +// trackProcessTree records the OS state needed to later reap the backend's +// whole process tree. It is best-effort: a host that already placed the +// backend in a tree we cannot take over makes assignment fail, which is logged +// as a warning rather than failing the backend load. +func (r *backendProcessRuntime) trackProcessTree(pid int) { + if r == nil || r.tree == nil { + return + } + if err := r.tree.assign(pid); err != nil { + xlog.Warn("Failed to track the backend process tree; an abrupt LocalAI exit may leave orphaned backend processes", "pid", pid, "error", err) + } +} + +// terminateProcessTree kills the backend's whole process tree and releases its +// handles. Idempotent, so concurrent stops of the same backend are safe. +func (r *backendProcessRuntime) terminateProcessTree() { + if r == nil || r.tree == nil { + return + } + r.tree.terminate() +} + func (r *backendProcessRuntime) cleanupScratch() { if r == nil { return diff --git a/pkg/model/process_start_windows.go b/pkg/model/process_start_windows.go new file mode 100644 index 000000000000..79f2a58005e8 --- /dev/null +++ b/pkg/model/process_start_windows.go @@ -0,0 +1,77 @@ +//go:build windows + +package model + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + process "github.com/mudler/go-processmanager" + "github.com/mudler/xlog" + "golang.org/x/sys/windows" +) + +// startProcessWindows starts the backend process with exec.Cmd (which wraps +// CreateProcess), assigns it to the platform's process-tree job object, then +// lets it run. The job assignment happens immediately after Start returns; +// there is a small race window before the process is in the job, but the +// launcher (powershell.exe or the backend binary) needs far longer than that +// to spawn children, and the OS reaps the whole tree once the job handle +// closes regardless. +func startProcessWindows(proc *process.Process, runtime *backendProcessRuntime, processName string, processArgs []string, workDir string, env []string) (pid int, err error) { + // Build command: --addr converted to --addr= for Windows backends + var winArgs []string + for i := 0; i < len(processArgs); i++ { + if processArgs[i] == "--addr" && i+1 < len(processArgs) { + winArgs = append(winArgs, "--addr="+processArgs[i+1]) + i++ + } else { + winArgs = append(winArgs, processArgs[i]) + } + } + + // Add the backend's lib directory to the front of PATH so DLLs are found, + // mirroring run.ps1's "$env:PATH = \"$lib;$env:PATH\"". PATH entries are + // joined with the path-list separator ';' — not the directory separator. + // The existing entry is replaced in place so the child environment never + // carries two PATH values. + if libDir := filepath.Join(workDir, "lib"); dirExists(libDir) { + patched := false + for i, e := range env { + if key, value, ok := strings.Cut(e, "="); ok && strings.EqualFold(key, "PATH") { + env[i] = "PATH=" + libDir + string(os.PathListSeparator) + value + patched = true + break + } + } + if !patched { + env = append(env, "PATH="+libDir+string(os.PathListSeparator)+os.Getenv("PATH")) + } + } + + cmd := exec.Command(processName, winArgs...) + cmd.Dir = workDir + cmd.Env = env + cmd.SysProcAttr = &windows.SysProcAttr{ + CreationFlags: windows.CREATE_NEW_PROCESS_GROUP, + } + + if err := cmd.Start(); err != nil { + return 0, err + } + + pid = cmd.Process.Pid + + // Assign to the job object after start, so any child processes join the job. + runtime.trackProcessTree(pid) + + // Write the pidfile so the process.Process handle (used for IsAlive/Stop) + // can locate the process. + if err = os.WriteFile(filepath.Join(proc.StateDir(), "pid"), []byte(fmt.Sprint(pid)), os.ModePerm); err != nil { + xlog.Debug("failed to write pid file", "error", err) + } + return pid, nil +} diff --git a/pkg/model/process_tree.go b/pkg/model/process_tree.go new file mode 100644 index 000000000000..65e8c5f772e3 --- /dev/null +++ b/pkg/model/process_tree.go @@ -0,0 +1,18 @@ +package model + +// processTree tracks the OS state that lets LocalAI reap a backend's entire +// process tree when the backend is stopped. The mechanics are platform +// specific and chosen at build time: process_tree_windows.go wraps the tree in +// a job object, while process_tree_other.go is a no-op because the process +// manager already owns the tree elsewhere. +// +// On Windows a backend is started through run.ps1, so go-processmanager tracks +// only the PowerShell wrapper's PID; without a job object owning the tree an +// abrupt stop (or local-ai.exe dying) would orphan the real backend binary. +type processTree interface { + // assign captures the process identified by pid so a later terminate reaps + // it together with any children it spawns afterwards. + assign(pid int) error + // terminate kills the tracked process tree and releases its handles. + terminate() +} diff --git a/pkg/model/process_tree_other.go b/pkg/model/process_tree_other.go new file mode 100644 index 000000000000..5f23185f0265 --- /dev/null +++ b/pkg/model/process_tree_other.go @@ -0,0 +1,14 @@ +//go:build !windows + +package model + +// noopProcessTree satisfies processTree off Windows. There the process manager +// starts the backend directly and owns its whole tree, so no extra tracking is +// needed to reap it on stop. +type noopProcessTree struct{} + +func newProcessTree() processTree { return noopProcessTree{} } + +func (noopProcessTree) assign(int) error { return nil } + +func (noopProcessTree) terminate() {} diff --git a/pkg/model/process_tree_windows.go b/pkg/model/process_tree_windows.go new file mode 100644 index 000000000000..4602d0699c33 --- /dev/null +++ b/pkg/model/process_tree_windows.go @@ -0,0 +1,75 @@ +//go:build windows + +package model + +import ( + "fmt" + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +// jobProcessTree implements processTree with a Windows job object configured +// with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE. +type jobProcessTree struct { + once sync.Once + job windows.Handle +} + +func newProcessTree() processTree { return &jobProcessTree{} } + +// assign wraps the process tree rooted at pid in a job object. A child joins +// its parent's job, so assigning the run.ps1 wrapper captures the backend it +// spawns as well. A failure means the host already placed us in a job we +// cannot escape (service managers, some CI runners); the caller then continues +// without the tree-kill guarantee. +func (t *jobProcessTree) assign(pid int) error { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return fmt.Errorf("creating job object: %w", err) + } + + // Terminate every member once the last job handle closes. This is what + // reaps the backend tree even when local-ai.exe itself is killed hard + // (taskkill /F, crash): the OS closes our job handle and fires the kill. + var limit windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION + limit.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&limit)), uint32(unsafe.Sizeof(limit)), + ); err != nil { + _ = windows.CloseHandle(job) + return fmt.Errorf("setting kill-on-close on job object: %w", err) + } + + processHandle, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(pid), + ) + if err != nil { + _ = windows.CloseHandle(job) + return fmt.Errorf("opening backend process %d: %w", pid, err) + } + defer func() { _ = windows.CloseHandle(processHandle) }() + if err := windows.AssignProcessToJobObject(job, processHandle); err != nil { + _ = windows.CloseHandle(job) + return fmt.Errorf("assigning backend process %d to job object: %w", pid, err) + } + + t.job = job + return nil +} + +// terminate kills every process that still belongs to the job (the wrapper and +// any backend child that outlived it), then releases the handle. Safe on an +// empty job and idempotent, so concurrent stops of one backend are safe. +func (t *jobProcessTree) terminate() { + t.once.Do(func() { + if t.job == 0 { + return + } + _ = windows.TerminateJobObject(t.job, 1) + _ = windows.CloseHandle(t.job) + t.job = 0 + }) +} diff --git a/pkg/model/windows_job_test.go b/pkg/model/windows_job_test.go new file mode 100644 index 000000000000..a6165d0c31d9 --- /dev/null +++ b/pkg/model/windows_job_test.go @@ -0,0 +1,115 @@ +//go:build windows + +package model + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +// PROCESS_QUERY_LIMITED_INFORMATION is not exported by x/sys/windows; the raw +// value from winnt.h is stable. +const processQueryLimitedInformation = 0x1000 + +// stillActive mirrors the SDK's STILL_ACTIVE (259): GetExitCodeProcess keeps +// returning it while the process runs. +const stillActive = 0x103 + +func processPIDAlive(pid int) bool { + handle, err := windows.OpenProcess(processQueryLimitedInformation, false, uint32(pid)) + if err != nil { + return false + } + defer func() { _ = windows.CloseHandle(handle) }() + var code uint32 + if err := windows.GetExitCodeProcess(handle, &code); err != nil { + return false + } + return code == stillActive +} + +// TestWindowsJobObjectKillsProcessTree reproduces the run.ps1 shape: a +// PowerShell wrapper that later spawns a child (the "backend binary") and +// keeps running. The wrapper is assigned to the job BEFORE it creates the +// child — the same ordering as createProcess — so the child inherits the job +// membership, and terminating the job must reap the child too. Killing only +// the wrapper would orphan it. +func TestWindowsJobObjectKillsProcessTree(t *testing.T) { + tempDir := t.TempDir() + childPIDFile := filepath.Join(tempDir, "child.pid") + // The wrapper parks on this marker file until the test has placed it in a + // job, mirroring how runs.ps1 spawns the backend only after PowerShell has + // started. Without the gate the child would be created outside the job. + gateFile := filepath.Join(tempDir, "go") + + script := filepath.Join(tempDir, "tree.ps1") + content := fmt.Sprintf( + "while (-not (Test-Path '%s')) { Start-Sleep -Milliseconds 50 }\n"+ + "$child = Start-Process powershell -ArgumentList '-NoProfile','-Command','Start-Sleep','30' -PassThru\n"+ + "[IO.File]::WriteAllText('%s', $child.Id.ToString())\n"+ + "Start-Sleep 60\n", + gateFile, childPIDFile, + ) + if err := os.WriteFile(script, []byte(content), 0o600); err != nil { + t.Fatalf("writing tree.ps1: %v", err) + } + + wrapper := exec.Command("powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script) + if err := wrapper.Start(); err != nil { + t.Fatalf("starting wrapper powershell: %v", err) + } + wrapperPID := wrapper.Process.Pid + defer func() { _ = wrapper.Process.Kill() }() + + // Assign FIRST, mirroring createProcess, before the child (backend) exists. + tree := newProcessTree() + if err := tree.assign(wrapperPID); err != nil { + t.Skipf("cannot assign the wrapper to a job object in this environment (nested non-breakaway job?): %v", err) + } + + // Release the gate; the wrapper now spawns its child inside the job. + if err := os.WriteFile(gateFile, []byte("go"), 0o600); err != nil { + t.Fatalf("touching the gate file: %v", err) + } + + var childPID int + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if raw, err := os.ReadFile(childPIDFile); err == nil { + if pid, perr := strconv.Atoi(strings.TrimSpace(string(raw))); perr == nil { + childPID = pid + break + } + } + time.Sleep(100 * time.Millisecond) + } + if childPID == 0 { + t.Fatal("wrapper did not report a child PID") + } + if !processPIDAlive(wrapperPID) || !processPIDAlive(childPID) { + t.Fatalf("expected wrapper(%d) and child(%d) alive before termination; wrapper alive=%v child alive=%v", + wrapperPID, childPID, processPIDAlive(wrapperPID), processPIDAlive(childPID)) + } + + tree.terminate() + + // Wait reads the wrapper's exit status; it returns once the job kill lands. + _ = wrapper.Wait() + deadline = time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if !processPIDAlive(wrapperPID) && !processPIDAlive(childPID) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Errorf("terminating the job did not reap the whole tree: wrapper alive=%v child alive=%v", + processPIDAlive(wrapperPID), processPIDAlive(childPID)) +} diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 23de67a2781e..b2ccc2c1fb99 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -25,6 +25,7 @@ const ( darwinX86 = "darwin-x86" metal = "metal" vulkan = "vulkan" + windows = "windows" nvidiaCuda13 = "nvidia-cuda-13" nvidiaCuda12 = "nvidia-cuda-12" @@ -36,15 +37,16 @@ const ( defaultRunFile = "/run/localai/capability" // Backend detection tokens (private) - backendTokenDarwin = "darwin" - backendTokenMLX = "mlx" - backendTokenMetal = "metal" - backendTokenL4T = "l4t" - backendTokenCUDA = "cuda" - backendTokenROCM = "rocm" - backendTokenHIP = "hip" - backendTokenSYCL = "sycl" - backendTokenCPU = "cpu" + backendTokenDarwin = "darwin" + backendTokenMLX = "mlx" + backendTokenMetal = "metal" + backendTokenL4T = "l4t" + backendTokenCUDA = "cuda" + backendTokenROCM = "rocm" + backendTokenHIP = "hip" + backendTokenSYCL = "sycl" + backendTokenCPU = "cpu" + backendTokenWindows = "windows" // Engine names (private). Unlike the tokens above these are whole backend // identities as a gallery entry's `backend:` field spells them, not build @@ -117,6 +119,9 @@ var backendBuildTagPreferenceRules = []backendPreferenceRule{ {metal, []string{backendTokenMetal, backendTokenCPU}}, {darwinX86, []string{darwinX86, backendTokenCPU}}, {vulkan, []string{vulkan, backendTokenCPU}}, + // Windows ships native windows/amd64 backend builds; prefer them over the + // Linux CPU builds, which cannot run on a Windows host. + {windows, []string{backendTokenWindows, backendTokenCPU}}, } // defaultBackendBuildTagTokens is what a host with no matching rule prefers. @@ -184,6 +189,11 @@ var engineNamePreferenceRules = []backendPreferenceRule{ // IsBackendCompatible admits any darwin-tokened engine on this capability // and will not drop it. {darwinX86, []string{engineLlamaCpp, engineVLLM, engineSGLang}}, + // A Windows host runs exactly the native llama.cpp build; vLLM and SGLang + // do not publish windows/amd64 images, so ranking them above llama.cpp + // (or even equal to it) could auto-select an engine a Windows host cannot + // run whenever a variant set is offered. + {windows, []string{engineLlamaCpp}}, } // defaultEnginePreferenceTokens is empty on purpose. It is now only reached by @@ -384,6 +394,15 @@ func (s *SystemState) getSystemCapabilities() string { return s.systemCapabilities } + // Windows backends are published as native windows/amd64 OCI images, so a + // Windows host gets its own capability to select them deterministically + // instead of falling through to "default" and racing the Linux CPU builds. + if runtime.GOOS == "windows" { + xlog.Info("Using windows capability", "env", capabilityEnv) + s.systemCapabilities = windows + return s.systemCapabilities + } + // If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { if s.GPUVendor == Nvidia { @@ -505,6 +524,11 @@ func (s *SystemState) IsBackendCompatible(name, uri string) bool { return capability == metal || capability == darwinX86 } + // Check for Windows-specific backends (native windows/amd64 builds) + if strings.Contains(combined, backendTokenWindows) { + return capability == windows + } + // Check for NVIDIA L4T-specific backends (arm64 Linux with NVIDIA GPU) // This must be checked before the general NVIDIA check as L4T backends // may also contain "cuda" or "nvidia" in their names diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index 0844e52f44a5..523c29dd5f82 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -19,8 +19,8 @@ var _ = Describe("getSystemCapabilities", func() { ) BeforeEach(func() { - if runtime.GOOS == "darwin" { - Skip("darwin short-circuits before reaching CUDA logic") + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + Skip("platform short-circuits before reaching CUDA logic") } origEnv = os.Getenv(capabilityEnv) @@ -168,6 +168,7 @@ var _ = Describe("BackendPreferenceTokens", func() { Entry("metal", metal, []string{"metal", "cpu"}), Entry("darwin-x86", darwinX86, []string{"darwin-x86", "cpu"}), Entry("vulkan", vulkan, []string{"vulkan", "cpu"}), + Entry("windows", windows, []string{"windows", "cpu"}), Entry("unknown capability", "some-future-accelerator", []string{"cpu"}), ) @@ -175,7 +176,7 @@ var _ = Describe("BackendPreferenceTokens", func() { // The generic form of the lock above: whatever anyone adds to the build // tag table later, an engine name in it is a merged vocabulary. engineNames := []string{"vllm", "sglang", "llama-cpp", "mlx"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, "default"} { + for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, windows, "default"} { Expect(tokensFor(capability)).ToNot(ContainElements(engineNames), "capability %q leaked an engine name into the build tag table", capability) } @@ -235,8 +236,8 @@ var _ = Describe("EnginePreferenceTokens", func() { It("names only engines, never a build tag", func() { // The mirror of the lock on BackendPreferenceTokens. A build tag here // matches no gallery `backend:` value and silently disables ranking. - buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86} { + buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86", "windows"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86, windows} { Expect(tokensFor(capability)).ToNot(ContainElements(buildTags), "capability %q leaked a build tag into the engine table", capability) } @@ -253,6 +254,10 @@ var _ = Describe("EnginePreferenceTokens", func() { Expect(tokensFor(darwinX86)).To(Equal([]string{"llama-cpp", "vllm", "sglang"})) }) + It("prefers llama.cpp on windows, the only engine with native windows builds", func() { + Expect(tokensFor(windows)).To(Equal([]string{"llama-cpp"})) + }) + It("ranks the GPU serving engines below llama.cpp rather than leaving them tied", func() { // Enumerating them is what stops download size deciding between vLLM and // SGLang once llama.cpp is out of the running. diff --git a/scripts/build/llama-cpp-windows.ps1 b/scripts/build/llama-cpp-windows.ps1 new file mode 100644 index 000000000000..fd2dc74119b6 --- /dev/null +++ b/scripts/build/llama-cpp-windows.ps1 @@ -0,0 +1,164 @@ +#!/usr/bin/env pwsh + +# PowerShell entry point for the native windows/amd64 llama-cpp backend build, +# driven from the Makefile on native Windows hosts (and on Windows CI via the +# msys2 shell, where powershell.exe is still reachable): +# +# make backends/llama-cpp-windows +# +# GNU make on Windows runs recipe lines through sh (Git for Windows), where +# /ucrt64/bin does not resolve and the mingw toolchain does not exist. This +# script hands the build to a real MSYS2 UCRT64 bash instead. It locates the +# MSYS2 install (MSYS2_ROOT, the standard install dirs, or any PATH entry that +# is really an MSYS2 root), installs MSYS2 + the toolchain on first use -- only +# after asking for confirmation -- and aborts with a non-zero status when the +# user declines, so the Makefile recipe stops before the OCI-image packaging +# step. The sh script's own re-exec dispatch (scripts/build/llama-cpp-windows.sh) +# is bypassed deliberately: starting under the real MSYS2 bash makes the +# /ucrt64/bin check take the direct branch. + +[CmdletBinding()] +param() + +$ErrorActionPreference = "Stop" + +# scripts/build -> repo root +$RepoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..\..")) + +# Toolchain package set, mirroring the CI setup in +# .github/workflows/backend_build_windows.yml. A fresh winget MSYS2 is a bare +# base system: pacman must install these before the script's first configure. +$ToolchainPackages = @( + 'git', 'make', 'cmake', 'mingw-w64-ucrt-x86_64-cmake', 'ninja', 'pkg-config', + 'perl', 'patch', 'unzip', 'curl', + 'mingw-w64-ucrt-x86_64-gcc', 'mingw-w64-ucrt-x86_64-gcc-libs', + 'mingw-w64-ucrt-x86_64-binutils', 'mingw-w64-ucrt-x86_64-ccache', + 'mingw-w64-ucrt-x86_64-vulkan-headers', 'mingw-w64-ucrt-x86_64-vulkan-loader', + 'mingw-w64-ucrt-x86_64-spirv-headers', 'mingw-w64-x86_64-shaderc' +) + +function ConvertTo-WindowsPath { + param([Parameter(Mandatory = $true)][string]$Path) + $Path = $Path.Trim().Trim('"') + # MSYS-style /c/Program Files/... (as exported by the Makefile's sh shell) + # has no drive letter for .NET; rewrite to C:\Program Files\... + if ($Path -match '^/([a-zA-Z])/(.+)$') { + $Path = "$($matches[1]):\$($matches[2])" + } + $Path = $Path -replace '/', '\' + return $Path.TrimEnd('\') +} + +function Test-PathSafe { + param([Parameter(Mandatory = $true)][string]$Path) + try { return (Test-Path -LiteralPath $Path) } + catch { return $false } +} + +function Find-Msys2Root { + # A bare usr\bin\bash.exe is not enough of a discriminator: Git for Windows + # ships one too, and its "root" has no pacman.conf. Requiring etc/pacman.conf + # keeps the PATH scan from dispatching the build into Git bash. + $candidates = New-Object System.Collections.Generic.List[string] + if ($env:MSYS2_ROOT) { $candidates.Add((ConvertTo-WindowsPath $env:MSYS2_ROOT)) } + foreach ($p in @('C:\msys64', 'C:\Program Files\MSYS2')) { $candidates.Add($p) } + foreach ($dir in ($env:PATH -split ';')) { + if ([string]::IsNullOrWhiteSpace($dir)) { continue } + $nd = ConvertTo-WindowsPath $dir + if (Test-PathSafe ([System.IO.Path]::Combine($nd, 'bash.exe'))) { + # usr\bin\bash.exe -> root = two levels up from the PATH entry. + $candidates.Add([System.IO.Path]::GetFullPath([System.IO.Path]::Combine($nd, '..', '..'))) + } + } + foreach ($c in $candidates) { + if ((Test-PathSafe (Join-Path $c 'usr\bin\bash.exe')) -and (Test-PathSafe (Join-Path $c 'etc\pacman.conf'))) { + return $c + } + } + return $null +} + +function Invoke-BashInMsys { + param( + [Parameter(Mandatory = $true)][string]$MsysRoot, + [Parameter(Mandatory = $true)][string]$Command + ) + # MSYSTEM selects the UCRT64 toolchain; a non-login invocation keeps the + # inherited Windows PATH (Go, Git) and the working directory, mirroring the + # sh script's own re-exec behavior. + $env:MSYSTEM = 'UCRT64' + $env:MSYS2_PATH_TYPE = 'inherit' + & (Join-Path $MsysRoot 'usr\bin\bash.exe') -lc $Command + $code = $LASTEXITCODE + if ($code -ne 0) { + Write-Host "ERROR: command failed under MSYS2 (exit $code)" -ForegroundColor Red + exit $code + } +} + +function Test-ToolchainComplete { + param([Parameter(Mandatory = $true)][string]$MsysRoot) + $pkgs = $ToolchainPackages -join ' ' + $env:MSYSTEM = 'UCRT64' + $env:MSYS2_PATH_TYPE = 'inherit' + & (Join-Path $MsysRoot 'usr\bin\bash.exe') -lc "pacman -Q $pkgs >/dev/null 2>&1" + return ($LASTEXITCODE -eq 0) +} + +function Install-Toolchain { + param([Parameter(Mandatory = $true)][string]$MsysRoot) + $pkgs = $ToolchainPackages -join ' ' + Invoke-BashInMsys $MsysRoot "pacman -Sy --needed --noconfirm $pkgs" + if (-not (Test-ToolchainComplete $MsysRoot)) { + Write-Host "ERROR: MSYS2 toolchain install finished but packages are still missing" -ForegroundColor Red + exit 1 + } +} + +function Install-Msys2 { + $answer = Read-Host 'MSYS2 was not found. Install MSYS2 and the mingw-w64 UCRT64 toolchain now (hundreds of MB)? [y/N]' + if ($answer -notmatch '^[Yy]$') { + Write-Host "Aborted. Install MSYS2 manually (winget install --id MSYS2.MSYS2) and retry 'make backends/llama-cpp-windows'." + exit 1 + } + if (-not (Get-Command winget -ErrorAction SilentlyContinue)) { + Write-Host 'ERROR: winget is not available; install MSYS2 manually from https://www.msys2.org/ and retry.' -ForegroundColor Red + exit 1 + } + Write-Host '==> Installing MSYS2 via winget ...' + winget install --id MSYS2.MSYS2 --exact --accept-package-agreements --accept-source-agreements --disable-interactivity + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: winget install of MSYS2 failed (exit $LASTEXITCODE)" -ForegroundColor Red + exit 1 + } + $root = Find-Msys2Root + if (-not $root) { + Write-Host 'ERROR: MSYS2 was installed but its root could not be located; install it manually and retry.' -ForegroundColor Red + exit 1 + } + return $root +} + +function Invoke-Build { + param([Parameter(Mandatory = $true)][string]$MsysRoot) + # forward-slash form keeps MSYS2 bash from reading the Windows backslashes as + # escape sequences. + $repo = $RepoRoot -replace '\\', '/' + Invoke-BashInMsys $MsysRoot "cd '$repo' && exec ./scripts/build/llama-cpp-windows.sh" +} + +$MsysRoot = Find-Msys2Root +if (-not $MsysRoot) { + $MsysRoot = Install-Msys2 +} + +if (-not (Test-ToolchainComplete $MsysRoot)) { + $answer = Read-Host 'MSYS2 found but the mingw-w64 UCRT64 toolchain is missing. Download and install the build packages (hundreds of MB)? [y/N]' + if ($answer -notmatch '^[Yy]$') { + Write-Host 'Aborted. Install the toolchain packages manually and retry.' + exit 1 + } + Install-Toolchain $MsysRoot +} + +Invoke-Build $MsysRoot \ No newline at end of file diff --git a/scripts/build/llama-cpp-windows.sh b/scripts/build/llama-cpp-windows.sh new file mode 100644 index 000000000000..c842c7314d7e --- /dev/null +++ b/scripts/build/llama-cpp-windows.sh @@ -0,0 +1,837 @@ +#!/bin/bash + +# Builds the native windows/amd64 llama-cpp backend image on a windows-latest +# GitHub runner under MSYS2 (UCRT64). Driven from the top-level Makefile via: +# +# make backends/llama-cpp-windows +# +# Unlike Linux (prebuilt base-grpc images) and Darwin (Homebrew gRPC), a +# Windows runner has neither, so gRPC is compiled from source into +# backend/cpp/grpc/installed_packages -- the exact layout the llama-cpp +# Makefile's BUILD_GRPC_FOR_BACKEND_LLAMA path expects, so the flags it would +# pass to cmake (absl_DIR / Protobuf_DIR / utf8_range_DIR / gRPC_DIR) resolve +# to real directories. +# +# This script drives cmake directly instead of the llama-cpp Makefile's +# llama-cpp-cpu-all / llama-cpp-grpc / llama-cpp-fallback targets: those copy +# binaries around as bare `grpc-server` paths, which MSYS2's cp cannot resolve +# to grpc-server.exe. The cmake flags below mirror exactly what those targets +# would pass (see backend/cpp/llama-cpp/Makefile). +# +# The build host also has no docker daemon, so the result is packaged as an OCI +# tar (--platform windows/amd64) that LocalAI's `backends install` extracts and +# runs as a native process -- see pkg/model/process.go (run.ps1) and +# backend/cpp/llama-cpp/run.ps1. + +set -ex + +# --------------------------------------------------------------------------- +# Auto-dispatch into MSYS2 (UCRT64). +# +# `make backends/llama-cpp-windows` runs this script through the recipe shell. +# GNU make for Windows falls back to cmd.exe unless it finds sh; the Makefile's +# OS-detection block (top of file) then points SHELL at Git for Windows' sh, so +# a local run normally starts under a MINGW64 bash. That bash cannot resolve +# /ucrt64/bin (the path maps to C:\Program Files\Git\ucrt64, which does not +# exist) and nothing below can build without it. Re-exec under the real MSYS2 +# bash instead. A non-login invocation deliberately keeps the inherited working +# directory (the repo root) and the Windows PATH (Go, Git); `bash -l` would +# reset PATH and cd to $HOME. MSYSTEM is not a usable discriminator here (Git +# bash reports MINGW64 too) - the existence of /ucrt64/bin under MSYS2's own +# runtime is the discriminator. +if [ ! -d /ucrt64/bin ]; then + if [ -x /c/msys64/usr/bin/bash.exe ]; then + echo "==> not running under MSYS2; re-exec'ing under C:/msys64 ..." >&2 + SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) + # Run the script by name, not `exec bash "$0"`: a bare `bash` here would + # resolve to Git for Windows' bash via the inherited PATH (re-dispatch loop); + # the shebang's /bin/bash is an absolute MSYS2 path and lands on the real + # MSYS2 bash, where the /ucrt64/bin check below takes the direct branch. + # + # Mirror msys2_shell.cmd -ucrt64 -use-full-path: a LOGIN bash with + # MSYSTEM=UCRT64 + MSYS2_PATH_TYPE=inherit. This is the one configuration in + # which MSYS2's own git works when launched from a foreign shell (a plain + # non-login `bash.exe -c` inherits the Git-for-Windows mount table and its + # https remote helper dies with "fatal: remote helper 'https' aborted + # session"). `bash -l` resets PATH to MSYS2 defaults and drops `go` (it is + # on the Windows PATH but GOROOT is unset), so re-add the bin dir that the + # invoking shell resolved before the profile runs. + GO_DIR=$(command -v go 2>/dev/null) + [ -n "$GO_DIR" ] && GO_DIR=$(dirname "$GO_DIR") + # The login profile rebuilds the environment from a whitelist and drops + # Windows vars the MSYS2 runtime does not know (USERPROFILE etc.), so + # carry the ones go/ccache need over from the invoking shell. + exec env MSYSTEM=UCRT64 MSYS2_PATH_TYPE=inherit /c/msys64/usr/bin/bash.exe -l -c \ + "export PATH='$GO_DIR':\$PATH; export USERPROFILE='$USERPROFILE'; cd '$REPO_ROOT' && exec ./scripts/build/llama-cpp-windows.sh" + else + echo "ERROR: this script must run under MSYS2 UCRT64 (C:\\msys64)." >&2 + echo " Install MSYS2 (winget install MSYS2.MSYS2) and retry 'make backends/llama-cpp-windows'." >&2 + exit 1 + fi +fi + +ROOT=$(pwd) +IMAGE_NAME="${IMAGE_NAME:-localai/llama-cpp-windows}" +PLATFORMARCH="${PLATFORMARCH:-windows/amd64}" + +# nproc exists on MSYS2; fall back like the llama-cpp Makefile does otherwise. +# Honor an explicit JOBS override (e.g. `JOBS=4 make backends/llama-cpp-windows`): +# on memory-limited Windows hosts -j$(nproc) can spike past available RAM and +# several parallel g++ processes get killed mid-compile with no error text +# (silent "Error 1" on unrelated .obj targets). +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 1)}" +export JOBS + +# Resolve cmake to the native UCRT64 build (/ucrt64/bin/cmake), not MSYS2's +# /usr/bin/cmake. The MSYS/Cygwin cmake reports UNIX=TRUE, so gRPC's +# `if(UNIX)` block in CMakeLists.txt appends ${CMAKE_DL_LIBS} m rt to every +# target's link line and every gRPC executable dies with "cannot find -ldl / +# -lrt" (no such libraries on Windows). The native cmake also sets MINGW +# (gRPC then defines -D_WIN32_WINNT=0x600 itself) and keeps plain exe names +# (the MSYS cmake emits protoc-24.3.0.exe). Everything else the script needs +# (make, git, perl, curl, unzip) lives in /usr/bin, so only cmake's resolution +# changes. +export PATH="/ucrt64/bin:$PATH" + +# Handle zlib conflicts between system MSYS2 and gRPC's bundled static zlib +SYS_ZLIB_DLLA="/ucrt64/lib/libz.dll.a" +SYS_ZLIB_DLLA_BAK="/ucrt64/lib/libz.dll.a.bak" +backup_sys_zlib() { + if [ -f "$SYS_ZLIB_DLLA" ] && [ ! -f "$SYS_ZLIB_DLLA_BAK" ]; then + mv "$SYS_ZLIB_DLLA" "$SYS_ZLIB_DLLA_BAK" + echo "==> Backed up system $SYS_ZLIB_DLLA" + fi +} +restore_sys_zlib() { + if [ -f "$SYS_ZLIB_DLLA_BAK" ] && [ ! -f "$SYS_ZLIB_DLLA" ]; then + mv "$SYS_ZLIB_DLLA_BAK" "$SYS_ZLIB_DLLA" + echo "==> Restored system $SYS_ZLIB_DLLA" + fi +} + +# The MSYS2 runtime does not export PROCESSOR_ARCHITECTURE, so cmake's Windows +# host detection yields an empty CMAKE_SYSTEM_PROCESSOR and ggml treats the +# machine as UNKNOWN (GGML_CPU_ALL_VARIANTS then refuses to configure). This +# backend is x86_64-only. +export PROCESSOR_ARCHITECTURE=AMD64 + +# go.exe resolves GOPATH (default %USERPROFILE%\go) and its build cache +# (%LocalAppData%\go-build) through Windows env vars the login profile drops; +# ccache needs a user-profile dir too. Fall back to the MSYS2 home for direct +# launches that forgot to set it. +export USERPROFILE="${USERPROFILE:-$(cygpath -w "$HOME")}" +export GOCACHE="${GOCACHE:-$(cygpath -w "$HOME")/go-build}" +mkdir -p "$GOCACHE" + +# ggml-vulkan compiles its shaders at build time with glslc, and CMake's +# FindVulkan declares it a REQUIRED component. MSYS2 only ships glslc in the +# mingw64 variant of shaderc (there is no ucrt64 build); it is a standalone +# tool, so its prefix can sit next to the UCRT64 one. When a Vulkan SDK +# (VULKAN_SDK, found by FindVulkan through the registry on Windows) already +# provides glslc, leave PATH alone. +# +# The mingw64 dir must be APPENDED, never prepended: the UCRT64-built tools +# (protoc, gcc, ...) resolve their runtime DLLs through PATH, and a mingw64 +# libstdc++/libgcc_s_seh placed ahead of /ucrt64/bin makes them load the wrong +# ABI runtime and die silently (protoc exits 127 mid-build). +if ! command -v glslc >/dev/null 2>&1 && [ -x /mingw64/bin/glslc.exe ]; then + export PATH="$PATH:/mingw64/bin" +fi + +# ggml-vulkan runs find_package(SPIRV-Headers CONFIG REQUIRED) at configure +# time; MSYS2 ships the config as mingw-w64-ucrt-x86_64-spirv-headers. +# Also ensure common image libraries (JPEG) needed by some components are installed. +if [ ! -f /ucrt64/share/cmake/SPIRV-Headers/SPIRV-HeadersConfig.cmake ] || \ + [ ! -f /ucrt64/lib/libjpeg.a ] && [ ! -f /ucrt64/lib/libjpeg.dll.a ]; then + echo "==> Installing required MSYS2 UCRT64 dependencies (SPIRV-Headers, libjpeg-turbo, etc.)" + pacman -S --noconfirm mingw-w64-ucrt-x86_64-spirv-headers mingw-w64-ucrt-x86_64-libjpeg-turbo mingw-w64-ucrt-x86_64-libpng mingw-w64-ucrt-x86_64-tiff 2>/dev/null || true +fi +# Ensure libjpeg is available even if condition logic was odd +if [ ! -f /ucrt64/lib/libjpeg.dll.a ] && [ ! -f /ucrt64/lib/libturbojpeg.dll.a ] && [ ! -f /ucrt64/lib/libjpeg.a ]; then + pacman -S --noconfirm mingw-w64-ucrt-x86_64-libjpeg-turbo 2>/dev/null || true +fi + +# actions/setup-go adds go to the runner PATH but only exports GOROOT for Go +# < 1.9; the msys2 shell resets PATH, so `go` is not reachable here. The +# workflow resolves the install dir (GO_TOOLCHAIN_ROOT) with the default shell +# and hands it over; GOROOT is the fallback for standalone runs. Re-add the +# bin dir (translated to an msys path) unless a go binary already resolves. +GO_BIN_DIR="${GO_TOOLCHAIN_ROOT:-$GOROOT}" +if ! command -v go >/dev/null 2>&1 && [ -n "$GO_BIN_DIR" ]; then + GO_BIN_DIR="$GO_BIN_DIR/bin" + case "$GO_BIN_DIR" in + [A-Za-z]:*) GO_BIN_DIR=$(cygpath -u "$GO_BIN_DIR") ;; + esac + export PATH="$GO_BIN_DIR:$PATH" +fi + +# --------------------------------------------------------------------------- +# gRPC from source (backend/cpp/grpc/installed_packages) +# --------------------------------------------------------------------------- +GRPC_DIR="$ROOT/backend/cpp/grpc" +mkdir -p "$GRPC_DIR/grpc_repo/grpc" "$GRPC_DIR/grpc_build" +cd "$GRPC_DIR/grpc_repo/grpc" +git init -q +# Re-runnable: a previous (possibly failed) run may already have wired origin. +git remote remove origin 2>/dev/null || true +git remote add origin https://github.com/grpc/grpc.git +git fetch origin refs/tags/v1.59.0:refs/tags/v1.59.0 --depth 1 +git checkout -q v1.59.0 +git submodule update --init --recursive --depth 1 --single-branch + +# abseil's cctz win32 local-time-zone path (USE_WIN32_LOCAL_TIME_ZONE) uses +# WindowsCreateStringReference / WindowsDeleteString / WindowsGetStringRawBuffer +# from , which the MSVC SDK pulls in transitively but mingw-w64's +# roapi.h/windows.globalization.h do not. Patch the one TU to include it +# explicitly; without this the whole gRPC build dies at time_zone_lookup.cc. +# (\R matches either LF or CRLF since git may have normalized line endings.) +perl -0pi -e 's/(#include \R#include \R)(#endif)/$1#include \n$2/' \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc + +# c-ares (bundled with gRPC 1.59.0) has a broken CMake build under mingw-w64 +# UCRT64 + CMake 4.x. Its CMakeLists uses CHECK_INCLUDE_FILES to detect +# winsock2.h / ws2tcpip.h / windows.h on WIN32, but the probes run BEFORE +# CMAKE_REQUIRED_DEFINITIONS is set (so without WIN32_LEAN_AND_MEAN), causing +# windows.h to pull in winsock.h which conflicts with winsock2.h. When the +# checks fail, CMAKE_EXTRA_INCLUDE_FILES stays empty, so the downstream +# CARES_TYPE_EXISTS / CHECK_SYMBOL_EXISTS probes for SOCKET, struct +# sockaddr_in6, struct addrinfo, recv, send etc. all fail too. That leaves +# the generated ares_config.h without HAVE_RECV / HAVE_SEND / +# HAVE_STRUCT_SOCKADDR_IN6 / HAVE_STRUCT_ADDRINFO / RECV_TYPE_* / SEND_TYPE_*, +# which trips setup_once.h's compile-time Error guards and makes ares_ipv6.h +# redefine structs already in ws2tcpip.h. +# +# Pre-seeding the cache vars via -D didn't work: CMake 4.x's CHECK_INCLUDE_FILES +# still runs and overwrites them. Instead, patch the ares_config.h.cmake +# template to append Windows-specific forced defines at the end. This way, +# regardless of what CMake detection produces, the generated ares_config.h +# always has the correct Windows values. +cat >> third_party/cares/cares/src/lib/ares_config.h.cmake <<'CARES_WIN32_FIX' + +/* ---- LocalAI mingw-w64 build fix ---- + * The CMake detection in this old c-ares (pinned by gRPC 1.59.0) doesn't + * reliably detect Windows socket types and functions under mingw-w64's + * UCRT64 headers + CMake 4.x. Force the known-correct values on _WIN32 + * so the build doesn't trip setup_once.h's Error guards or ares_ipv6.h's + * struct redefinitions. Values mirror config-win32.h (the hand-crafted + * config c-ares uses for non-CMake Windows builds). */ +#ifdef _WIN32 +#ifndef HAVE_WINDOWS_H +#define HAVE_WINDOWS_H 1 +#endif +#ifndef HAVE_WINSOCK_H +#define HAVE_WINSOCK_H 1 +#endif +#ifndef HAVE_WINSOCK2_H +#define HAVE_WINSOCK2_H 1 +#endif +#ifndef HAVE_WS2TCPIP_H +#define HAVE_WS2TCPIP_H 1 +#endif +#ifndef HAVE_ASSERT_H +#define HAVE_ASSERT_H 1 +#endif +#ifndef HAVE_ERRNO_H +#define HAVE_ERRNO_H 1 +#endif +#ifndef HAVE_GETOPT_H +#define HAVE_GETOPT_H 1 +#endif +#ifndef HAVE_LIMITS_H +#define HAVE_LIMITS_H 1 +#endif +#ifndef HAVE_PROCESS_H +#define HAVE_PROCESS_H 1 +#endif +#ifndef HAVE_SIGNAL_H +#define HAVE_SIGNAL_H 1 +#endif +#ifndef HAVE_TIME_H +#define HAVE_TIME_H 1 +#endif +#ifndef HAVE_UNISTD_H +#define HAVE_UNISTD_H 1 +#endif +#ifndef HAVE_STDBOOL_H +#define HAVE_STDBOOL_H 1 +#endif +#ifndef HAVE_STDINT_H +#define HAVE_STDINT_H 1 +#endif +#ifndef HAVE_STDLIB_H +#define HAVE_STDLIB_H 1 +#endif +#ifndef HAVE_STRING_H +#define HAVE_STRING_H 1 +#endif +#ifndef HAVE_SOCKLEN_T +#define HAVE_SOCKLEN_T 1 +#endif +#ifndef HAVE_TYPE_SOCKET +#define HAVE_TYPE_SOCKET 1 +#endif +#ifndef HAVE_BOOL_T +#define HAVE_BOOL_T 1 +#endif +#ifndef HAVE_SSIZE_T +#define HAVE_SSIZE_T 1 +#endif +#ifndef HAVE_LONGLONG +#define HAVE_LONGLONG 1 +#endif +#ifndef HAVE_SIG_ATOMIC_T +#define HAVE_SIG_ATOMIC_T 1 +#endif +#ifndef HAVE_STRUCT_ADDRINFO +#define HAVE_STRUCT_ADDRINFO 1 +#endif +#ifndef HAVE_STRUCT_IN6_ADDR +#define HAVE_STRUCT_IN6_ADDR 1 +#endif +#ifndef HAVE_STRUCT_SOCKADDR_IN6 +#define HAVE_STRUCT_SOCKADDR_IN6 1 +#endif +#ifndef HAVE_STRUCT_SOCKADDR_STORAGE +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 +#endif +#ifndef HAVE_STRUCT_TIMEVAL +#define HAVE_STRUCT_TIMEVAL 1 +#endif +#ifndef HAVE_AF_INET6 +#define HAVE_AF_INET6 1 +#endif +#ifndef HAVE_PF_INET6 +#define HAVE_PF_INET6 1 +#endif +#ifndef HAVE_FIONBIO +#define HAVE_FIONBIO 1 +#endif +#ifndef HAVE_CLOSESOCKET +#define HAVE_CLOSESOCKET 1 +#endif +#ifndef HAVE_CONNECT +#define HAVE_CONNECT 1 +#endif +#ifndef HAVE_FREEADDRINFO +#define HAVE_FREEADDRINFO 1 +#endif +#ifndef HAVE_GETADDRINFO +#define HAVE_GETADDRINFO 1 +#endif +#ifndef HAVE_GETHOSTBYADDR +#define HAVE_GETHOSTBYADDR 1 +#endif +#ifndef HAVE_GETHOSTBYNAME +#define HAVE_GETHOSTBYNAME 1 +#endif +#ifndef HAVE_GETHOSTNAME +#define HAVE_GETHOSTNAME 1 +#endif +#ifndef HAVE_GETNAMEINFO +#define HAVE_GETNAMEINFO 1 +#endif +#ifndef HAVE_GETTIMEOFDAY +#define HAVE_GETTIMEOFDAY 1 +#endif +#ifndef HAVE_INET_NTOP +#define HAVE_INET_NTOP 1 +#endif +#ifndef HAVE_INET_PTON +#define HAVE_INET_PTON 1 +#endif +#ifndef HAVE_IOCTLSOCKET +#define HAVE_IOCTLSOCKET 1 +#endif +/* NOTE: HAVE_IOCTL_FIONBIO is deliberately NOT defined. c-ares's + * setsocknonblock() prefers HAVE_IOCTL_FIONBIO (POSIX ioctl(FIONBIO)) over + * HAVE_IOCTLSOCKET_FIONBIO (Windows ioctlsocket(FIONBIO)), and the POSIX + * branch calls ioctl() which does not exist on Windows. config-win32.h — + * the hand-crafted config c-ares uses for non-CMake Windows builds — also + * omits it, defining only HAVE_IOCTLSOCKET_FIONBIO. */ +#ifndef HAVE_IOCTLSOCKET_FIONBIO +#define HAVE_IOCTLSOCKET_FIONBIO 1 +#endif +#ifndef HAVE_RECV +#define HAVE_RECV 1 +#endif +#ifndef HAVE_RECVFROM +#define HAVE_RECVFROM 1 +#endif +#ifndef HAVE_SEND +#define HAVE_SEND 1 +#endif +#ifndef HAVE_SETSOCKOPT +#define HAVE_SETSOCKOPT 1 +#endif +#ifndef HAVE_SOCKET +#define HAVE_SOCKET 1 +#endif +#ifndef HAVE_STRDUP +#define HAVE_STRDUP 1 +#endif +#ifndef HAVE_STRICMP +#define HAVE_STRICMP 1 +#endif +#ifndef HAVE_STRNICMP +#define HAVE_STRNICMP 1 +#endif +#ifndef HAVE_GETENV +#define HAVE_GETENV 1 +#endif +/* HAVE_IOCTL_FIONBIO is intentionally absent and HAVE_IOCTLSOCKET_FIONBIO + * is already defined above; see the NOTE above the first + * HAVE_IOCTLSOCKET_FIONBIO define. */ +#ifndef HAVE_GETADDRINFO_THREADSAFE +#define HAVE_GETADDRINFO_THREADSAFE 1 +#endif +#ifndef HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1 +#endif +#ifndef RECV_TYPE_ARG1 +#define RECV_TYPE_ARG1 SOCKET +#endif +#ifndef RECV_TYPE_ARG2 +#define RECV_TYPE_ARG2 char * +#endif +#ifndef RECV_TYPE_ARG3 +#define RECV_TYPE_ARG3 int +#endif +#ifndef RECV_TYPE_ARG4 +#define RECV_TYPE_ARG4 int +#endif +#ifndef RECV_TYPE_RETV +#define RECV_TYPE_RETV int +#endif +#ifndef SEND_QUAL_ARG2 +#define SEND_QUAL_ARG2 +#endif +#ifndef SEND_TYPE_ARG1 +#define SEND_TYPE_ARG1 SOCKET +#endif +#ifndef SEND_TYPE_ARG2 +#define SEND_TYPE_ARG2 char * +#endif +#ifndef SEND_TYPE_ARG3 +#define SEND_TYPE_ARG3 int +#endif +#ifndef SEND_TYPE_ARG4 +#define SEND_TYPE_ARG4 int +#endif +#ifndef SEND_TYPE_RETV +#define SEND_TYPE_RETV int +#endif +#ifndef RECVFROM_TYPE_ARG1 +#define RECVFROM_TYPE_ARG1 SOCKET +#endif +#ifndef RECVFROM_TYPE_ARG2 +#define RECVFROM_TYPE_ARG2 char * +#endif +#ifndef RECVFROM_TYPE_ARG3 +#define RECVFROM_TYPE_ARG3 int +#endif +#ifndef RECVFROM_TYPE_ARG4 +#define RECVFROM_TYPE_ARG4 int +#endif +#ifndef RECVFROM_TYPE_ARG5 +#define RECVFROM_TYPE_ARG5 "struct sockaddr *" +#endif +#ifndef RECVFROM_TYPE_ARG6 +#define RECVFROM_TYPE_ARG6 "int *" +#endif +#ifndef RECVFROM_TYPE_RETV +#define RECVFROM_TYPE_RETV int +#endif +#ifndef RETSIGTYPE +#define RETSIGTYPE void +#endif +#ifndef CARES_TYPEOF_ARES_SOCKLEN_T +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#endif +#ifndef CARES_TYPEOF_ARES_SSIZE_T +#define CARES_TYPEOF_ARES_SSIZE_T "long long" +#endif +#endif /* _WIN32 */ +CARES_WIN32_FIX + +# The c-ares CMake detection above fails every HAVE_* probe (SOCKLEN_T, +# TYPE_SOCKET, STRUCT_ADDRINFO, ...), so its CMakeLists never adds the Winsock +# library to the link line. Without it the static libcares.a references +# WSAStartup / select / ntohl / gethostname / ... which then fail to link with +# "undefined reference to __imp_*". +# +# The static lib target (c-ares_static) links CARES_DEPENDENT_LIBS +# (ws2_32/advapi32/iphlpapi) PUBLICly, so gRPC's own consumers are fine. But +# the diagnostic tools (ahost/adig/acountry) link ${PROJECT_NAME} = "c-ares" +# by name; with CARES_SHARED=OFF there is no "c-ares" target, so CMake treats +# it as a plain library name and resolves it to libcares.a WITHOUT the PUBLIC +# Winsock deps -- the tools then fail to link with "undefined reference to +# __imp_*". The backend image doesn't need these standalone diagnostic +# utilities (gRPC links libcares.a, not the tools), so disable them entirely +# via -DCARES_BUILD_TOOLS=OFF in the cmake configure below. + +# boringssl's ssl_file.cc and ssl_x509.cc use X509_NAME as a C++ type, but +# Windows' wincrypt.h (pulled in transitively by in some +# mingw-w64 header chains) #defines X509_NAME as L"Name", a string literal. +# The macro turns `const X509_NAME *const *a` into `const L"Name" *const *a` +# and the whole file fails to compile. A per-file #undef doesn't work because +# wincrypt.h is re-included transitively through boringssl's own headers. +# Instead, define WIN32_LEAN_AND_MEAN globally via CMAKE_C/CXX_FLAGS so +# windows.h never pulls in wincrypt.h at all, eliminating the collision +# everywhere. This is the standard Windows idiom for avoiding wincrypt.h +# macro pollution (X509_NAME, X509_CERT, etc.) in code that uses OpenSSL/ +# boringssl types. +# +# _WIN32_WINNT=0x600 must ride along on the same flags: abseil's +# win32_waiter.cc guards its Win32Waiter definitions on +# _WIN32_WINNT >= _WIN32_WINNT_VISTA, and win32_waiter.h is included before +# any header sets a default _WIN32_WINNT, so without an explicit define the +# gate stays off. per_thread_sem.cc pulls win32_waiter.h in late (after +# thread_identity.h has already defined the default), so it compiles with the +# gate ON and the synchronization archive ends up referencing Win32Waiter +# while win32_waiter.cc.o is empty -- protoc dies with "undefined reference +# to Win32Waiter::*" that no --start-group can fix, because the symbols are +# not in the archive at all. (gRPC's own CMakeLists only adds +# -D_WIN32_WINNT=0x600 under `if (MINGW)`, which the native cmake sets but +# the MSYS2/Cygwin cmake that CI resolves via PATH does not.) +# +# The flags are spelled out on each cmake configure below instead of through +# CMAKE_ARGS: ${CMAKE_ARGS} is word-split on spaces by bash, so a +# space-containing -D value cannot travel through it. +export CMAKE_ARGS="${CMAKE_ARGS:-}" + +# boringssl's bssl CLI tool uses Winsock functions (socket, connect, select, +# WSAStartup, ...) directly in transport_common.cc, but its CMakeLists only +# links bssl with ssl crypto, omitting ws2_32. A global +# -DCMAKE_EXE_LINKER_FLAGS=-lws2_32 cannot fix this: CMake places those flags +# before the object files on the link line, so ld cannot use the library to +# satisfy the references that only appear later, and bssl still fails with +# "undefined reference to `__imp_select'". Patch the CMakeLists instead. +# gRPC adds the whole boringssl-with-bazel dir via add_subdirectory, so the +# bssl target lives in the ROOT CMakeLists.txt (its binary dir shows as +# third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/), not src/CMakeLists.txt. +# Both files declare the same `target_link_libraries(bssl ssl crypto)` line, so +# rewrite it in place to add ws2_32. Editing the exact declaration (instead of +# appending a guarded block) cannot be skipped by target visibility or block +# evaluation order: when the line is read, ssl and crypto are already linked +# and the bssl target already exists. +for bssl_cmake in \ + third_party/boringssl-with-bazel/CMakeLists.txt \ + third_party/boringssl-with-bazel/src/CMakeLists.txt +do + if [ -f "$bssl_cmake" ]; then + sed -i 's/^target_link_libraries(bssl ssl crypto)/target_link_libraries(bssl ssl crypto ws2_32)/' "$bssl_cmake" + grep -q 'target_link_libraries(bssl ssl crypto ws2_32)' "$bssl_cmake" || \ + { echo "ERROR: failed to patch bssl link in $bssl_cmake" >&2; exit 1; } + fi +done + +# zlib bundled with gRPC 1.59.0 ships win32/zlib1.rc which the mingw-w64 +# binutils 2.46+ windres rejects ("zlib1.rc:7: syntax error" — the VERSIONINFO +# resource format it emits is no longer accepted). The .rc is only used to +# embed Windows version metadata into the DLL; it is unnecessary for a static +# zlib linked into protobuf/gRPC. Overwriting the file with a minimal valid +# resource is format-independent (no need to know the pinned submodule's +# CMakeLists spelling, and the file is unconditionally named win32/zlib1.rc on +# WIN32) and produces no side effects for a static library. +cat > third_party/zlib/win32/zlib1.rc <<'ZLIB_RC_FIX' +1 VERSIONINFO +FILEVERSION 1,3,0,0 +PRODUCTVERSION 1,3,0,0 +BEGIN +END +ZLIB_RC_FIX + +# The grpc Makefile's MSYS/MINGW branches already add OPENSSL_NO_ASM=ON. We +# still pass an explicit generator + compiler: a bare cmake on windows-latest +# would pick the Visual Studio generator, producing an MSVC gRPC that the +# mingw-built grpc-server cannot link against. +cd "$GRPC_DIR/grpc_build" +# gRPC 1.59.0's bundled c-ares still declares cmake_minimum_required < 3.5, +# which cmake >= 4 removed support for. MSYS2 ships cmake 4.x, so configure +# with the minimum policy version c-ares' CMakeLists can still read. +# Static-library resolution on MinGW is order-sensitive: GNU ld scans each +# archive once and a definition that lives before the reference in the link +# line is never revisited. That is why protoc dies with "__imp_Sym*" +# (absl's symbolize needs dbghelp) unless dbghelp is reachable from a cyclic +# group: CMake places CMAKE_EXE_LINKER_FLAGS before the object files, so a +# bare -ldbghelp would be scanned before any reference to it exists and never +# revisited. The --start-group here is what makes it work - it extends to the +# end of the link line (ld closes it implicitly) and makes ld rescan all +# archives cyclically until no new member is pulled, so dbghelp resolves +# regardless of archive order. (The historical "Win32Waiter::*" failures are +# NOT a link-order problem; see the WIN32_LEAN_AND_MEAN/_WIN32_WINNT comment +# above - those symbols were missing from the archive entirely.) The same +# applies to shared links. +cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x600" \ + -DCMAKE_CXX_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x600" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_PREFIX_PATH="${INSTALLED_PACKAGES};${CMAKE_PREFIX_PATH:-}" \ + ${CMAKE_ARGS:-} \ + -DgRPC_INSTALL=ON \ + -DEXECUTABLE_OUTPUT_PATH=../installed_packages/grpc/bin \ + -DLIBRARY_OUTPUT_PATH=../installed_packages/grpc/lib \ + -DgRPC_BUILD_TESTS=OFF \ + -DCARES_BUILD_TOOLS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CPP_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + -Dprotobuf_WITH_ZLIB=ON \ + -DZLIB_ROOT=${INSTALLED_PACKAGES} \ + -DRE2_BUILD_TESTING=OFF \ + -DCMAKE_INSTALL_PREFIX=../installed_packages \ + -DOPENSSL_NO_ASM=ON \ + ../grpc_repo/grpc +# Diagnostics: print the bssl link line that CMake generated. bssl uses Winsock +# directly and dies with __imp_select/__imp_* undefined references ~30 minutes +# into the build if ws2_32 is not on its link line, so surface it now while the +# CI log still has the configure context. Verify ws2_32 actually landed. +echo "=== cmake flavor (must be the UCRT64 native build; MSYS2's /usr/bin/cmake +splits the Windows exe names and, unlike the native build, does not set MINGW) ===" +which cmake +cmake --version | head -1 +echo "=== bssl link (expect ws2_32) ===" +if [ -f third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/linkLibs.rsp ]; then + cat third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/linkLibs.rsp +elif [ -f third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/build.make ]; then + grep 'Linking CXX executable bssl' -A1 third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/build.make || true +else + echo "WARNING: bssl build files not generated" +fi +grep -rq 'ws2_32' third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/ && \ + echo "OK: bssl link includes ws2_32" || echo "FAIL: ws2_32 missing from bssl link" +# Diagnostics: protoc's link rule must carry --start-group + dbghelp, otherwise +# the absl static archives (symbolize) fail ~40 minutes in with undefined +# references that CMake cannot reorder away. +echo "=== protoc link (expect --start-group and -ldbghelp) ===" +if [ -f third_party/protobuf/CMakeFiles/protoc.dir/build.make ]; then + grep 'Linking CXX executable' -A1 third_party/protobuf/CMakeFiles/protoc.dir/build.make || true +else + echo "WARNING: protoc build files not generated" +fi +grep -rq -- '--start-group' third_party/protobuf/CMakeFiles/protoc.dir/ && \ + echo "OK: protoc link has --start-group" || echo "FAIL: --start-group missing from protoc link" +cmake --build . -j "$JOBS" +# Diagnostics: the _WIN32_WINNT=0x600 flag above must have compiled +# win32_waiter.cc with its gate open; confirm the archive actually defines +# Win32Waiter (a successful protoc link already implies it, but surface the +# count for the log). +echo "=== absl synchronization archive (expect Win32Waiter T symbols) ===" +# LIBRARY_OUTPUT_PATH=../installed_packages/grpc/lib is relative to the top +# build dir, so the archive lands next to the build tree, not under +# third_party/abseil-cpp. +nm ../installed_packages/grpc/lib/libabsl_synchronization.a 2>/dev/null | \ + grep -c 'Win32Waiter' || echo "WARNING: no Win32Waiter symbols in libabsl_synchronization.a" +cmake --build . --target install + +INSTALLED_PACKAGES="$GRPC_DIR/installed_packages" +ADDED_CMAKE_ARGS="-Dabsl_DIR=${INSTALLED_PACKAGES}/lib/cmake/absl \ + -DProtobuf_DIR=${INSTALLED_PACKAGES}/lib/cmake/protobuf \ + -Dutf8_range_DIR=${INSTALLED_PACKAGES}/lib/cmake/utf8_range \ + -DgRPC_DIR=${INSTALLED_PACKAGES}/lib/cmake/grpc \ + -DZLIB_ROOT=${INSTALLED_PACKAGES} \ + -DZLIB_INCLUDE_DIR=${INSTALLED_PACKAGES}/include \ + -DZLIB_LIBRARY=${INSTALLED_PACKAGES}/lib/libzlibstatic.a \ + -DZLIB_INCLUDE_DIRS=${INSTALLED_PACKAGES}/include \ + -DZLIB_LIBRARIES=${INSTALLED_PACKAGES}/lib/libzlibstatic.a \ + -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include" + +# Restore system zlib for llama.cpp builds (to avoid conflicts) if it was backed up +restore_sys_zlib 2>/dev/null || true +# But we want to force static zlib for llama.cpp - backup again before building variants +backup_sys_zlib 2>/dev/null || true +# find_program(protoc / grpc_cpp_plugin) in the grpc-server CMakeLists searches +# PATH, so the freshly installed protoc + grpc_cpp_plugin must be reachable. +export PATH="${INSTALLED_PACKAGES}/bin:${PATH}" + +# --------------------------------------------------------------------------- +# llama.cpp source at the pinned version, with the LocalAI grpc-server patch +# --------------------------------------------------------------------------- +cd "$ROOT/backend/cpp/llama-cpp" +LLAMA_VERSION=$(grep '^LLAMA_VERSION' Makefile | head -1 | cut -d= -f2 | cut -d'?' -f1 | tr -d ' ') +mkdir -p llama.cpp +cd llama.cpp +git init -q +# Re-runnable like the gRPC clone above; -B resets a stale `build` branch. +git remote remove origin 2>/dev/null || true +git remote add origin https://github.com/ggerganov/llama.cpp +# Shallow-fetch the pinned commit (allowAnySHA1InWant is on for public repos), +# then the pinned submodules shallowly. Equivalent content to `make llama.cpp` +# without downloading the full llama.cpp history on every build. +git fetch origin "$LLAMA_VERSION" --depth 1 +# prepare.sh patches the source tree in place; a previous (re-run) build may +# have left the working tree dirty, which would make the checkout below fail. +# Restore the pristine tree BEFORE checking out (tracked edits, untracked +# .rej/.orig and the generated tools/grpc-server staging dir). +git reset -q --hard +git clean -qfd +mkdir -p tools/grpc-server +git checkout -q -B build "$LLAMA_VERSION" +git submodule update --init --recursive --depth 1 --single-branch +cd "$ROOT/backend/cpp/llama-cpp" +bash prepare.sh + +# --------------------------------------------------------------------------- +# The three cmake variants (flags mirror backend/cpp/llama-cpp/Makefile) +# --------------------------------------------------------------------------- +build_variant() { + local name="$1" + local targets="$2" + shift 2 + # ggml turns ccache on when it finds it, but on MSYS2 ccache wants + # Windows-style USERPROFILE/LOCALAPPDATA the login profile drops - disable + # it (the variant builds are from scratch anyway). + rm -rf "llama.cpp/build-${name}" + # Force zlib to use the gRPC-installed static one to avoid mixing with system DLL import lib + export PKG_CONFIG_PATH="${INSTALLED_PACKAGES}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + cmake -G "Unix Makefiles" \ + -S llama.cpp -B "llama.cpp/build-${name}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0xA00" \ + -DCMAKE_CXX_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0xA00" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_PREFIX_PATH="${INSTALLED_PACKAGES};${CMAKE_PREFIX_PATH:-}" \ + ${CMAKE_ARGS:-} \ + -DGGML_NATIVE=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_CURL=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_CCACHE=OFF \ + "$@" \ + $ADDED_CMAKE_ARGS + cmake --build "llama.cpp/build-${name}" --config Release -j "$JOBS" --target $targets +} + +# CPU_ALL_VARIANTS: one grpc-server plus the dlopen-able libggml-cpu-*.dll set. +# ggml/llama go shared so the dynamic CPU backends work; gRPC stays static. +# GGML_VULKAN=ON adds the dlopen-able libggml-vulkan.dll to the same image, so +# the single grpc-server auto-detects a Vulkan device at runtime (ggml-vulkan +# loads vulkan-1.dll dynamically - it is never in the import table) and falls +# back to CPU when the host has no Vulkan driver. No separate variant or launcher +# change is needed: one image works everywhere and GPU offload just works when +# a model requests gpu_layers. +build_variant "cpu-all" "grpc-server ggml" \ + -DBUILD_SHARED_LIBS=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_VULKAN=ON + +# gRPC-RPC server + the ggml-rpc-server companion binary. +build_variant "rpc" "grpc-server ggml-rpc-server" \ + -DGGML_RPC=ON \ + -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off \ + -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# Static fallback, mirrors llama-cpp-fallback. +build_variant "fallback" "grpc-server" \ + -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off \ + -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# --------------------------------------------------------------------------- +# Stage the image contents +# --------------------------------------------------------------------------- +STAGE="$ROOT/build/windows" +rm -rf "$STAGE" +mkdir -p "$STAGE" "$ROOT/backend-images" + +cp llama.cpp/build-cpu-all/bin/grpc-server.exe "$STAGE/llama-cpp-cpu-all.exe" +# ggml's shared backends are loadable DLLs; they go next to the executable so +# the registry finds them without any PATH setup (mirrors run.sh LD_LIBRARY_PATH). +find llama.cpp/build-cpu-all/bin -maxdepth 1 -name '*.dll' -exec cp -v {} "$STAGE/" \; + +cp llama.cpp/build-rpc/bin/grpc-server.exe "$STAGE/llama-cpp-grpc.exe" +cp llama.cpp/build-rpc/bin/ggml-rpc-server.exe "$STAGE/llama-cpp-rpc-server.exe" + +cp llama.cpp/build-fallback/bin/grpc-server.exe "$STAGE/llama-cpp-fallback.exe" + +cd "$ROOT" +# The launcher is the PowerShell script pkg/model/process.go starts on Windows +# (run.ps1, mirroring run.sh); run.sh below is the stub that keeps +# discovery/validation/upgrade uniform with the other platforms. +cp backend/cpp/llama-cpp/run.ps1 "$STAGE/run.ps1" +cp backend/cpp/llama-cpp/run.sh "$STAGE/run.sh" + +# Bundle the runtime DLLs (mingw gcc runtime, openssl when gRPC links it +# dynamically) so the image runs on a host with no MSYS2. ggml's shared +# backends are already in $STAGE; scanning them too (not just the .exe files) +# catches the runtime deps of the loadable modules (ggml-vulkan.dll pulls in +# libgcc/libstdc++/libwinpthread) and lets the vulkan loader be bundled below. +for bin in "$STAGE"/*.exe "$STAGE"/*.dll; do + objdump -p "$bin" | awk '/DLL Name:/ {print $3}' | while read -r dll; do + if [ -f "/ucrt64/bin/$dll" ] && [ ! -e "$STAGE/$dll" ]; then + cp -v "/ucrt64/bin/$dll" "$STAGE/" + fi + done +done + +# ggml-vulkan loads vulkan-1.dll dynamically (volk-style LoadLibrary), so it is +# not in any import table and the scan above cannot see it. Bundle the loader +# from the MSYS2 vulkan-loader package anyway: on a host with no Vulkan driver +# the loader still initializes, ggml-vulkan registers zero devices, and CPU +# inference keeps working - but on a host whose driver does not drop its own +# copy into system32 the GPU would otherwise be invisible. +if [ -f "/ucrt64/bin/vulkan-1.dll" ] && [ ! -e "$STAGE/vulkan-1.dll" ]; then + cp -v "/ucrt64/bin/vulkan-1.dll" "$STAGE/" +fi + +echo "Bundled DLLs:" +ls -la "$STAGE" + +# --------------------------------------------------------------------------- +# Package as an OCI image tar +# --------------------------------------------------------------------------- +# local-ai.exe is only the tool that assembles the OCI tar - the image carries +# $STAGE, not this binary. cmd/local-ai still needs the prereqs of the upstream +# `make build`: pkg/grpc/proto/*.pb.go (gitignored, absent in a fresh checkout; +# regenerate with the protoc built above + the pinned Go plugins, mirroring +# `make protogen-go`) and the embedded react-ui dist (stubbed like lint.yml; +# the real bundle would need node, and this binary never serves it). +GO_PLUGIN_DIR="$(go env GOPATH)/bin" +GO_PLUGIN_DIR_MSYS="$(cygpath -u "$GO_PLUGIN_DIR")" +if [ ! -x "$GO_PLUGIN_DIR_MSYS/protoc-gen-go" ]; then + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 +fi +if [ ! -x "$GO_PLUGIN_DIR_MSYS/protoc-gen-go-grpc" ]; then + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af +fi +mkdir -p pkg/grpc/proto +"$GRPC_DIR/installed_packages/bin/protoc.exe" --experimental_allow_proto3_optional \ + -Ibackend/ \ + --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative \ + --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + --plugin=protoc-gen-go="$GO_PLUGIN_DIR/protoc-gen-go.exe" \ + --plugin=protoc-gen-go-grpc="$GO_PLUGIN_DIR/protoc-gen-go-grpc.exe" \ + backend/backend.proto +mkdir -p pkg/motion/proto +"$GRPC_DIR/installed_packages/bin/protoc.exe" \ + -Ipkg/motion/proto \ + --go_out=pkg/motion/proto --go_opt=paths=source_relative \ + --plugin=protoc-gen-go="$GO_PLUGIN_DIR/protoc-gen-go.exe" \ + pkg/motion/proto/motion.proto +mkdir -p core/http/react-ui/dist +[ -f core/http/react-ui/dist/index.html ] || touch core/http/react-ui/dist/index.html + +if [ ! -f local-ai.exe ] && [ ! -f local-ai ]; then + CGO_ENABLED=0 go build -o local-ai.exe ./cmd/local-ai +fi +mkdir -p backend-images +./local-ai util create-oci-image \ + build/windows/. \ + --output ./backend-images/llama-cpp.tar \ + --image-name "$IMAGE_NAME" \ + --platform "$PLATFORMARCH" + +rm -rf build/windows \ No newline at end of file diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js index 08a024fc63e8..caa1fdc6410a 100644 --- a/scripts/changed-backends.js +++ b/scripts/changed-backends.js @@ -16,11 +16,12 @@ import { const matrixYml = yaml.load(fs.readFileSync(BACKEND_MATRIX_FILE, "utf8")); const includes = matrixYml.include; const includesDarwin = matrixYml.includeDarwin; +const includesWindows = matrixYml.includeWindows || []; const eventPath = process.env.GITHUB_EVENT_PATH; const event = JSON.parse(fs.readFileSync(eventPath, "utf8")); -const allBackendPaths = getAllBackendPaths(includes, includesDarwin); +const allBackendPaths = getAllBackendPaths(includes, includesDarwin, includesWindows); const token = process.env.GITHUB_TOKEN; const octokit = new Octokit({ auth: token }); @@ -107,6 +108,7 @@ async function getPreviousMatrix(event) { return { include: previous.include || [], includeDarwin: previous.includeDarwin || [], + includeWindows: previous.includeWindows || [], }; } catch (err) { console.log( @@ -267,9 +269,11 @@ function emitFullMatrix() { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${multiarch.length > 0 ? 'true' : 'false'}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-windows=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-windows=${JSON.stringify({ include: includesWindows })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); emitSinglearchShards(singlearch); for (const backend of allBackendPaths.keys()) { @@ -280,9 +284,10 @@ function emitFullMatrix() { function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Changed files:", changedFiles); - const { filtered, filteredDarwin, changedBackends } = filterMatrix({ + const { filtered, filteredDarwin, filteredWindows, changedBackends } = filterMatrix({ includes, includesDarwin, + includesWindows, changedFiles, previousMatrix, protoRevisions, @@ -290,13 +295,16 @@ function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Filtered files:", filtered); console.log("Filtered files Darwin:", filteredDarwin); + console.log("Filtered files Windows:", filteredWindows); const { multiarch, singlearch } = splitByArch(filtered); const hasBackendsMultiarch = multiarch.length > 0 ? 'true' : 'false'; const hasBackendsDarwin = filteredDarwin.length > 0 ? 'true' : 'false'; + const hasBackendsWindows = filteredWindows.length > 0 ? 'true' : 'false'; console.log("Has single-arch backends?:", singlearch.length > 0 ? 'true' : 'false'); console.log("Has multi-arch backends?:", hasBackendsMultiarch); console.log("Has Darwin backends?:", hasBackendsDarwin); + console.log("Has Windows backends?:", hasBackendsWindows); const mergeMatrixMultiarch = computeMergeMatrix(multiarch); const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false'; @@ -304,9 +312,11 @@ function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${hasBackendsMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-windows=${hasBackendsWindows}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-windows=${JSON.stringify({ include: filteredWindows })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); emitSinglearchShards(singlearch); diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 5511e1bc7343..c0c9089e8aa4 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -136,8 +136,24 @@ export function inferBackendPathDarwin(item) { return `backend/${item.lang}/${item.backend}/`; } +export function inferBackendPathWindows(item) { + // llama-cpp on Windows builds from the C++ sources (via the MSYS2 build + // script, scripts/build/llama-cpp-windows.sh), not a backend/go/llama-cpp + // tree (which doesn't exist). The Windows job is matrix-driven with lang=go + // for runner/toolchain selection, but the source path is C++ — exactly the + // Darwin convention. + if (item.backend === "llama-cpp") { + return `backend/cpp/llama-cpp/`; + } + if (!item.lang) { + return `backend/python/${item.backend}/`; + } + + return `backend/${item.lang}/${item.backend}/`; +} + // Build a deduplicated map of backend name -> path prefix from all matrix entries -export function getAllBackendPaths(includes, includesDarwin) { +export function getAllBackendPaths(includes, includesDarwin, includesWindows = []) { const paths = new Map(); for (const item of includes) { const p = inferBackendPath(item); @@ -151,6 +167,12 @@ export function getAllBackendPaths(includes, includesDarwin) { paths.set(item.backend, p); } } + for (const item of includesWindows) { + const p = inferBackendPathWindows(item); + if (p && !paths.has(item.backend)) { + paths.set(item.backend, p); + } + } return paths; } @@ -180,6 +202,14 @@ const DARWIN_BESPOKE_BUILDERS = new Set([ const isDarwinGenericGo = item => !!item.lang && !DARWIN_BESPOKE_BUILDERS.has(item.backend); +// backend_build_windows.yml builds every Windows entry through +// `make backends/llama-cpp-windows` -> scripts/build/llama-cpp-windows.sh; there +// is no generic Go builder for Windows yet. Keep this set in sync with that +// workflow. +const WINDOWS_BESPOKE_BUILDERS = new Set(["llama-cpp"]); +const isWindowsGenericGo = item => + !!item.lang && !WINDOWS_BESPOKE_BUILDERS.has(item.backend); + const isLinuxPython = item => item.dockerfile.endsWith("python"); // Dockerfile.golang is the only Linux dockerfile that compiles Go (it runs @@ -373,6 +403,7 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === BACKEND_PROTO_FILE, linux: always, darwin: always, + windows: always, }, { // COPY'd into every Python image (Dockerfile.python) and into every Darwin @@ -380,6 +411,7 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file.startsWith("backend/python/common/"), linux: isLinuxPython, darwin: isDarwinPython, + windows: never, }, { // Compiled into every Go backend binary (see GO_BACKEND_PKG_PREFIXES). @@ -390,11 +422,18 @@ export const SHARED_BUILD_INPUTS = [ // lang=go for runner selection only — their sources are C++ and link no // Go, so isDarwinGenericGo is the correct predicate here, exactly as it // is for scripts/build/golang-darwin.sh. + // + // Windows: llama-cpp's launcher is the run.ps1 script (no Go, no compile) + // and local-ai.exe is rebuilt from the current checkout in the workflow + // itself, so a pkg/ change never invalidates a published Windows backend + // image. A future generic Go Windows builder would switch this to + // isWindowsGenericGo. matches: file => GO_BACKEND_PKG_PREFIXES.some(prefix => file.startsWith(prefix)) && !file.endsWith("_test.go"), linux: isLinuxGo, darwin: isDarwinGenericGo, + windows: never, }, { // The reusable build workflows own build-args, packaging and push for @@ -403,11 +442,19 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === ".github/workflows/backend_build.yml", linux: always, darwin: never, + windows: never, }, { matches: file => file === ".github/workflows/backend_build_darwin.yml", linux: never, darwin: always, + windows: never, + }, + { + matches: file => file === ".github/workflows/backend_build_windows.yml", + linux: never, + darwin: never, + windows: always, }, { // Decides which GPU libraries end up inside an image. Every Linux image @@ -419,31 +466,43 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === "scripts/build/package-gpu-libs.sh", linux: always, darwin: never, + windows: never, }, { matches: file => file === "scripts/build/python-darwin.sh", linux: never, darwin: isDarwinPython, + windows: never, }, { matches: file => file === "scripts/build/golang-darwin.sh", linux: never, darwin: isDarwinGenericGo, + windows: never, }, { matches: file => file === "scripts/build/llama-cpp-darwin.sh", linux: never, darwin: item => item.backend === "llama-cpp", + windows: never, }, { matches: file => file === "scripts/build/ds4-darwin.sh", linux: never, darwin: item => item.backend === "ds4", + windows: never, }, { matches: file => file === "scripts/build/privacy-filter-darwin.sh", linux: never, darwin: item => item.backend === "privacy-filter", + windows: never, + }, + { + matches: file => file === "scripts/build/llama-cpp-windows.sh", + linux: never, + darwin: never, + windows: item => item.backend === "llama-cpp", }, { // Catch-all, deliberately last and deliberately broad: anything else under @@ -456,6 +515,7 @@ export const SHARED_BUILD_INPUTS = [ file.startsWith("scripts/build/") && !file.endsWith("_test.sh"), linux: always, darwin: always, + windows: always, }, ]; @@ -491,7 +551,7 @@ export const BACKEND_MATRIX_FILE = ".github/backend-matrix.yml"; // Identity of a matrix entry across revisions. tag-suffix names the image; // per-arch legs of the same image are distinguished by platform-tag. Verified -// unique across all 432 Linux and 57 Darwin entries. +// unique across all 432 Linux, 57 Darwin and 1 Windows entries. export function matrixEntryKey(item) { return JSON.stringify([item["tag-suffix"] || "", item["platform-tag"] || ""]); } @@ -528,12 +588,14 @@ function matchedSharedRules(changedFiles) { return rules; } -// Filter both matrices against a changed-file list. Returns the surviving -// entries plus the set of backend names considered changed, which drives the -// per-backend boolean outputs consumed by test-extra.yml. +// Filter the Linux, Darwin and Windows matrices against a changed-file list. +// Returns the surviving entries plus the set of backend names considered +// changed, which drives the per-backend boolean outputs consumed by +// test-extra.yml. export function filterMatrix({ includes, includesDarwin, + includesWindows, changedFiles, previousMatrix, protoRevisions, @@ -566,6 +628,10 @@ export function filterMatrix({ matrixFileChanged && previousMatrix ? changedEntryKeys(includesDarwin, previousMatrix.includeDarwin || []) : null; + const changedWindowsKeys = + matrixFileChanged && previousMatrix + ? changedEntryKeys(includesWindows || [], previousMatrix.includeWindows || []) + : null; const filtered = includes.filter(item => { const backendPath = inferBackendPath(item); @@ -585,12 +651,21 @@ export function filterMatrix({ return sharedRules.some(rule => rule.darwin(item)); }); + const filteredWindows = (includesWindows || []).filter(item => { + const backendPath = inferBackendPathWindows(item); + if (changedFiles.some(file => file.startsWith(backendPath))) return true; + if (matrixDiffUnavailable) return true; + if (changedWindowsKeys && changedWindowsKeys.has(matrixEntryKey(item))) return true; + return sharedRules.some(rule => rule.windows(item)); + }); + const changedBackends = new Set(); for (const item of filtered) changedBackends.add(item.backend); for (const item of filteredDarwin) changedBackends.add(item.backend); - for (const [backend, pathPrefix] of getAllBackendPaths(includes, includesDarwin)) { + for (const item of filteredWindows) changedBackends.add(item.backend); + for (const [backend, pathPrefix] of getAllBackendPaths(includes, includesDarwin, includesWindows)) { if (backendChanged(backend, pathPrefix, changedFiles)) changedBackends.add(backend); } - return { filtered, filteredDarwin, changedBackends }; + return { filtered, filteredDarwin, filteredWindows, changedBackends }; } diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs index 25d942ae642d..f52251c1f22f 100644 --- a/scripts/lib/backend-filter_test.mjs +++ b/scripts/lib/backend-filter_test.mjs @@ -13,6 +13,7 @@ import { SHARED_BUILD_INPUTS, inferBackendPath, inferBackendPathDarwin, + inferBackendPathWindows, } from "./backend-filter.mjs"; test("kimodocpp maps to its native Go wrapper on Linux and Darwin", () => { @@ -92,8 +93,12 @@ const includesDarwin = [ { backend: "ds4", lang: "go", "tag-suffix": "-metal-darwin-arm64-ds4", "build-type": "metal" }, ]; +const includesWindows = [ + { backend: "llama-cpp", lang: "go", "tag-suffix": "-windows-amd64-llama-cpp" }, +]; + const run = (changedFiles, previousMatrix) => - filterMatrix({ includes, includesDarwin, changedFiles, previousMatrix }); + filterMatrix({ includes, includesDarwin, includesWindows, changedFiles, previousMatrix }); const names = entries => entries.map(e => e.backend).sort(); @@ -207,12 +212,13 @@ test("a bespoke Darwin build script rebuilds only its own backend", () => { }); test("an unclassified scripts/build/ file conservatively rebuilds everything", () => { - const { filtered, filteredDarwin } = run([ + const { filtered, filteredDarwin, filteredWindows } = run([ "scripts/build/package-something-new.sh", ]); assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); + assert.equal(filteredWindows.length, includesWindows.length); }); test("tests for the packaging scripts do not rebuild anything", () => { @@ -268,6 +274,87 @@ test("audio-cpp resolves to its C++ sources on Darwin, not backend/go", () => { ); }); +// --------------------------------------------------------------------------- +// Windows: native windows/amd64 backend images built under MSYS2 +// --------------------------------------------------------------------------- + +test("llama-cpp resolves to its C++ sources on Windows, not backend/go", () => { + // lang=go on a Windows entry only selects the runner and toolchain (the + // MSYS2 build compiles C++); routing it by that field would point at + // backend/go/llama-cpp/, which does not exist. + assert.equal( + inferBackendPathWindows({ + backend: "llama-cpp", + lang: "go", + "tag-suffix": "-windows-amd64-llama-cpp", + }), + "backend/cpp/llama-cpp/", + ); +}); + +test("llama-cpp source changes trigger the Windows build", () => { + const { filtered, filteredDarwin, filteredWindows, changedBackends } = run([ + "backend/cpp/llama-cpp/grpc-server.cpp", + ]); + + assert.deepEqual(names(filtered), ["llama-cpp", "turboquant"]); + assert.deepEqual(names(filteredDarwin), ["llama-cpp"]); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); + assert.ok(changedBackends.has("llama-cpp")); +}); + +test("backend.proto rebuilds the Windows entry too", () => { + const { filteredWindows } = run(["backend/backend.proto"]); + + assert.equal(filteredWindows.length, includesWindows.length); +}); + +test("the reusable Windows build workflow rebuilds Windows only", () => { + const { filtered, filteredDarwin, filteredWindows } = run([ + ".github/workflows/backend_build_windows.yml", + ]); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.equal(filteredWindows.length, includesWindows.length); +}); + +test("the Windows llama-cpp build script rebuilds only its backend", () => { + const { filtered, filteredDarwin, filteredWindows } = run([ + "scripts/build/llama-cpp-windows.sh", + ]); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); +}); + +test("a pkg/ subtree change never rebuilds the Windows entry", () => { + // The launcher is the run.ps1 script (no Go, no compile) and local-ai.exe + // is rebuilt in the workflow itself, so the shipped image does not depend on + // LocalAI pkg/ code. + const { filteredWindows } = run(["pkg/grpc/server.go"]); + + assert.deepEqual(filteredWindows, []); +}); + +test("editing the Windows matrix entry rebuilds it", () => { + const previousMatrix = previousOf( + e => e, + e => e, + e => ({ ...e, "tag-suffix": "-windows-amd64-llama-cpp-old" }) + ); + + const { filtered, filteredDarwin, filteredWindows } = run( + [".github/backend-matrix.yml"], + previousMatrix + ); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); +}); + // --------------------------------------------------------------------------- // pkg/ subtrees linked into every Go backend binary // --------------------------------------------------------------------------- @@ -344,9 +431,10 @@ test("tests for the linked pkg subtrees do not rebuild anything", () => { // A matrix file is only ever compared against a previous revision of itself, // so build the "before" side by mutating a copy of the fixtures. -const previousOf = (linuxPatch = e => e, darwinPatch = e => e) => ({ +const previousOf = (linuxPatch = e => e, darwinPatch = e => e, windowsPatch = e => e) => ({ include: includes.map(e => linuxPatch({ ...e })), includeDarwin: includesDarwin.map(e => darwinPatch({ ...e })), + includeWindows: includesWindows.map(e => windowsPatch({ ...e })), }); test("editing an existing entry's base-image rebuilds exactly that entry", () => { @@ -383,15 +471,17 @@ test("a brand-new matrix entry for an existing backend rebuilds only itself", () const previousMatrix = { include: includes.filter(e => e.backend !== "kokoros"), includeDarwin: includesDarwin, + includeWindows: includesWindows, }; - const { filtered, filteredDarwin } = run( + const { filtered, filteredDarwin, filteredWindows } = run( [".github/backend-matrix.yml"], previousMatrix ); assert.deepEqual(names(filtered), ["kokoros"]); assert.deepEqual(filteredDarwin, []); + assert.deepEqual(filteredWindows, []); }); test("editing a Darwin entry rebuilds only that Darwin entry", () => { @@ -432,11 +522,17 @@ test("removing an entry rebuilds nothing", () => { "base-image": "nvidia/cuda:11.8.0-devel-ubuntu22.04", }], includeDarwin: includesDarwin, + includeWindows: includesWindows, }; - const { filtered } = run([".github/backend-matrix.yml"], previousMatrix); + const { filtered, filteredDarwin, filteredWindows } = run( + [".github/backend-matrix.yml"], + previousMatrix + ); assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(filteredWindows, []); }); test("a previous matrix is ignored when backend-matrix.yml did not change", () => { @@ -454,10 +550,11 @@ test("a previous matrix is ignored when backend-matrix.yml did not change", () = test("an unavailable previous matrix conservatively rebuilds everything", () => { // Same posture as changed-backends.js's run-all fallbacks: if we cannot // resolve what the entries used to be, we must not claim nothing changed. - const { filtered, filteredDarwin } = run([".github/backend-matrix.yml"], null); + const { filtered, filteredDarwin, filteredWindows } = run([".github/backend-matrix.yml"], null); assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); + assert.equal(filteredWindows.length, includesWindows.length); }); // --- backend.proto: additive changes must not rebuild the world ------------- diff --git a/tests/addons/foundation/cluster/catalog.go b/tests/addons/foundation/cluster/catalog.go index dbb69780a528..09b028959849 100644 --- a/tests/addons/foundation/cluster/catalog.go +++ b/tests/addons/foundation/cluster/catalog.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package cluster @@ -198,3 +200,4 @@ func (l *Library) AddArchive(id, version string, archive []byte) (string, error) }) return digest, nil } + diff --git a/tests/addons/foundation/cluster/cluster.go b/tests/addons/foundation/cluster/cluster.go index 8f0f8c3f7913..26d6f467f9c6 100644 --- a/tests/addons/foundation/cluster/cluster.go +++ b/tests/addons/foundation/cluster/cluster.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package cluster @@ -105,3 +107,4 @@ func (c *Cluster) Session(node, user string) Client { func (c *Cluster) Control(path string, body any) (*Response, error) { return Client{Base: "http://" + c.Config.ControlAddr}.Do("POST", "/"+path, body, map[string]string{"X-Foundation-Control": c.Config.Control}) } + diff --git a/tests/addons/foundation/cluster/config.go b/tests/addons/foundation/cluster/config.go index 54d22597d2e3..0fd7d4b8ef97 100644 --- a/tests/addons/foundation/cluster/config.go +++ b/tests/addons/foundation/cluster/config.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT // Package cluster builds the foundation acceptance fixture: one add-on owner @@ -290,3 +292,4 @@ func (c *Config) Carrier(name string) (*gateway.CarrierConfig, error) { Certificate: pair, Roots: roots, Pins: pins, SigningKey: ed25519.PrivateKey(signRaw), VerificationKeys: keys, }, nil } + diff --git a/tests/addons/foundation/cluster/nodes.go b/tests/addons/foundation/cluster/nodes.go index 39a830366f9d..bd86a087eec5 100644 --- a/tests/addons/foundation/cluster/nodes.go +++ b/tests/addons/foundation/cluster/nodes.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package cluster @@ -352,3 +354,4 @@ func (c Client) Do(method, path string, body any, extra map[string]string) (*Res data, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) return &Response{Status: resp.StatusCode, Header: resp.Header, Body: data}, err } + diff --git a/tests/addons/foundation/cluster/packages.go b/tests/addons/foundation/cluster/packages.go index d216a8b73248..110f3b56f4d4 100644 --- a/tests/addons/foundation/cluster/packages.go +++ b/tests/addons/foundation/cluster/packages.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package cluster @@ -40,3 +42,4 @@ func ToolPackage(version string, fixture map[string]any) Package { func ServicePackage(version string, fixture map[string]any) Package { return Package{ID: "found-svc", Version: version, Effects: "yes", Fixture: fixture, Capabilities: map[string]any{"service": true}} } + diff --git a/tests/addons/foundation/cluster/server.go b/tests/addons/foundation/cluster/server.go index a8a3daeda3c5..52d577c6970d 100644 --- a/tests/addons/foundation/cluster/server.go +++ b/tests/addons/foundation/cluster/server.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package cluster @@ -180,3 +182,4 @@ func (c *Config) ControlServer(app *application.Application) (*http.Server, erro // StateDir is where one instance keeps its models, data and generated files. func (c *Config) StateDir(role string) string { return filepath.Join(c.Nodes[role].Dir, "state") } + diff --git a/tests/addons/foundation/cluster/teardown_test.go b/tests/addons/foundation/cluster/teardown_test.go index 457871a4f111..125e8985b977 100644 --- a/tests/addons/foundation/cluster/teardown_test.go +++ b/tests/addons/foundation/cluster/teardown_test.go @@ -1,3 +1,5 @@ +//go:build !windows + // SPDX-License-Identifier: MIT package cluster @@ -96,3 +98,4 @@ func TestStopReapsOrphansLeftByKill(t *testing.T) { n.Stop() assertGone(t, append(first, second...)...) } + diff --git a/tests/e2e/windows/windows_smoke_test.go b/tests/e2e/windows/windows_smoke_test.go new file mode 100644 index 000000000000..d69534324c8d --- /dev/null +++ b/tests/e2e/windows/windows_smoke_test.go @@ -0,0 +1,367 @@ +// Windows host smoke test for the native-windows backend path. +// +// Unlike the in-process e2e suite (tests/e2e/e2e_suite_test.go, which drives +// LocalAI through httpapi.API inside the test binary), this suite boots the +// real local-ai.exe with a --backends-path backend laid out the way the +// gallery contract ships it (run.sh stub for discovery + run.ps1 to launch), +// then exercises the HTTP API and the job-object process reaper from +// pkg/model. It is the regression guard for the Windows-native backend +// support: run.ps1 execution, backend discovery, and kill-on-close tree +// cleanup only ever happen on a live Windows host. +package windows_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/phayes/freeport" +) + +const buildTimeout = 15 * time.Minute + +var ( + serverProcess *os.Process + serverStdout io.WriteCloser + serverURL string + serverLogPath string + httpClient *http.Client + + modelsPath string + backendsPath string + tempDir string + + // Binaries under test. Names carry smoke- to avoid colliding with any + // in-tree artifact; both are built into tempDir. + localAIExe string + mockBackendExe string + + serverKilled bool +) + +// procRow is one row of the live Win32_Process snapshot. +type procRow struct { + PID int `json:"ProcessId"` + PPID int `json:"ParentProcessId"` + Name string `json:"Name"` + Cmd string `json:"CommandLine"` +} + +// repoRoot walks up from the package directory to the module root (the first +// ancestor containing go.mod). The test run's working directory is a package +// directory of the LocalAI module, so this terminates quickly. +func repoRoot(start string) string { + dir := start + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + Fail("could not locate the LocalAI module root from " + start) + } + dir = parent + } +} + +// buildBinary builds the package into out, mirroring the Makefile's +// CGO_ENABLED=0 so the build is environment-independent. +func buildBinary(pkg, out string) { + wd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + ctx, cancel := context.WithTimeout(context.Background(), buildTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "build", "-o", out, pkg) + cmd.Dir = repoRoot(wd) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + output, err := cmd.CombinedOutput() + if err != nil { + Fail(fmt.Sprintf("go build %s -> %s failed: %s\n%s", pkg, out, err, output)) + } +} + +// resolveBinary returns an existing binary from the given env override, or +// builds the package into tempDir. +func resolveBinary(env, pkg, name string) string { + if p := os.Getenv(env); p != "" { + Expect(p).To(BeAnExistingFile(), env+" must point at an existing executable") + return p + } + out := filepath.Join(tempDir, name) + buildBinary(pkg, out) + return out +} + +// powershellInvoke runs a PowerShell snippet and returns its raw stdout. +// Probes legitimately return non-zero exit codes (e.g. Get-Process for missing +// ids), so the exit status is deliberately ignored and only stdout matters. +func powershellInvoke(script string) string { + cmd := exec.Command("powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script) + out, err := cmd.CombinedOutput() + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "powershell probe (stdout-only) returned %v\n%s\n", err, out) + } + return string(out) +} + +// processSnapshot returns one row per live process, keyed for parent-child +// walks. CIM yields ProcessId/ParentProcessId in one query, which is the +// reliable way to reconstruct backend trees on Windows (no job/toolhelp API +// dependency and no command-line substring heuristic). +func processSnapshot() []procRow { + const query = "@(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,Name,CommandLine) | ConvertTo-Json -Compress" + var rows []procRow + out := powershellInvoke(query) + if err := json.Unmarshal([]byte(out), &rows); err != nil { + if rows = rows[:0]; len(out) == 0 { + // Empty snapshot (extremely unlikely) — the spec will fail with a + // clear message rather than panic. + _, _ = fmt.Fprintf(GinkgoWriter, "process query produced no output\n") + return nil + } + _, _ = fmt.Fprintf(GinkgoWriter, "process query did not unmarshal: %v\n%s\n", err, out) + } + return rows +} + +// treeForPID collects the process subtree rooted at root (processes whose +// ancestry chains back to it), excluding the root itself. Used to identify the +// wrapper + backend processes owned by the local-ai.exe under test without +// matching against command-line text (which can collide with unrelated +// processes on a developer machine). +func treeForPID(snapshot []procRow, root int) []procRow { + children := make(map[int][]procRow) + for _, r := range snapshot { + children[r.PPID] = append(children[r.PPID], r) + } + var tree []procRow + frontier := []int{root} + seen := map[int]bool{root: true} + for len(frontier) > 0 { + next := []int{} + for _, pid := range frontier { + for _, child := range children[pid] { + if seen[child.PID] { + continue + } + seen[child.PID] = true + tree = append(tree, child) + next = append(next, child.PID) + } + } + frontier = next + } + sort.Slice(tree, func(i, j int) bool { return tree[i].PID < tree[j].PID }) + return tree +} + +// describeV returns a readable listing of a process subtree for failure +// messages. +func describeV(rows []procRow) []string { + out := make([]string, len(rows)) + for i, r := range rows { + out[i] = fmt.Sprintf("pid=%d ppid=%d name=%s cmd=%q", r.PID, r.PPID, r.Name, r.Cmd) + } + return out +} + +// survivingPIDs filters ids down to those the current snapshot still holds. +func survivingPIDs(snapshot []procRow, ids []int) []int { + alive := make(map[int]bool) + for _, r := range snapshot { + alive[r.PID] = true + } + var out []int + for _, id := range ids { + if alive[id] { + out = append(out, id) + } + } + return out +} + +var _ = BeforeSuite(func() { + if runtime.GOOS != "windows" { + Skip("Windows host smoke test only runs on Windows") + } + + httpClient = &http.Client{Timeout: 30 * time.Second} + + var err error + tempDir, err = os.MkdirTemp("", "localai-windows-smoke-*") + Expect(err).ToNot(HaveOccurred()) + + localAIExe = resolveBinary("LOCAL_AI_EXE", "github.com/mudler/LocalAI/cmd/local-ai", "local-ai.exe") + mockBackendExe = resolveBinary("MOCK_BACKEND_EXE", "github.com/mudler/LocalAI/tests/e2e/mock-backend", "mock-backend.exe") + + modelsPath = filepath.Join(tempDir, "models") + backendsPath = filepath.Join(tempDir, "backends") + Expect(os.MkdirAll(modelsPath, 0755)).To(Succeed()) + + // Backend layout mirroring the gallery contract: the binary lives in + // /mock-backend/ with run.sh (kept for uniform discovery — + // ListSystemBackends looks for run.sh) and run.ps1 (what startProcess + // actually executes on a Windows host). + backendDir := filepath.Join(backendsPath, "mock-backend") + Expect(os.MkdirAll(backendDir, 0755)).To(Succeed()) + + Expect(os.WriteFile(filepath.Join(backendDir, "run.sh"), []byte("#!/bin/sh\nexit 0\n"), 0644)).To(Succeed()) + runPS1 := "$ErrorActionPreference = 'Stop'\n" + + "& \"$PSScriptRoot\\mock-backend.exe\" @args\n" + + "exit $LASTEXITCODE\n" + Expect(os.WriteFile(filepath.Join(backendDir, "run.ps1"), []byte(runPS1), 0644)).To(Succeed()) + + backendBinary, err := os.ReadFile(mockBackendExe) + Expect(err).ToNot(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(backendDir, "mock-backend.exe"), backendBinary, 0755)).To(Succeed()) + + modelCfg := "name: mock-model\n" + + "backend: mock-backend\n" + + "parameters:\n" + + " model: mock-model.bin\n" + Expect(os.WriteFile(filepath.Join(modelsPath, "mock-model.yaml"), []byte(modelCfg), 0644)).To(Succeed()) + + port, err := freeport.GetFreePort() + Expect(err).ToNot(HaveOccurred()) + serverURL = fmt.Sprintf("http://127.0.0.1:%d/v1", port) + + // Capture local-ai.exe diagnostics so CI failures are not blind. + serverLogPath = filepath.Join(tempDir, "local-ai.log") + serverStdout, err = os.Create(serverLogPath) + Expect(err).ToNot(HaveOccurred()) + + // Run local-ai.exe from the temp dir so its state (.local_user_id etc.) + // never lands in the repository working copy. + cmd := exec.Command(localAIExe, + "--address", fmt.Sprintf("127.0.0.1:%d", port), + "--models-path", modelsPath, + "--backends-path", backendsPath, + "--log-level", "debug", + ) + cmd.Dir = tempDir + cmd.Stdout = serverStdout + cmd.Stderr = serverStdout + Expect(cmd.Start()).To(Succeed()) + serverProcess = cmd.Process + + // Server must come up and register the discovered mock-backend. + Eventually(func() error { + resp, err := httpClient.Get(serverURL + "/models") + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET /v1/models: HTTP %d", resp.StatusCode) + } + var out struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + for _, m := range out.Data { + if m.ID == "mock-model" { + return nil + } + } + return fmt.Errorf("mock-model not listed yet (got %d models)", len(out.Data)) + }, "2m", "2s").Should(Succeed(), dumpServerLog) +}) + +// dumpServerLog returns the tail of the local-ai.exe log for CI debugging. +func dumpServerLog() string { + if serverLogPath == "" { + return "" + } + data, err := os.ReadFile(serverLogPath) + if err != nil { + return fmt.Sprintf("server log (%s) unreadable: %s", serverLogPath, err) + } + const max = 2000 + if len(data) > max { + data = data[len(data)-max:] + } + return fmt.Sprintf("server log (%s):\n%s", serverLogPath, data) +} + +var _ = Describe("LocalAI Windows smoke", func() { + It("serves a chat completion through the mock backend via run.ps1", func() { + body := strings.NewReader(`{"model":"mock-model","messages":[{"role":"user","content":"hello"}]}`) + resp, err := httpClient.Post(serverURL+"/chat/completions", "application/json", body) + Expect(err).ToNot(HaveOccurred(), dumpServerLog) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusOK), dumpServerLog) + + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + Expect(json.NewDecoder(resp.Body).Decode(&out)).To(Succeed(), dumpServerLog) + Expect(out.Choices).ToNot(BeEmpty(), dumpServerLog) + Expect(out.Choices[0].Message.Content).To(ContainSubstring("This is a mocked response.")) + }) + + // Must run last: it hard-kills the server under test. ginkgo runs specs in + // declaration order. + It("reaps the backend tree when local-ai.exe is hard-killed (job object)", func() { + // Identify the wrapper + backend as the process subtree of the server, + // not by command-line text (which matches unrelated processes). + before := processSnapshot() + tree := treeForPID(before, serverProcess.Pid) + Expect(tree).ToNot(BeEmpty(), + "no child processes of local-ai.exe (pid %d) found — the chat spec must have loaded the backend first", serverProcess.Pid) + + var treePIDs []int + for _, r := range tree { + treePIDs = append(treePIDs, r.PID) + } + + // Hard-kill: no graceful shutdown runs, so a surviving backend means + // kill-on-close failed, not a stop path. + Expect(serverProcess.Kill()).To(Succeed()) + _, _ = serverProcess.Wait() + serverKilled = true + + Eventually(func() []int { + return survivingPIDs(processSnapshot(), treePIDs) + }, "30s", "1s").Should(BeEmpty(), + "these processes survived local-ai.exe kill-on-close (job object not applied?):\n%s", + strings.Join(describeV(tree), "\n")) + }) +}) + +var _ = AfterSuite(func() { + if runtime.GOOS != "windows" { + return + } + if serverProcess != nil && !serverKilled { + _ = serverProcess.Kill() + _, _ = serverProcess.Wait() + } + if serverStdout != nil { + _ = serverStdout.Close() + } + if serverLogPath != "" { + _, _ = fmt.Fprintln(GinkgoWriter, dumpServerLog()) + } + if tempDir != "" { + _ = os.RemoveAll(tempDir) + } +}) diff --git a/tests/e2e/windows/windows_suite_test.go b/tests/e2e/windows/windows_suite_test.go new file mode 100644 index 000000000000..266ca03baf2e --- /dev/null +++ b/tests/e2e/windows/windows_suite_test.go @@ -0,0 +1,13 @@ +package windows_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestWindowsSmoke(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "LocalAI Windows smoke test suite") +}