From 6e1768cfad7f520b853cb3cf5040ded68287c6fc Mon Sep 17 00:00:00 2001 From: "Plamen K. Kosseff" Date: Sat, 17 Oct 2026 20:01:04 +0000 Subject: [PATCH] feat(backends): --disable-backend-install refuses every backend acquisition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hardening switch for deployments that must run exactly the backend set they were tested with (production, air-gapped, immutable images, package-managed hosts). With LOCALAI_DISABLE_BACKEND_INSTALL=1 every install and upgrade path refuses — gallery, OCI/URL/path, node-scoped, auto-upgrade, boot-time external backends — HTTP routes answer 403 up front, and a model whose backend is missing is refused before any download: 409 on the API, an error on preload and import. Whether a backend is present is asked of the execution target: the local backend directory on a single node, the workers' listings on a distributed frontend, which installs nothing itself. Backends already present, including system-path ones, keep working. Assisted-by: Claude Code:claude-fable-5 Signed-off-by: Plamen K. Kosseff --- core/application/startup.go | 13 +- core/cli/run.go | 6 + core/cli/worker/worker.go | 1 + core/cli/worker/worker_backend_common_test.go | 47 ++++ core/cli/worker/worker_ds4.go | 1 + core/cli/worker/worker_llamacpp.go | 1 + core/cli/worker/worker_mlx_distributed.go | 1 + core/cli/worker/worker_p2p.go | 1 + core/cli/worker/worker_p2p_mlx.go | 1 + core/cli/worker/worker_vllm.go | 1 + core/config/application_config.go | 8 + core/gallery/backend_install_disabled_test.go | 208 ++++++++++++++++ core/gallery/backends.go | 91 ++++++- core/gallery/model_artifacts.go | 15 +- core/gallery/models.go | 14 ++ core/gallery/upgrade.go | 3 + core/http/endpoints/localai/backend.go | 3 + core/http/endpoints/localai/gallery.go | 37 ++- core/http/middleware/backend_install.go | 27 +++ core/http/middleware/backend_install_test.go | 42 ++++ .../public/locales/en/importModel.json | 1 + .../react-ui/public/locales/en/operate.json | 3 + .../react-ui/src/components/hub/hubConfig.js | 2 +- core/http/react-ui/src/pages/Backends.jsx | 57 ++++- core/http/react-ui/src/pages/ImportModel.jsx | 18 ++ core/http/react-ui/src/utils/capabilities.js | 1 + core/http/routes/localai.go | 8 +- core/http/routes/nodes.go | 5 +- core/http/routes/ui_api.go | 21 +- .../ui_api_backend_install_disabled_test.go | 222 ++++++++++++++++++ .../galleryop/backend_delete_optype_test.go | 2 + core/services/galleryop/backends.go | 5 +- core/services/galleryop/managers.go | 3 + core/services/galleryop/managers_local.go | 15 +- core/services/galleryop/models.go | 3 + core/services/galleryop/operation.go | 7 + core/services/nodes/managers_distributed.go | 15 ++ .../nodes/managers_distributed_test.go | 3 + core/services/worker/config.go | 1 + core/services/worker/install_disabled_test.go | 52 ++++ core/services/worker/lifecycle.go | 1 + core/services/worker/worker.go | 1 + core/services/workerctl/backend.go | 12 +- core/services/workerctl/wire_test.go | 8 +- core/startup/model_preload.go | 5 +- core/startup/model_preload_test.go | 82 +++++++ docs/content/features/api-discovery.md | 3 +- docs/content/features/backends.md | 49 ++++ docs/content/features/distributed-mode.md | 16 ++ docs/content/reference/cli-reference.md | 1 + pkg/mcp/localaitools/inproc/client.go | 6 + pkg/mcp/localaitools/prompts/20_tools.md | 4 +- .../prompts/skills/upgrade_backend.md | 4 + pkg/system/state.go | 12 + swagger/docs.go | 28 +++ swagger/swagger.json | 28 +++ swagger/swagger.yaml | 21 ++ 57 files changed, 1201 insertions(+), 45 deletions(-) create mode 100644 core/cli/worker/worker_backend_common_test.go create mode 100644 core/gallery/backend_install_disabled_test.go create mode 100644 core/http/middleware/backend_install.go create mode 100644 core/http/middleware/backend_install_test.go create mode 100644 core/http/routes/ui_api_backend_install_disabled_test.go create mode 100644 core/services/worker/install_disabled_test.go diff --git a/core/application/startup.go b/core/application/startup.go index b1be5f751ba5..06d2c1ec4fd2 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -479,7 +479,7 @@ func New(opts ...config.AppOption) (*Application, error) { // Start background upgrade checker for backends. // In distributed mode, uses PostgreSQL advisory lock so only one frontend // instance runs periodic checks (avoids duplicate upgrades across replicas). - if len(options.BackendGalleries) > 0 { + if len(options.BackendGalleries) > 0 && !options.DisableBackendInstall { // Pass a lazy getter for the backend manager so the checker always // uses the active one — DistributedBackendManager is swapped in above // and asks workers for their installed backends, which is what @@ -536,14 +536,21 @@ func New(opts ...config.AppOption) (*Application, error) { modelConfigResync.Start(options.Context, options.Distributed.ModelConfigResyncIntervalOrDefault(), distSvc.Nats) } + // Preloads run outside the operation loop, so they do not get the + // availability the loop stamps on each op; it comes from the same manager. + preloadOptions := []gallery.InstallOption{gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)} + if gs := application.GalleryService(); gs != nil { + preloadOptions = append(preloadOptions, gallery.WithBackendAvailability(gs.BackendManager().BackendAvailable)) + } + if options.PreloadJSONModels != "" { - if err := galleryop.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels, options.RequireBackendIntegrity, gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)); err != nil { + if err := galleryop.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels, options.RequireBackendIntegrity, preloadOptions...); err != nil { return nil, err } } if options.PreloadModelsFromPath != "" { - if err := galleryop.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath, options.RequireBackendIntegrity, gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)); err != nil { + if err := galleryop.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath, options.RequireBackendIntegrity, preloadOptions...); err != nil { return nil, err } } diff --git a/core/cli/run.go b/core/cli/run.go index 77f2f9cce0c4..90b4d5776346 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -91,6 +91,7 @@ type RunCMD struct { DisableRuntimeSettings bool `env:"LOCALAI_DISABLE_RUNTIME_SETTINGS,DISABLE_RUNTIME_SETTINGS" default:"false" help:"Disables the runtime settings. When set to true, the server will not load the runtime settings from the runtime_settings.json file" group:"api"` DisablePredownloadScan bool `env:"LOCALAI_DISABLE_PREDOWNLOAD_SCAN" help:"If true, disables the best-effort security scanner before downloading any files." group:"hardening" default:"false"` RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, backend installs without a configured signature verification policy (for OCI URIs) or SHA256 (for tarball/HTTP URIs) are rejected. Default is to warn and install. Set this in production once your gallery's verification: block is populated." group:"hardening" default:"false"` + DisableBackendInstall bool `env:"LOCALAI_DISABLE_BACKEND_INSTALL,DISABLE_BACKEND_INSTALL" help:"If true, refuse to install or upgrade backends from galleries, OCI images, URLs or paths, including automatic installs, so the running backend set stays exactly what was deployed and tested; only backends already present or provided via --backends-system-path are used." group:"hardening" default:"false"` OpaqueErrors bool `env:"LOCALAI_OPAQUE_ERRORS" default:"false" help:"If true, all error responses are replaced with blank 500 errors. This is intended only for hardening against information leaks and is normally not recommended." group:"hardening"` UseSubtleKeyComparison bool `env:"LOCALAI_SUBTLE_KEY_COMPARISON" default:"false" help:"If true, API Key validation comparisons will be performed using constant-time comparisons rather than simple equality. This trades off performance on each request for resiliancy against timing attacks." group:"hardening"` DisableApiKeyRequirementForHttpGet bool `env:"LOCALAI_DISABLE_API_KEY_REQUIREMENT_FOR_HTTP_GET" default:"false" help:"If true, a valid API key is not required to issue GET requests to portions of the web ui. This should only be enabled in secure testing environments" group:"hardening"` @@ -302,6 +303,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) (result error) { system.WithBackendDevSuffix(r.BackendDevSuffix), system.WithPreferDevelopmentBackends(r.PreferDevelopmentBackends), system.WithRequireBackendIntegrity(r.RequireBackendIntegrity), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return err @@ -785,6 +787,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) (result error) { opts = append(opts, config.WithRequireBackendIntegrity(r.RequireBackendIntegrity)) } + if r.DisableBackendInstall { + opts = append(opts, config.WithDisableBackendInstall(r.DisableBackendInstall)) + } + if r.PreferDevelopmentBackends { opts = append(opts, config.WithPreferDevelopmentBackends(r.PreferDevelopmentBackends)) } diff --git a/core/cli/worker/worker.go b/core/cli/worker/worker.go index 792c604f5457..e52d2c9f5789 100644 --- a/core/cli/worker/worker.go +++ b/core/cli/worker/worker.go @@ -5,6 +5,7 @@ type WorkerFlags struct { BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"` BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends used for inferencing" group:"backends"` RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` + DisableBackendInstall bool `env:"LOCALAI_DISABLE_BACKEND_INSTALL,DISABLE_BACKEND_INSTALL" help:"If true, never install the worker's backend from a gallery; use the one already present under the backends or system path, or exit with an error." group:"hardening" default:"false"` ExtraLLamaCPPArgs string `name:"llama-cpp-args" env:"LOCALAI_EXTRA_LLAMA_CPP_ARGS,EXTRA_LLAMA_CPP_ARGS" help:"Extra arguments to pass to llama-cpp-rpc-server"` } diff --git a/core/cli/worker/worker_backend_common_test.go b/core/cli/worker/worker_backend_common_test.go new file mode 100644 index 000000000000..fcc9ed31b1ce --- /dev/null +++ b/core/cli/worker/worker_backend_common_test.go @@ -0,0 +1,47 @@ +package worker + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("findBackendPath with backend installation disabled", func() { + var state *system.SystemState + + BeforeEach(func() { + root := GinkgoT().TempDir() + var err error + state, err = system.GetSystemState( + system.WithBackendPath(filepath.Join(root, "backends")), + system.WithBackendSystemPath(filepath.Join(root, "system")), + system.WithDisableBackendInstall(true), + ) + Expect(err).NotTo(HaveOccurred()) + Expect(os.MkdirAll(state.Backend.BackendsPath, 0o750)).To(Succeed()) + Expect(os.MkdirAll(state.Backend.BackendsSystemPath, 0o750)).To(Succeed()) + }) + + It("uses a backend that is already present", func() { + dir := filepath.Join(state.Backend.BackendsSystemPath, "vllm") + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + meta, _ := json.Marshal(&gallery.BackendMetadata{Name: "vllm"}) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), meta, 0o644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte("#!/bin/sh\n"), 0o755)).To(Succeed()) + + path, err := findBackendPath("vllm", "[]", state, false) + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(dir)) + }) + + It("refuses to install a missing backend", func() { + _, err := findBackendPath("vllm", "[]", state, false) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + }) +}) diff --git a/core/cli/worker/worker_ds4.go b/core/cli/worker/worker_ds4.go index f2139dbd01a2..c0513a66ca1e 100644 --- a/core/cli/worker/worker_ds4.go +++ b/core/cli/worker/worker_ds4.go @@ -81,6 +81,7 @@ func (r *DS4Distributed) Run(ctx *cliContext.Context) error { systemState, err := system.GetSystemState( system.WithBackendPath(r.BackendsPath), system.WithBackendSystemPath(r.BackendsSystemPath), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return err diff --git a/core/cli/worker/worker_llamacpp.go b/core/cli/worker/worker_llamacpp.go index fa36590f2053..229cf0fd0e56 100644 --- a/core/cli/worker/worker_llamacpp.go +++ b/core/cli/worker/worker_llamacpp.go @@ -72,6 +72,7 @@ func (r *LLamaCPP) Run(ctx *cliContext.Context) error { systemState, err := system.GetSystemState( system.WithBackendPath(r.BackendsPath), system.WithBackendSystemPath(r.BackendsSystemPath), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return err diff --git a/core/cli/worker/worker_mlx_distributed.go b/core/cli/worker/worker_mlx_distributed.go index 57265ef24430..32566d17ae4f 100644 --- a/core/cli/worker/worker_mlx_distributed.go +++ b/core/cli/worker/worker_mlx_distributed.go @@ -23,6 +23,7 @@ func (r *MLXDistributed) Run(ctx *cliContext.Context) error { systemState, err := system.GetSystemState( system.WithBackendPath(r.BackendsPath), system.WithBackendSystemPath(r.BackendsSystemPath), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return err diff --git a/core/cli/worker/worker_p2p.go b/core/cli/worker/worker_p2p.go index c7ff254ea451..78dbd201ebfe 100644 --- a/core/cli/worker/worker_p2p.go +++ b/core/cli/worker/worker_p2p.go @@ -30,6 +30,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error { systemState, err := system.GetSystemState( system.WithBackendPath(r.BackendsPath), system.WithBackendSystemPath(r.BackendsSystemPath), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return err diff --git a/core/cli/worker/worker_p2p_mlx.go b/core/cli/worker/worker_p2p_mlx.go index 7edd1673def2..e7afc24317dd 100644 --- a/core/cli/worker/worker_p2p_mlx.go +++ b/core/cli/worker/worker_p2p_mlx.go @@ -30,6 +30,7 @@ func (r *P2PMLX) Run(ctx *cliContext.Context) error { systemState, err := system.GetSystemState( system.WithBackendPath(r.BackendsPath), system.WithBackendSystemPath(r.BackendsSystemPath), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return err diff --git a/core/cli/worker/worker_vllm.go b/core/cli/worker/worker_vllm.go index 8596546b5849..31b1f1da4880 100644 --- a/core/cli/worker/worker_vllm.go +++ b/core/cli/worker/worker_vllm.go @@ -72,6 +72,7 @@ func (r *VLLMDistributed) Run(ctx *cliContext.Context) error { systemState, err := system.GetSystemState( system.WithBackendPath(r.BackendsPath), system.WithBackendSystemPath(r.BackendsSystemPath), + system.WithDisableBackendInstall(r.DisableBackendInstall), ) if err != nil { return fmt.Errorf("getting system state: %w", err) diff --git a/core/config/application_config.go b/core/config/application_config.go index 74d60fb9f677..00fdbd05a2a8 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -147,6 +147,10 @@ type ApplicationConfig struct { // --require-backend-integrity / LOCALAI_REQUIRE_BACKEND_INTEGRITY. RequireBackendIntegrity bool + // DisableBackendInstall mirrors SystemState.DisableBackendInstall for the + // HTTP layer: admission guards and the capability surfaces read it here. + DisableBackendInstall bool + SingleBackend bool // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends int // Maximum number of active backends (0 = unlimited, 1 = single backend mode) WatchDogIdle bool @@ -621,6 +625,10 @@ func WithRequireBackendIntegrity(v bool) AppOption { return func(o *ApplicationConfig) { o.RequireBackendIntegrity = v } } +func WithDisableBackendInstall(v bool) AppOption { + return func(o *ApplicationConfig) { o.DisableBackendInstall = v } +} + func WithPreferDevelopmentBackends(v bool) AppOption { return func(o *ApplicationConfig) { o.PreferDevelopmentBackends = v } } diff --git a/core/gallery/backend_install_disabled_test.go b/core/gallery/backend_install_disabled_test.go new file mode 100644 index 000000000000..dda1a633e03c --- /dev/null +++ b/core/gallery/backend_install_disabled_test.go @@ -0,0 +1,208 @@ +package gallery_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +// --disable-backend-install: nothing may land in BackendsPath from outside, +// backends already present (managed or system path) keep working, and a model +// install skips the backend step instead of failing on it. +var _ = Describe("Disabled backend installation", func() { + var tempDir string + var state *system.SystemState + var ml *model.ModelLoader + galleryRevision := 0 + + // A runnable backend directory, as ListSystemBackends recognizes one. + mkBackend := func(root, name string) { + dir := filepath.Join(root, name) + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + meta, _ := json.Marshal(&gallery.BackendMetadata{Name: name}) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), meta, 0o644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte("#!/bin/sh\n"), 0o755)).To(Succeed()) + } + + // A file:// model gallery, inside the models path (the only root a file + // URL is trusted from); the listing is cached on the name and URL pair, + // so every spec gets a gallery of its own. + newGallery := func(entries ...gallery.GalleryModel) []config.Gallery { + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + name := fmt.Sprintf("install-disabled-%d", galleryRevision) + galleryRevision++ + path := filepath.Join(state.Model.ModelsPath, name+".yaml") + Expect(os.WriteFile(path, out, 0o600)).To(Succeed()) + return []config.Gallery{{Name: name, URL: "file://" + path}} + } + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "backend-install-disabled-*") + Expect(err).NotTo(HaveOccurred()) + state, err = system.GetSystemState( + system.WithBackendPath(filepath.Join(tempDir, "backends")), + system.WithBackendSystemPath(filepath.Join(tempDir, "system-backends")), + system.WithModelPath(filepath.Join(tempDir, "models")), + system.WithDisableBackendInstall(true), + ) + Expect(err).NotTo(HaveOccurred()) + for _, dir := range []string{state.Model.ModelsPath, state.Backend.BackendsPath, state.Backend.BackendsSystemPath} { + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + } + ml = model.NewModelLoader(state) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + It("InstallBackend refuses with the typed error and writes nothing", func() { + src, err := os.MkdirTemp("", "backend-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(src) + Expect(os.WriteFile(filepath.Join(src, "run.sh"), []byte("#!/bin/sh\n"), 0o755)).To(Succeed()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend" + backend.URI = src + + err = gallery.InstallBackend(context.Background(), state, ml, backend, nil, false) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + _, statErr := os.Stat(filepath.Join(state.Backend.BackendsPath, "test-backend")) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + }) + + It("InstallBackendFromGallery refuses before consulting any gallery", func() { + err := gallery.InstallBackendFromGallery(context.Background(), nil, state, ml, "vllm", nil, false, false) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + }) + + It("InstallBackendFromGallery still accepts a backend that is already installed", func() { + mkBackend(state.Backend.BackendsSystemPath, "llama-cpp") + Expect(gallery.InstallBackendFromGallery(context.Background(), nil, state, ml, "llama-cpp", nil, false, false)).To(Succeed()) + }) + + It("UpgradeBackend refuses even for an installed backend", func() { + mkBackend(state.Backend.BackendsPath, "llama-cpp") + err := gallery.UpgradeBackend(context.Background(), state, ml, nil, "llama-cpp", nil, false) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + Expect(filepath.Join(state.Backend.BackendsPath, "llama-cpp", "run.sh")).To(BeAnExistingFile()) + }) + + It("RequiredBackendInstalled reports the entry's backend and its presence", func() { + mkBackend(state.Backend.BackendsSystemPath, "llama-cpp") + galleries := newGallery( + gallery.GalleryModel{Metadata: gallery.Metadata{Name: "needs-vllm", Backend: "vllm"}}, + gallery.GalleryModel{Metadata: gallery.Metadata{Name: "needs-llama", Backend: "llama-cpp"}}, + gallery.GalleryModel{Metadata: gallery.Metadata{Name: "no-backend"}}, + ) + + backend, installed, err := gallery.RequiredBackendInstalled(context.Background(), galleries, state, "needs-vllm", "", nil) + Expect(err).NotTo(HaveOccurred()) + Expect(backend).To(Equal("vllm")) + Expect(installed).To(BeFalse()) + + backend, installed, err = gallery.RequiredBackendInstalled(context.Background(), galleries, state, "needs-llama", "", nil) + Expect(err).NotTo(HaveOccurred()) + Expect(backend).To(Equal("llama-cpp")) + Expect(installed).To(BeTrue()) + + backend, installed, err = gallery.RequiredBackendInstalled(context.Background(), galleries, state, "no-backend", "", nil) + Expect(err).NotTo(HaveOccurred()) + Expect(backend).To(BeEmpty()) + Expect(installed).To(BeTrue()) + + // A distributed frontend has no local backends; the execution + // target's answer decides. + onCluster := func(string) (bool, error) { return true, nil } + backend, installed, err = gallery.RequiredBackendInstalled(context.Background(), galleries, state, "needs-vllm", "", onCluster) + Expect(err).NotTo(HaveOccurred()) + Expect(backend).To(Equal("vllm")) + Expect(installed).To(BeTrue()) + }) + + It("AvailableOnCluster reads worker presence, not local files", func() { + cluster := gallery.SystemBackends{ + "vllm": {Name: "vllm", Nodes: []gallery.NodeBackendRef{{NodeID: "w1", NodeName: "worker-1", NodeStatus: "healthy"}}}, + "orphan": {Name: "orphan"}, + // Listed because the node still answers, but the router will not + // place a load there until its heartbeat returns. + "ailing": {Name: "ailing", Nodes: []gallery.NodeBackendRef{{NodeID: "w3", NodeName: "worker-3", NodeStatus: "unhealthy"}}}, + // Workers send the alias on the concrete's row, never as a row of its own. + "llama-cpp-dev": {Name: "llama-cpp-dev", Metadata: &gallery.BackendMetadata{Name: "llama-cpp-dev", Alias: "llama-cpp"}, + Nodes: []gallery.NodeBackendRef{{NodeID: "w2", NodeName: "worker-2", NodeStatus: "healthy"}}}, + "stale": {Name: "stale", Metadata: &gallery.BackendMetadata{Name: "stale", Alias: "gone"}}, + } + Expect(gallery.AvailableOnCluster(cluster, "vllm")).To(BeTrue()) + Expect(gallery.AvailableOnCluster(cluster, "orphan")).To(BeFalse()) + Expect(gallery.AvailableOnCluster(cluster, "missing")).To(BeFalse()) + Expect(gallery.AvailableOnCluster(cluster, "ailing")).To(BeFalse()) + Expect(gallery.AvailableOnCluster(cluster, "llama-cpp")).To(BeTrue()) + Expect(gallery.AvailableOnCluster(cluster, "gone")).To(BeFalse()) + }) + + It("a model install trusts the execution target's backend availability", func() { + // Empty local backend directories, as on a distributed frontend. + Expect(os.WriteFile(filepath.Join(state.Model.ModelsPath, "weights.gguf"), []byte("weights"), 0o600)).To(Succeed()) + galleries := newGallery(gallery.GalleryModel{ + Metadata: gallery.Metadata{Name: "on-worker", Backend: "vllm"}, + ConfigFile: map[string]any{"backend": "vllm", "parameters": map[string]any{"model": "weights.gguf"}}, + }) + onCluster := func(string) (bool, error) { return true, nil } + + err := gallery.InstallModelFromGallery(context.Background(), galleries, nil, state, ml, + "on-worker", gallery.GalleryModel{}, func(string, string, string, float64) {}, false, false, false, + gallery.WithBackendAvailability(onCluster)) + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Join(state.Model.ModelsPath, "on-worker.yaml")).To(BeAnExistingFile()) + }) + + It("a model install is refused before anything is downloaded when its backend is missing", func() { + galleries := newGallery(gallery.GalleryModel{ + Metadata: gallery.Metadata{ + Name: "needs-vllm", + Backend: "vllm", + // A download that cannot succeed: reaching it would fail with a + // different error than the refusal asserted below. + AdditionalFiles: []gallery.File{{Filename: "weights.gguf", URI: "http://127.0.0.1:9/weights.gguf"}}, + }, + ConfigFile: map[string]any{"backend": "vllm", "parameters": map[string]any{"model": "weights.gguf"}}, + }) + + err := gallery.InstallModelFromGallery(context.Background(), galleries, nil, state, ml, + "needs-vllm", gallery.GalleryModel{}, func(string, string, string, float64) {}, false, true, false) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + Expect(err.Error()).To(ContainSubstring(`"vllm"`)) + Expect(filepath.Join(state.Model.ModelsPath, "needs-vllm.yaml")).NotTo(BeAnExistingFile()) + Expect(filepath.Join(state.Model.ModelsPath, "weights.gguf")).NotTo(BeAnExistingFile()) + }) + + It("a model install proceeds when its backend is already present", func() { + mkBackend(state.Backend.BackendsSystemPath, "llama-cpp") + // Seeded weights keep the install off the network: an already-present + // destination with no declared sha256 counts as fetched. + Expect(os.WriteFile(filepath.Join(state.Model.ModelsPath, "weights.gguf"), []byte("weights"), 0o600)).To(Succeed()) + galleries := newGallery(gallery.GalleryModel{ + Metadata: gallery.Metadata{Name: "has-llama", Backend: "llama-cpp"}, + ConfigFile: map[string]any{"backend": "llama-cpp", "parameters": map[string]any{"model": "weights.gguf"}}, + }) + + err := gallery.InstallModelFromGallery(context.Background(), galleries, nil, state, ml, + "has-llama", gallery.GalleryModel{}, func(string, string, string, float64) {}, false, true, false) + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Join(state.Model.ModelsPath, "has-llama.yaml")).To(BeAnExistingFile()) + }) +}) diff --git a/core/gallery/backends.go b/core/gallery/backends.go index 6cb5be2bf3b5..cc360bb8e0fd 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -23,8 +23,14 @@ import ( cp "github.com/otiai10/copy" ) -// ErrBackendNotFound is returned when a backend is not found in the system. -var ErrBackendNotFound = errors.New("backend not found") +var ( + // ErrBackendNotFound is returned when a backend is not found in the system. + ErrBackendNotFound = errors.New("backend not found") + // ErrBackendInstallDisabled is returned by every install/upgrade path when + // the server runs with --disable-backend-install. Callers that install a + // backend as a side effect (model installs) treat it as a skip. + ErrBackendInstallDisabled = errors.New("backend installation is disabled on this server (--disable-backend-install)") +) // lookupImageDigest asks a registry for the digest of an image reference. It // is what install and upgrade record for upgrade detection. @@ -228,6 +234,9 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery, if name == "" { return fmt.Errorf("backend name is empty") } + if systemState.DisableBackendInstall { + return fmt.Errorf("%w: %s", ErrBackendInstallDisabled, name) + } xlog.Debug("Installing backend from gallery", "galleries", galleries, "name", name) @@ -282,7 +291,41 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery, return InstallBackend(ctx, systemState, modelLoader, backend, downloadStatus, requireIntegrity) } +// RequiredBackendInstalled reports which backend the gallery entry `name` +// would be installed with — the one variant selection picks, or the entry's +// own — and whether `available` says it can serve models. A nil `available` +// means the local backend directory. An entry that names no backend reports +// "" and true: nothing to check. +func RequiredBackendInstalled(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState, name, pin string, available BackendAvailability) (string, bool, error) { + models, err := AvailableGalleryModels(galleries, systemState) + if err != nil { + return "", false, err + } + entry := FindGalleryElement(models, name) + if entry == nil { + return "", false, fmt.Errorf("no model found with name %q", name) + } + if entry.HasVariants() { + resolved, _, err := ResolveVariant(models, entry, HostResolveEnv(ctx, systemState), pin) + if err != nil { + return "", false, err + } + entry = resolved + } + if entry.Backend == "" { + return "", true, nil + } + if available == nil { + available = LocalBackendAvailability(systemState) + } + ok, err := available(entry.Backend) + return entry.Backend, ok, err +} + func InstallBackend(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, config *GalleryBackend, downloadStatus func(string, string, string, float64), requireIntegrity bool) error { + if systemState.DisableBackendInstall { + return fmt.Errorf("%w: %s", ErrBackendInstallDisabled, config.Name) + } // Get configurable fallback tag values from SystemState latestTag, masterTag, devSuffix := getFallbackTagValues(systemState) @@ -575,6 +618,50 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error return os.RemoveAll(backendDirectory) } +// BackendAvailability answers whether a backend can serve models for this +// server: installed and runnable on a single node, or installed on a healthy +// worker when inference runs on a cluster. +type BackendAvailability func(name string) (bool, error) + +// LocalBackendAvailability reports the backends installed on this host. +func LocalBackendAvailability(systemState *system.SystemState) BackendAvailability { + return func(name string) (bool, error) { + backends, err := ListSystemBackends(systemState) + if err != nil { + return false, err + } + b, ok := backends.Get(name) + return ok && isBackendRunnable(b), nil + } +} + +// AvailableOnCluster reports whether a cluster-wide listing shows the backend +// on at least one healthy node. Such entries describe workers, not local +// files, so presence is the test, and only nodes the router may place a load +// on count. Workers list only concrete backends and keep alias resolution to +// themselves, so a name is also matched against each entry's alias: a model +// naming the alias runs on the worker that holds the concrete. +func AvailableOnCluster(backends SystemBackends, name string) bool { + if b, ok := backends.Get(name); ok && servedByHealthyNode(b) { + return true + } + for _, b := range backends { + if b.Metadata != nil && b.Metadata.Alias == name && servedByHealthyNode(b) { + return true + } + } + return false +} + +func servedByHealthyNode(b SystemBackend) bool { + for _, n := range b.Nodes { + if n.NodeStatus == "healthy" { + return true + } + } + return false +} + // isBackendRunnable reports whether the given backend entry can actually be // invoked. A meta backend is runnable only if its concrete's run.sh still // exists on disk; concrete backends are considered runnable as long as their diff --git a/core/gallery/model_artifacts.go b/core/gallery/model_artifacts.go index 2fa46a0d31b4..af3fb6a524c0 100644 --- a/core/gallery/model_artifacts.go +++ b/core/gallery/model_artifacts.go @@ -17,12 +17,23 @@ type ArtifactMaterializer interface { } type installOptions struct { - materializer ArtifactMaterializer - variant string + materializer ArtifactMaterializer + variant string + backendAvailability BackendAvailability } type InstallOption func(*installOptions) +// WithBackendAvailability supplies the execution target's view of installed +// backends; absent, the local host's. +func WithBackendAvailability(available BackendAvailability) InstallOption { + return func(options *installOptions) { + if available != nil { + options.backendAvailability = available + } + } +} + func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption { return func(options *installOptions) { if materializer != nil { diff --git a/core/gallery/models.go b/core/gallery/models.go index 6d4ceb346231..055186e0d9ea 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -673,6 +673,20 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver return nil, fmt.Errorf("failed to validate updated config YAML: %v", err) } + if systemState.DisableBackendInstall && modelConfig.Backend != "" { + available := installOptions.backendAvailability + if available == nil { + available = LocalBackendAvailability(systemState) + } + ok, err := available(modelConfig.Backend) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%w: model %q needs backend %q, which is not installed", ErrBackendInstallDisabled, name, modelConfig.Backend) + } + } + if len(config.Files) == 0 { artifactSpec, inferred, hasArtifact, err := modelConfig.PrimaryArtifactSpec(basePath) if err != nil { diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go index 09a548e37595..68f89525b836 100644 --- a/core/gallery/upgrade.go +++ b/core/gallery/upgrade.go @@ -257,6 +257,9 @@ func summarizeNodeDrift(nodes []NodeBackendRef) (majority struct{ version, diges // UpgradeBackend upgrades a single backend to the latest gallery version using // an atomic swap with backup-based rollback on failure. func UpgradeBackend(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, galleries []config.Gallery, backendName string, downloadStatus func(string, string, string, float64), requireIntegrity bool) error { + if systemState.DisableBackendInstall { + return fmt.Errorf("%w: %s", ErrBackendInstallDisabled, backendName) + } // Look up the installed backend installedBackends, err := ListSystemBackends(systemState) if err != nil { diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index d8a5612a18e3..3c7ab31c3cc8 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -138,6 +138,8 @@ func (mgs *BackendEndpointService) GetAllStatusEndpoint() echo.HandlerFunc { // @Tags backends // @Param request body GalleryBackend true "query params" // @Success 200 {object} schema.BackendResponse "Response" +// @Failure 403 {object} map[string]interface{} "Backend installation is disabled on this server (code backend_install_disabled)" +// @Failure 409 {object} map[string]interface{} "Hardware-specific build in distributed mode (code concrete_backend_requires_target)" // @Router /backends/apply [post] func (mgs *BackendEndpointService) ApplyBackendEndpoint(systemState *system.SystemState) echo.HandlerFunc { return func(c echo.Context) error { @@ -330,6 +332,7 @@ func (mgs *BackendEndpointService) CheckUpgradesEndpoint() echo.HandlerFunc { // @Tags backends // @Param name path string true "Backend name" // @Success 200 {object} schema.BackendResponse "Response" +// @Failure 403 {object} map[string]interface{} "Backend installation is disabled on this server (code backend_install_disabled)" // @Router /backends/upgrade/{name} [post] func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc { return func(c echo.Context) error { diff --git a/core/http/endpoints/localai/gallery.go b/core/http/endpoints/localai/gallery.go index 61d5d0169648..d71a5bb21a84 100644 --- a/core/http/endpoints/localai/gallery.go +++ b/core/http/endpoints/localai/gallery.go @@ -3,6 +3,7 @@ package localai import ( "encoding/json" "fmt" + "net/http" "github.com/google/uuid" "github.com/labstack/echo/v4" @@ -16,6 +17,7 @@ import ( ) type ModelGalleryEndpointService struct { + appConfig *config.ApplicationConfig galleries []config.Gallery backendGalleries []config.Gallery modelPath string @@ -23,6 +25,27 @@ type ModelGalleryEndpointService struct { configLoader *config.ModelConfigLoader } +// RefuseModelWithoutBackend answers 409 when backend installation is disabled +// and the model's backend is not already present: with no way to acquire it, +// the install is certain to produce an unusable model. It reports whether it +// wrote the refusal (the handler must then return its error and stop), so a +// nil write error is never mistaken for "not refused". Resolution errors are +// not refusals — the install path reports them in its own words. +func RefuseModelWithoutBackend(c echo.Context, appConfig *config.ApplicationConfig, galleries []config.Gallery, name, pin string, available gallery.BackendAvailability) (bool, error) { + if !appConfig.DisableBackendInstall { + return false, nil + } + backend, installed, err := gallery.RequiredBackendInstalled(c.Request().Context(), galleries, appConfig.SystemState, name, pin, available) + if err != nil || installed { + return false, nil + } + return true, c.JSON(http.StatusConflict, map[string]any{ + "error": fmt.Sprintf("Model %q needs backend %q, which is not installed, and backend installation is disabled on this server.", name, backend), + "code": "backend_not_installed", + "backend": backend, + }) +} + type GalleryModel struct { ID string `json:"id"` // Variant installs one specific build of an entry that declares variants, @@ -33,11 +56,12 @@ type GalleryModel struct { gallery.GalleryModel } -func CreateModelGalleryEndpointService(galleries []config.Gallery, backendGalleries []config.Gallery, systemState *system.SystemState, galleryApplier *galleryop.GalleryService, configLoader *config.ModelConfigLoader) ModelGalleryEndpointService { +func CreateModelGalleryEndpointService(appConfig *config.ApplicationConfig, galleryApplier *galleryop.GalleryService, configLoader *config.ModelConfigLoader) ModelGalleryEndpointService { return ModelGalleryEndpointService{ - galleries: galleries, - backendGalleries: backendGalleries, - modelPath: systemState.Model.ModelsPath, + appConfig: appConfig, + galleries: appConfig.Galleries, + backendGalleries: appConfig.BackendGalleries, + modelPath: appConfig.SystemState.Model.ModelsPath, galleryApplier: galleryApplier, configLoader: configLoader, } @@ -74,6 +98,7 @@ func (mgs *ModelGalleryEndpointService) GetAllStatusEndpoint() echo.HandlerFunc // @Tags models // @Param request body GalleryModel true "query params" // @Success 200 {object} schema.GalleryResponse "Response" +// @Failure 409 {object} map[string]interface{} "The model's backend is not installed and backend installation is disabled (code backend_not_installed)" // @Router /models/apply [post] func (mgs *ModelGalleryEndpointService) ApplyModelGalleryEndpoint() echo.HandlerFunc { return func(c echo.Context) error { @@ -83,6 +108,10 @@ func (mgs *ModelGalleryEndpointService) ApplyModelGalleryEndpoint() echo.Handler return err } + if refused, err := RefuseModelWithoutBackend(c, mgs.appConfig, mgs.galleries, input.ID, input.Variant, mgs.galleryApplier.BackendManager().BackendAvailable); refused { + return err + } + uuid, err := uuid.NewUUID() if err != nil { return err diff --git a/core/http/middleware/backend_install.go b/core/http/middleware/backend_install.go new file mode 100644 index 000000000000..1578ce665b5e --- /dev/null +++ b/core/http/middleware/backend_install.go @@ -0,0 +1,27 @@ +package middleware + +import ( + "net/http" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" +) + +// RequireBackendInstallEnabled rejects backend install and upgrade requests +// up front when the server runs with --disable-backend-install. The install +// path refuses them too (gallery.ErrBackendInstallDisabled), but that only +// surfaces as a failed background job; refusing at admission gives the +// client an immediate, explicit answer. +func RequireBackendInstallEnabled(appConfig *config.ApplicationConfig) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if appConfig.DisableBackendInstall { + return c.JSON(http.StatusForbidden, map[string]any{ + "error": "Backend installation is disabled on this server (--disable-backend-install). Backends are provided by the system; install the backend outside LocalAI.", + "code": "backend_install_disabled", + }) + } + return next(c) + } + } +} diff --git a/core/http/middleware/backend_install_test.go b/core/http/middleware/backend_install_test.go new file mode 100644 index 000000000000..44a5b08954cf --- /dev/null +++ b/core/http/middleware/backend_install_test.go @@ -0,0 +1,42 @@ +package middleware_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/http/middleware" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("RequireBackendInstallEnabled", func() { + run := func(disabled bool) (*httptest.ResponseRecorder, bool) { + called := false + handler := func(c echo.Context) error { called = true; return c.NoContent(http.StatusOK) } + req := httptest.NewRequest(http.MethodPost, "/backends/apply", nil) + rec := httptest.NewRecorder() + c := echo.New().NewContext(req, rec) + err := RequireBackendInstallEnabled(&config.ApplicationConfig{DisableBackendInstall: disabled})(handler)(c) + Expect(err).NotTo(HaveOccurred()) + return rec, called + } + + It("passes requests through when installs are allowed", func() { + rec, called := run(false) + Expect(called).To(BeTrue()) + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + + It("answers 403 with a stable code when installs are disabled", func() { + rec, called := run(true) + Expect(called).To(BeFalse()) + Expect(rec.Code).To(Equal(http.StatusForbidden)) + var body map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &body)).To(Succeed()) + Expect(body["code"]).To(Equal("backend_install_disabled")) + Expect(body["error"]).To(ContainSubstring("disable-backend-install")) + }) +}) diff --git a/core/http/react-ui/public/locales/en/importModel.json b/core/http/react-ui/public/locales/en/importModel.json index 84b783ccee6a..2201b95529f8 100644 --- a/core/http/react-ui/public/locales/en/importModel.json +++ b/core/http/react-ui/public/locales/en/importModel.json @@ -38,6 +38,7 @@ "backendHint": "Narrow the list by what the model does, then pick one. Leave it on auto-detect and LocalAI decides. Entries marked \"manual pick\" are never auto-detected.", "backendErrorHint": "Could not load the backend list. Auto-detect only.", "backendNotInstalled": "This backend isn't installed yet. Submitting import will download it first.", + "backendInstallDisabled": "This backend isn't installed, and this server does not install backends. The import will be refused until the backend is installed outside LocalAI.", "modelName": "Name", "modelNamePlaceholder": "Leave empty to use filename", "modelNameHint": "Custom name for the model. If empty, the filename will be used.", diff --git a/core/http/react-ui/public/locales/en/operate.json b/core/http/react-ui/public/locales/en/operate.json index a6a04b6c889b..b88747fc2180 100644 --- a/core/http/react-ui/public/locales/en/operate.json +++ b/core/http/react-ui/public/locales/en/operate.json @@ -365,6 +365,9 @@ "actionsFor": "Actions for {{name}}", "protected": "Protected", "protectedTitle": "System backends are managed outside the gallery", + "installDisabled": "Backend installation is disabled on this server. Backends are provided by the system and installed outside LocalAI.", + "installDisabledTitle": "Backend installation is disabled on this server", + "installDisabledShort": "Install disabled", "logs": "Logs", "usedBy_one": "Used by {{count}} model:", "usedBy_other": "Used by {{count}} models:", diff --git a/core/http/react-ui/src/components/hub/hubConfig.js b/core/http/react-ui/src/components/hub/hubConfig.js index 0647a4332a74..423c45a04ea1 100644 --- a/core/http/react-ui/src/components/hub/hubConfig.js +++ b/core/http/react-ui/src/components/hub/hubConfig.js @@ -11,7 +11,7 @@ // Gating is per item and decides the tab: a tab shows when at least one of its // items is visible to the viewer, so a tab nobody can use is never drawn. // `/api/features` only emits: agents, mcp, fine_tuning, quantization, -// distributed, localai_assistant. Capability-style flags (face_recognition, +// distributed, backend_install, localai_assistant. Capability-style flags (face_recognition, // skills, ...) come from hasFeature(), not the features map. // // Item and tab fields: diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index b9874a11d71b..7ba1ee8f8529 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -3,6 +3,8 @@ import { useState, useEffect, useCallback, useMemo, useRef } from 'react' import { Link, useOutletContext, useSearchParams } from 'react-router-dom' import { useTranslation } from 'react-i18next' import { backendsApi, modelsApi, nodesApi } from '../utils/api' +import { apiUrl } from '../utils/basePath' +import { FEATURE_BACKEND_INSTALL } from '../utils/capabilities' import { useDebouncedCallback } from '../hooks/useDebounce' import { useOperations } from '../hooks/useOperations' import { useDistributedMode } from '../hooks/useDistributedMode' @@ -61,6 +63,16 @@ export default function Backends() { const [sortOrder, setSortOrder] = useState('asc') const [page, setPage] = useState(1) const [showManualInstall, setShowManualInstall] = useState(false) + // Servers run with --disable-backend-install answer 403 to every install + // and upgrade; the page shows that up front instead of after the click. + const [installEnabled, setInstallEnabled] = useState(true) + + useEffect(() => { + fetch(apiUrl('/api/features')) + .then(r => r.json()) + .then(data => setInstallEnabled(data[FEATURE_BACKEND_INSTALL] !== false)) + .catch(() => {}) + }, []) const [manualUri, setManualUri] = useState('') const [manualName, setManualName] = useState('') const [manualAlias, setManualAlias] = useState('') @@ -663,6 +675,10 @@ export default function Backends() { : null } else if (state.kind === 'removing') { action = null + } else if (!installEnabled) { + action = b.installed + ? null + : {t('backends.installDisabledShort')} } else if (state.kind === 'failed') { action = } else if (isNodeScoped) { @@ -807,6 +823,18 @@ export default function Backends() { : null } else if (b.IsSystem) { action = {t('backends.protected')} + } else if (!installEnabled) { + // Update and reinstall acquire; delete does not. + action = ( + handleDelete(name, 'installed') }, + ]} + /> + ) } else { action = ( <> @@ -850,7 +878,7 @@ export default function Backends() { )} {usedBy(name)}
- {!b.IsSystem && upgrade && ( + {installEnabled && !b.IsSystem && upgrade && ( @@ -917,18 +945,22 @@ export default function Backends() {
- {updateNames.length > 0 && ( + {installEnabled && updateNames.length > 0 && ( )} - - + {installEnabled && ( + + )} + {installEnabled && ( + + )}
@@ -983,6 +1015,13 @@ export default function Backends() { + {!installEnabled && ( +
+ + {t('backends.installDisabled')} +
+ )} + {activeView === 'catalog' && targetNode && (
@@ -1021,7 +1060,7 @@ export default function Backends() { )} - {recommended && ( + {installEnabled && recommended && (

{t('backends.recommendTitle', { name: recommended.name || recommended.id })}

diff --git a/core/http/react-ui/src/pages/ImportModel.jsx b/core/http/react-ui/src/pages/ImportModel.jsx index 41709cd733ef..18b9548cff12 100644 --- a/core/http/react-ui/src/pages/ImportModel.jsx +++ b/core/http/react-ui/src/pages/ImportModel.jsx @@ -3,6 +3,8 @@ import { useState, useRef, useCallback, useEffect, useMemo } from 'react' import { Link, useNavigate, useOutletContext } from 'react-router-dom' import { useTranslation } from 'react-i18next' import { modelsApi, backendsApi } from '../utils/api' +import { apiUrl } from '../utils/basePath' +import { FEATURE_BACKEND_INSTALL } from '../utils/capabilities' import { formatBytes } from '../utils/format' import { useResources } from '../hooks/useResources' import { detectSource, fitFor, importChecks, importRequest, machineFacts, requestText } from '../utils/tools' @@ -181,6 +183,14 @@ export default function ImportModel() { const [backends, setBackends] = useState([]) const [backendsLoading, setBackendsLoading] = useState(true) const [backendsError, setBackendsError] = useState(false) + const [installEnabled, setInstallEnabled] = useState(true) + + useEffect(() => { + fetch(apiUrl('/api/features')) + .then(r => r.json()) + .then(data => setInstallEnabled(data[FEATURE_BACKEND_INSTALL] !== false)) + .catch(() => {}) + }, []) // Wide enough for the reference to sit beside the form. Tracked in state // rather than read at render so a resize re-renders the page. @@ -482,6 +492,14 @@ export default function ImportModel() { if (!prefs.backend) return null const selected = backends.find(b => b.name === prefs.backend) if (!selected || selected.installed) return null + if (!installEnabled) { + return ( +

+ + {t('form.backendInstallDisabled')} +

+ ) + } return (

diff --git a/core/http/react-ui/src/utils/capabilities.js b/core/http/react-ui/src/utils/capabilities.js index 61a3c1fda7e9..1c84227ee7ec 100644 --- a/core/http/react-ui/src/utils/capabilities.js +++ b/core/http/react-ui/src/utils/capabilities.js @@ -2,6 +2,7 @@ // Server feature keys (returned by GET /api/features), kept as constants so // callers don't sprinkle string literals across the UI. export const FEATURE_LOCALAI_ASSISTANT = 'localai_assistant' +export const FEATURE_BACKEND_INSTALL = 'backend_install' export const CAP_CHAT = 'FLAG_CHAT' export const CAP_COMPLETION = 'FLAG_COMPLETION' diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 142e755932c4..4f6fa06d9434 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -64,7 +64,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, // Edit model page router.GET("/models/edit/:name", localai.GetEditModelPage(cl, appConfig), adminMiddleware) - modelGalleryEndpointService := localai.CreateModelGalleryEndpointService(appConfig.Galleries, appConfig.BackendGalleries, appConfig.SystemState, galleryService, cl) + modelGalleryEndpointService := localai.CreateModelGalleryEndpointService(appConfig, galleryService, cl) router.POST("/models/apply", modelGalleryEndpointService.ApplyModelGalleryEndpoint(), adminMiddleware) router.POST("/models/delete/:name", modelGalleryEndpointService.DeleteModelGalleryEndpoint(), adminMiddleware) @@ -78,7 +78,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, appConfig.SystemState, galleryService, app.UpgradeChecker()) - router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(appConfig.SystemState), adminMiddleware) + router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(appConfig.SystemState), adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware) router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware) router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app)), adminMiddleware) @@ -87,7 +87,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, router.GET("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint(), adminMiddleware) router.GET("/backends/upgrades", backendGalleryEndpointService.GetUpgradesEndpoint(), adminMiddleware) router.POST("/backends/upgrades/check", backendGalleryEndpointService.CheckUpgradesEndpoint(), adminMiddleware) - router.POST("/backends/upgrade/:name", backendGalleryEndpointService.UpgradeBackendEndpoint(), adminMiddleware) + router.POST("/backends/upgrade/:name", backendGalleryEndpointService.UpgradeBackendEndpoint(), adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) // Custom model import endpoint router.POST("/models/import", localai.ImportModelEndpoint(cl, galleryService, appConfig, opcache), adminMiddleware) @@ -458,6 +458,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, "config_patch": true, "vram_estimate": true, "model_storage": true, + "backend_install": !appConfig.DisableBackendInstall, "mcp": !appConfig.DisableMCP, "agents": appConfig.AgentPool.Enabled, "p2p": appConfig.P2PToken != "", @@ -479,6 +480,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, "fine_tuning": true, "quantization": true, "distributed": appConfig.Distributed.Enabled, + "backend_install": !appConfig.DisableBackendInstall, "localai_assistant": !appConfig.DisableLocalAIAssistant && app.LocalAIAssistant() != nil, }) }) diff --git a/core/http/routes/nodes.go b/core/http/routes/nodes.go index 2e55cd10b545..7b784342854f 100644 --- a/core/http/routes/nodes.go +++ b/core/http/routes/nodes.go @@ -8,6 +8,7 @@ import ( "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/pkg/natsauth" @@ -89,11 +90,11 @@ func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloade // Backend management on workers admin.GET("/:id/backends", localai.ListBackendsOnNodeEndpoint(unloader, registry)) - admin.POST("/:id/backends/install", localai.InstallBackendOnNodeEndpoint(unloader, galleryService, opcache, appConfig)) + admin.POST("/:id/backends/install", localai.InstallBackendOnNodeEndpoint(unloader, galleryService, opcache, appConfig), middleware.RequireBackendInstallEnabled(appConfig)) // Upgrade is a distinct route (not install) because the worker's // backend.install handler short-circuits when the backend already exists // on disk; only the Upgrade op path force-reinstalls. - admin.POST("/:id/backends/upgrade", localai.UpgradeBackendOnNodeEndpoint(galleryService, opcache, appConfig)) + admin.POST("/:id/backends/upgrade", localai.UpgradeBackendOnNodeEndpoint(galleryService, opcache, appConfig), middleware.RequireBackendInstallEnabled(appConfig)) admin.POST("/:id/backends/delete", localai.DeleteBackendOnNodeEndpoint(unloader)) // Model management on workers diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index b53371381beb..28aef9a60640 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -5,6 +5,7 @@ import "os" import ( "cmp" "context" + "errors" "fmt" "math" "net/http" @@ -22,6 +23,7 @@ import ( "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/galleryop" @@ -1062,6 +1064,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model variant := c.QueryParam("variant") xlog.Debug("API job submitted to install", "galleryID", galleryID, "variant", variant) + if refused, err := localai.RefuseModelWithoutBackend(c, appConfig, appConfig.Galleries, galleryID, variant, galleryService.BackendManager().BackendAvailable); refused { + return err + } + id, err := uuid.NewUUID() if err != nil { return c.JSON(http.StatusInternalServerError, map[string]any{ @@ -1175,8 +1181,15 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model } _, err = gallery.InstallModel(context.Background(), appConfig.SystemState, model.Name, &config, model.Overrides, nil, false, - gallery.WithArtifactMaterializer(appConfig.ModelArtifactMaterializer)) + gallery.WithArtifactMaterializer(appConfig.ModelArtifactMaterializer), + gallery.WithBackendAvailability(galleryService.BackendManager().BackendAvailable)) if err != nil { + if errors.Is(err, gallery.ErrBackendInstallDisabled) { + return c.JSON(http.StatusConflict, map[string]any{ + "error": err.Error(), + "code": "backend_not_installed", + }) + } return c.JSON(http.StatusInternalServerError, map[string]any{ "error": err.Error(), }) @@ -1550,7 +1563,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "jobID": uid, "message": "Backend installation started", }) - }, adminMiddleware) + }, adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) // Install backend from external source (OCI image, URL, or path) app.POST("/api/backends/install-external", func(c echo.Context) error { @@ -1613,7 +1626,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "jobID": uid, "message": "External backend installation started", }) - }, adminMiddleware) + }, adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) app.POST("/api/backends/delete/:id", func(c echo.Context) error { backendID := c.Param("id") @@ -1782,7 +1795,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "statusUrl": fmt.Sprintf("/api/backends/job/%s", uid), "message": "Backend upgrade started", }) - }, adminMiddleware) + }, adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) // P2P APIs app.GET("/api/p2p/workers", func(c echo.Context) error { diff --git a/core/http/routes/ui_api_backend_install_disabled_test.go b/core/http/routes/ui_api_backend_install_disabled_test.go new file mode 100644 index 000000000000..4c9cf058f38f --- /dev/null +++ b/core/http/routes/ui_api_backend_install_disabled_test.go @@ -0,0 +1,222 @@ +package routes_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("UI API with backend installation disabled", func() { + var ( + app *echo.Echo + tempDir string + appConfig *config.ApplicationConfig + ) + galleryRevision := 0 + + post := func(path string, body any) *httptest.ResponseRecorder { + var buf bytes.Buffer + if body != nil { + Expect(json.NewEncoder(&buf).Encode(body)).To(Succeed()) + } + req := httptest.NewRequest(http.MethodPost, path, &buf) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + return rec + } + + decode := func(rec *httptest.ResponseRecorder) map[string]any { + var body map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &body)).To(Succeed()) + return body + } + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "backend-install-disabled-routes-*") + Expect(err).NotTo(HaveOccurred()) + + systemState, err := system.GetSystemState( + system.WithBackendPath(filepath.Join(tempDir, "backends")), + system.WithModelPath(filepath.Join(tempDir, "models")), + system.WithDisableBackendInstall(true), + ) + Expect(err).NotTo(HaveOccurred()) + Expect(os.MkdirAll(systemState.Backend.BackendsPath, 0o750)).To(Succeed()) + Expect(os.MkdirAll(systemState.Model.ModelsPath, 0o750)).To(Succeed()) + + // A file:// model gallery inside the models path (the only root a file + // URL is trusted from): one entry needing a backend nobody has, one + // declaring none. Unique name per spec — the listing is cached. + entries := []gallery.GalleryModel{ + {Metadata: gallery.Metadata{Name: "needs-vllm", Backend: "vllm"}}, + {Metadata: gallery.Metadata{Name: "no-backend"}}, + } + out, err := yaml.Marshal(entries) + Expect(err).NotTo(HaveOccurred()) + galleryName := fmt.Sprintf("disabled-%d", galleryRevision) + galleryRevision++ + galleryPath := filepath.Join(systemState.Model.ModelsPath, galleryName+".yaml") + Expect(os.WriteFile(galleryPath, out, 0o600)).To(Succeed()) + + modelLoader := model.NewModelLoader(systemState) + configLoader := config.NewModelConfigLoader(tempDir) + appConfig = config.NewApplicationConfig(config.WithContext(context.Background())) + appConfig.SystemState = systemState + appConfig.DisableBackendInstall = true + appConfig.Galleries = []config.Gallery{{Name: galleryName, URL: "file://" + galleryPath}} + appConfig.BackendGalleries = []config.Gallery{} + + galleryService := galleryop.NewGalleryService(appConfig, modelLoader) + // Not started: nothing here should reach the queue, and a 200 for the + // pass-through case is asserted on the response alone. + app = echo.New() + noopMw := func(next echo.HandlerFunc) echo.HandlerFunc { return next } + routes.RegisterUIAPIRoutes(app, configLoader, modelLoader, appConfig, galleryService, galleryop.NewOpCache(galleryService), nil, noopMw) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + It("refuses backend installs and upgrades with 403", func() { + for _, path := range []string{ + "/api/backends/install/llama-cpp", + "/api/backends/install-external", + "/api/backends/upgrade/llama-cpp", + } { + rec := post(path, map[string]string{"uri": "quay.io/example/backend:latest", "name": "x"}) + Expect(rec.Code).To(Equal(http.StatusForbidden), path) + Expect(decode(rec)["code"]).To(Equal("backend_install_disabled"), path) + } + }) + + It("refuses a model whose backend is missing with 409 and names the backend", func() { + rec := post("/api/models/install/"+appConfig.Galleries[0].Name+"@needs-vllm", nil) + Expect(rec.Code).To(Equal(http.StatusConflict)) + body := decode(rec) + Expect(body["code"]).To(Equal("backend_not_installed")) + Expect(body["backend"]).To(Equal("vllm")) + }) + + It("lets a model that names no backend through", func() { + rec := post("/api/models/install/"+appConfig.Galleries[0].Name+"@no-backend", nil) + Expect(rec.Code).To(Equal(http.StatusOK)) + }) +}) + +// A distributed frontend keeps no backends of its own; what counts is what the +// workers have. clusterBackendManager stands in for the NATS-backed manager +// and answers availability from a fixed set. +type clusterBackendManager struct { + available map[string]bool +} + +func (clusterBackendManager) InstallBackend(_ context.Context, _ *galleryop.ManagementOp[gallery.GalleryBackend, any], _ galleryop.ProgressCallback) error { + return nil +} +func (clusterBackendManager) DeleteBackend(_ string) error { return nil } +func (clusterBackendManager) ListBackends() (gallery.SystemBackends, error) { + return gallery.SystemBackends{}, nil +} +func (clusterBackendManager) UpgradeBackend(_ context.Context, _ *galleryop.ManagementOp[gallery.GalleryBackend, any], _ galleryop.ProgressCallback) error { + return nil +} +func (clusterBackendManager) CheckUpgrades(_ context.Context) (map[string]gallery.UpgradeInfo, error) { + return nil, nil +} +func (clusterBackendManager) IsDistributed() bool { return true } +func (m clusterBackendManager) BackendAvailable(name string) (bool, error) { + return m.available[name], nil +} + +var _ = Describe("Model installs on a distributed frontend with backend installation disabled", func() { + var ( + app *echo.Echo + tempDir string + galleryName string + ) + galleryRevision := 0 + + setup := func(workersHave map[string]bool) { + var err error + tempDir, err = os.MkdirTemp("", "backend-install-disabled-cluster-*") + Expect(err).NotTo(HaveOccurred()) + + // Empty backends directory: the frontend installs nothing locally. + systemState, err := system.GetSystemState( + system.WithBackendPath(filepath.Join(tempDir, "backends")), + system.WithModelPath(filepath.Join(tempDir, "models")), + system.WithDisableBackendInstall(true), + ) + Expect(err).NotTo(HaveOccurred()) + Expect(os.MkdirAll(systemState.Backend.BackendsPath, 0o750)).To(Succeed()) + Expect(os.MkdirAll(systemState.Model.ModelsPath, 0o750)).To(Succeed()) + + entries := []gallery.GalleryModel{{Metadata: gallery.Metadata{Name: "needs-vllm", Backend: "vllm"}}} + out, err := yaml.Marshal(entries) + Expect(err).NotTo(HaveOccurred()) + galleryName = fmt.Sprintf("cluster-%d", galleryRevision) + galleryRevision++ + galleryPath := filepath.Join(systemState.Model.ModelsPath, galleryName+".yaml") + Expect(os.WriteFile(galleryPath, out, 0o600)).To(Succeed()) + + modelLoader := model.NewModelLoader(systemState) + configLoader := config.NewModelConfigLoader(tempDir) + appConfig := config.NewApplicationConfig(config.WithContext(context.Background())) + appConfig.SystemState = systemState + appConfig.DisableBackendInstall = true + appConfig.Galleries = []config.Gallery{{Name: galleryName, URL: "file://" + galleryPath}} + appConfig.BackendGalleries = []config.Gallery{} + + galleryService := galleryop.NewGalleryService(appConfig, modelLoader) + galleryService.SetBackendManager(clusterBackendManager{available: workersHave}) + app = echo.New() + noopMw := func(next echo.HandlerFunc) echo.HandlerFunc { return next } + routes.RegisterUIAPIRoutes(app, configLoader, modelLoader, appConfig, galleryService, galleryop.NewOpCache(galleryService), nil, noopMw) + } + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + post := func(path string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, path, nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + return rec + } + + It("accepts a model whose backend a worker already has", func() { + setup(map[string]bool{"vllm": true}) + rec := post("/api/models/install/" + galleryName + "@needs-vllm") + Expect(rec.Code).To(Equal(http.StatusOK), rec.Body.String()) + }) + + It("refuses a model whose backend no worker has", func() { + setup(map[string]bool{}) + rec := post("/api/models/install/" + galleryName + "@needs-vllm") + Expect(rec.Code).To(Equal(http.StatusConflict), rec.Body.String()) + var body map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &body)).To(Succeed()) + Expect(body["code"]).To(Equal("backend_not_installed")) + Expect(body["backend"]).To(Equal("vllm")) + }) +}) diff --git a/core/services/galleryop/backend_delete_optype_test.go b/core/services/galleryop/backend_delete_optype_test.go index b73ce9f87dfc..ff12b278df30 100644 --- a/core/services/galleryop/backend_delete_optype_test.go +++ b/core/services/galleryop/backend_delete_optype_test.go @@ -43,6 +43,8 @@ func (blockingBackendManager) CheckUpgrades(_ context.Context) (map[string]galle func (blockingBackendManager) IsDistributed() bool { return false } +func (blockingBackendManager) BackendAvailable(_ string) (bool, error) { return false, nil } + // What the backend channel persists about a removal. The row is all a replica // that restarts mid-operation has to go on, so a removal recorded as an install // is a removal that renders as "Installing backend X" and offers a Cancel diff --git a/core/services/galleryop/backends.go b/core/services/galleryop/backends.go index daa99016838d..e527c6d1721a 100644 --- a/core/services/galleryop/backends.go +++ b/core/services/galleryop/backends.go @@ -128,7 +128,10 @@ func (g *GalleryService) backendHandler(op *ManagementOp[gallery.GalleryBackend, return nil } xlog.Error("error installing backend", "error", err, "backend", op.GalleryElementName) - if !op.Delete { + // A refused op wrote nothing, so there is nothing to clean up — and + // the name may belong to an installed backend a forced reinstall + // targeted. + if !op.Delete && !errors.Is(err, gallery.ErrBackendInstallDisabled) { // If we didn't install the backend, we need to make sure we don't have a leftover directory gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName) } diff --git a/core/services/galleryop/managers.go b/core/services/galleryop/managers.go index 6435a9963249..de806db032e3 100644 --- a/core/services/galleryop/managers.go +++ b/core/services/galleryop/managers.go @@ -27,4 +27,7 @@ type BackendManager interface { // on /api/backends/apply in distributed mode — a CPU build silently // landing on every GPU node is the footgun this guards against. IsDistributed() bool + // BackendAvailable reports whether models naming this backend can run + // on the execution target this manager represents. + BackendAvailable(name string) (bool, error) } diff --git a/core/services/galleryop/managers_local.go b/core/services/galleryop/managers_local.go index 70fc018e000b..28d646a59f17 100644 --- a/core/services/galleryop/managers_local.go +++ b/core/services/galleryop/managers_local.go @@ -51,7 +51,8 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g case op.GalleryElement != nil: installedModel, err := gallery.InstallModel(ctx, m.systemState, op.GalleryElement.Name, op.GalleryElement, op.Req.Overrides, progressCb, m.enforcePredownloadScans, - gallery.WithArtifactMaterializer(m.artifactMaterializer)) + gallery.WithArtifactMaterializer(m.artifactMaterializer), + gallery.WithBackendAvailability(op.BackendAvailability)) if err != nil { return err } @@ -62,7 +63,10 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g } return nil case op.GalleryElementName != "": - opts := []gallery.InstallOption{gallery.WithArtifactMaterializer(m.artifactMaterializer)} + opts := []gallery.InstallOption{ + gallery.WithArtifactMaterializer(m.artifactMaterializer), + gallery.WithBackendAvailability(op.BackendAvailability), + } if op.Variant != "" { opts = append(opts, gallery.WithVariant(op.Variant)) } @@ -73,7 +77,8 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g default: return installModelFromRemoteConfig(ctx, m.systemState, m.modelLoader, op.Req, progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, op.BackendGalleries, m.requireBackendIntegrity, - gallery.WithArtifactMaterializer(m.artifactMaterializer)) + gallery.WithArtifactMaterializer(m.artifactMaterializer), + gallery.WithBackendAvailability(op.BackendAvailability)) } } @@ -130,3 +135,7 @@ func (b *LocalBackendManager) InstallBackend(ctx context.Context, op *Management } func (b *LocalBackendManager) IsDistributed() bool { return false } + +func (b *LocalBackendManager) BackendAvailable(name string) (bool, error) { + return gallery.LocalBackendAvailability(b.systemState)(name) +} diff --git a/core/services/galleryop/models.go b/core/services/galleryop/models.go index cfb1d59f56c8..6b1077cc1d56 100644 --- a/core/services/galleryop/models.go +++ b/core/services/galleryop/models.go @@ -142,6 +142,9 @@ func (g *GalleryService) modelHandlerLocked(op *ManagementOp[gallery.GalleryMode deleteStarted = true err = g.modelManager.DeleteModel(op.GalleryElementName) } else { + if op.BackendAvailability == nil { + op.BackendAvailability = g.backendManager.BackendAvailable + } err = g.modelManager.InstallModel(operationCtx, op, progressCallback) } if err != nil { diff --git a/core/services/galleryop/operation.go b/core/services/galleryop/operation.go index 1bfee64205e7..d99c55022111 100644 --- a/core/services/galleryop/operation.go +++ b/core/services/galleryop/operation.go @@ -10,6 +10,7 @@ import ( "time" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/pkg/xsync" @@ -46,6 +47,12 @@ type ManagementOp[T any, E any] struct { // build on one node without touching the rest of the cluster. TargetNodeID string + // BackendAvailability is the execution target's view of installed + // backends, set by the gallery service when it dispatches a model op so + // the install layer does not have to know whether inference is local or + // on workers. + BackendAvailability gallery.BackendAvailability + // Variant pins a model install to one of the gallery entry's declared // variants, by that variant's model name. Empty means auto-select: LocalAI // picks the largest variant this host's backend support and memory can diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 0ea607cf0a46..c15df86c56ae 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -384,6 +384,7 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro IsMeta: b.IsMeta, Metadata: &gallery.BackendMetadata{ Name: b.Name, + Alias: b.Alias, InstalledAt: b.InstalledAt, GalleryURL: b.GalleryURL, Version: b.Version, @@ -460,6 +461,9 @@ func (d *DistributedBackendManager) clearSatisfiedInstallRows(ctx context.Contex // behavior so the periodic reconciler and /api/backends/install/:id keep // working unchanged. func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *galleryop.ManagementOp[gallery.GalleryBackend, any], progressCb galleryop.ProgressCallback) error { + if d.systemState.DisableBackendInstall { + return fmt.Errorf("%w: %s", gallery.ErrBackendInstallDisabled, op.GalleryElementName) + } galleriesJSON, _ := json.Marshal(op.Galleries) backendName := op.GalleryElementName @@ -542,6 +546,9 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall // new master + old worker still converges. Drop the fallback once every // worker in the fleet is on 2026-05-08 or newer. func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, op *galleryop.ManagementOp[gallery.GalleryBackend, any], progressCb galleryop.ProgressCallback) error { + if d.systemState.DisableBackendInstall { + return fmt.Errorf("%w: %s", gallery.ErrBackendInstallDisabled, op.GalleryElementName) + } opID := op.ID name := op.GalleryElementName galleriesJSON, _ := json.Marshal(d.backendGalleries) @@ -639,6 +646,14 @@ func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, op *gall // /api/backends/apply (which would otherwise silently land on every node). func (d *DistributedBackendManager) IsDistributed() bool { return true } +func (d *DistributedBackendManager) BackendAvailable(name string) (bool, error) { + backends, err := d.ListBackends() + if err != nil { + return false, err + } + return gallery.AvailableOnCluster(backends, name), nil +} + // CheckUpgrades checks for available backend upgrades across the cluster. // // The previous implementation delegated to d.local, which called diff --git a/core/services/nodes/managers_distributed_test.go b/core/services/nodes/managers_distributed_test.go index 490f3e098c75..856eae6f40df 100644 --- a/core/services/nodes/managers_distributed_test.go +++ b/core/services/nodes/managers_distributed_test.go @@ -285,6 +285,9 @@ func (stubLocalBackendManager) CheckUpgrades(_ context.Context) (map[string]gall return nil, nil } func (stubLocalBackendManager) IsDistributed() bool { return false } +func (stubLocalBackendManager) BackendAvailable(_ string) (bool, error) { + return false, nil +} var _ = Describe("DistributedBackendManager", func() { var ( diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 365d9f5b40a3..65d3235adafe 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -36,6 +36,7 @@ type Config struct { Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of model galleries (used to resolve --prefetch-models on boot)" group:"server" default:"${galleries}"` ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models" group:"server"` RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` + DisableBackendInstall bool `env:"LOCALAI_DISABLE_BACKEND_INSTALL,DISABLE_BACKEND_INSTALL" help:"If true, this worker refuses to install or upgrade backends, including installs requested by the frontend when a model is loaded; only backends already present or provided via LOCALAI_BACKENDS_SYSTEM_PATH are used." group:"hardening" default:"false"` // PrefetchModels lets a worker download gallery model artifacts (GGUFs, etc.) // from its own outbound internet at boot, instead of waiting for the master to diff --git a/core/services/worker/install_disabled_test.go b/core/services/worker/install_disabled_test.go new file mode 100644 index 000000000000..380de660c24e --- /dev/null +++ b/core/services/worker/install_disabled_test.go @@ -0,0 +1,52 @@ +package worker + +import ( + "errors" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Worker with backend installation disabled", func() { + var sup *backendSupervisor + var state *system.SystemState + + BeforeEach(func() { + root := GinkgoT().TempDir() + var err error + state, err = system.GetSystemState( + system.WithBackendPath(filepath.Join(root, "backends")), + system.WithBackendSystemPath(filepath.Join(root, "system-backends")), + system.WithModelPath(filepath.Join(root, "models")), + system.WithDisableBackendInstall(true), + ) + Expect(err).NotTo(HaveOccurred()) + for _, dir := range []string{state.Backend.BackendsPath, state.Backend.BackendsSystemPath, state.Model.ModelsPath} { + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + } + sup = &backendSupervisor{ + cfg: &Config{DisableBackendInstall: true}, + ml: model.NewModelLoader(state), + systemState: state, + processes: map[string]*backendProcess{}, + } + }) + + It("refuses a frontend install request for a backend it does not have", func() { + _, err := sup.installBackend(workerctl.BackendInstallRequest{Backend: "vllm", ModelID: "m"}, false, nil) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + _, statErr := os.Stat(filepath.Join(state.Backend.BackendsPath, "vllm")) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + }) + + It("refuses an upgrade request", func() { + _, err := sup.upgradeBackend(workerctl.BackendUpgradeRequest{Backend: "vllm"}, nil) + Expect(errors.Is(err, gallery.ErrBackendInstallDisabled)).To(BeTrue(), err.Error()) + }) +}) diff --git a/core/services/worker/lifecycle.go b/core/services/worker/lifecycle.go index 9e626dc90458..beec763d74b1 100644 --- a/core/services/worker/lifecycle.go +++ b/core/services/worker/lifecycle.go @@ -372,6 +372,7 @@ func (s *backendSupervisor) backendList(_ context.Context, _ workerctl.BackendLi IsMeta: b.IsMeta, } if b.Metadata != nil { + info.Alias = b.Metadata.Alias info.InstalledAt = b.Metadata.InstalledAt info.GalleryURL = b.Metadata.GalleryURL info.Version = b.Metadata.Version diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 4e7ba4e5c8ca..575889b3eba9 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -303,5 +303,6 @@ func (cfg *Config) systemState() (*system.SystemState, error) { system.WithBackendSystemPath(cfg.BackendsSystemPath), system.WithRequireBackendIntegrity(cfg.RequireBackendIntegrity), system.WithPreferDevelopmentBackends(cfg.PreferDevelopmentBackends), + system.WithDisableBackendInstall(cfg.DisableBackendInstall), ) } diff --git a/core/services/workerctl/backend.go b/core/services/workerctl/backend.go index 6321b12ae9a3..46214c1e6478 100644 --- a/core/services/workerctl/backend.go +++ b/core/services/workerctl/backend.go @@ -110,9 +110,15 @@ type BackendListReply struct { // NodeBackendInfo describes a backend installed on a worker node. type NodeBackendInfo struct { - Name string `json:"name"` - IsSystem bool `json:"is_system"` - IsMeta bool `json:"is_meta"` + Name string `json:"name"` + IsSystem bool `json:"is_system"` + IsMeta bool `json:"is_meta"` + // Alias is the alternative name a model config may use for this + // backend. The worker resolves it on its own disk when it loads the + // model; the frontend needs it only to judge whether such a model can + // run anywhere in the cluster. It rides on the concrete's own row, so + // upgrade detection still sees one row per installed backend. + Alias string `json:"alias,omitempty"` InstalledAt string `json:"installed_at,omitempty"` GalleryURL string `json:"gallery_url,omitempty"` // Version, URI and Digest enable cluster-wide upgrade detection: diff --git a/core/services/workerctl/wire_test.go b/core/services/workerctl/wire_test.go index 319525686771..bfbfb672ee35 100644 --- a/core/services/workerctl/wire_test.go +++ b/core/services/workerctl/wire_test.go @@ -45,17 +45,17 @@ var _ = Describe("Control payload wire format", func() { Entry("BackendListReply, every field set", workerctl.BackendListReply{ Backends: []workerctl.NodeBackendInfo{{ - Name: "n", IsSystem: true, IsMeta: true, InstalledAt: "t", GalleryURL: "gu", + Name: "n", IsSystem: true, IsMeta: true, Alias: "a", InstalledAt: "t", GalleryURL: "gu", Version: "v", URI: "u", Digest: "d", }}, Error: "e", - }, `{"backends":[{"name":"n","is_system":true,"is_meta":true,"installed_at":"t","gallery_url":"gu","version":"v","uri":"u","digest":"d"}],"error":"e"}`), + }, `{"backends":[{"name":"n","is_system":true,"is_meta":true,"alias":"a","installed_at":"t","gallery_url":"gu","version":"v","uri":"u","digest":"d"}],"error":"e"}`), Entry("BackendListReply, zero value", workerctl.BackendListReply{}, `{"backends":null}`), Entry("NodeBackendInfo, every field set", workerctl.NodeBackendInfo{ - Name: "n", IsSystem: true, IsMeta: true, InstalledAt: "t", GalleryURL: "gu", + Name: "n", IsSystem: true, IsMeta: true, Alias: "a", InstalledAt: "t", GalleryURL: "gu", Version: "v", URI: "u", Digest: "d", - }, `{"name":"n","is_system":true,"is_meta":true,"installed_at":"t","gallery_url":"gu","version":"v","uri":"u","digest":"d"}`), + }, `{"name":"n","is_system":true,"is_meta":true,"alias":"a","installed_at":"t","gallery_url":"gu","version":"v","uri":"u","digest":"d"}`), Entry("NodeBackendInfo, zero value", workerctl.NodeBackendInfo{}, `{"name":"","is_system":false,"is_meta":false}`), Entry("BackendStopRequest, every field set", workerctl.BackendStopRequest{Backend: "b", Force: true}, diff --git a/core/startup/model_preload.go b/core/startup/model_preload.go index 8dd1678283d9..fd4d76a8cc4c 100644 --- a/core/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -35,7 +35,10 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal var err error installOptions := slices.Clone(extraOptions) if galleryService != nil { - installOptions = append(installOptions, gallery.WithArtifactMaterializer(galleryService.ModelArtifactMaterializer())) + installOptions = append(installOptions, + gallery.WithArtifactMaterializer(galleryService.ModelArtifactMaterializer()), + gallery.WithBackendAvailability(galleryService.BackendManager().BackendAvailable), + ) } for _, url := range models { // Check if it's a model gallery, or print a warning diff --git a/core/startup/model_preload_test.go b/core/startup/model_preload_test.go index 525f183cfa88..a5a45a17aabb 100644 --- a/core/startup/model_preload_test.go +++ b/core/startup/model_preload_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/services/galleryop" . "github.com/mudler/LocalAI/core/startup" "github.com/mudler/LocalAI/pkg/model" @@ -14,6 +15,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" ) var _ = Describe("Preload test", func() { @@ -81,3 +83,83 @@ var _ = Describe("Preload test", func() { }) }) }) + +// clusterBackendManager answers availability from a fixed set, as the +// NATS-backed manager would from the workers' listings. +type clusterBackendManager struct{ available map[string]bool } + +func (clusterBackendManager) InstallBackend(context.Context, *galleryop.ManagementOp[gallery.GalleryBackend, any], galleryop.ProgressCallback) error { + return nil +} +func (clusterBackendManager) DeleteBackend(string) error { return nil } +func (clusterBackendManager) ListBackends() (gallery.SystemBackends, error) { + return gallery.SystemBackends{}, nil +} +func (clusterBackendManager) UpgradeBackend(context.Context, *galleryop.ManagementOp[gallery.GalleryBackend, any], galleryop.ProgressCallback) error { + return nil +} +func (clusterBackendManager) CheckUpgrades(context.Context) (map[string]gallery.UpgradeInfo, error) { + return nil, nil +} +func (clusterBackendManager) IsDistributed() bool { return true } +func (m clusterBackendManager) BackendAvailable(name string) (bool, error) { + return m.available[name], nil +} + +var _ = Describe("Preload with backend installation disabled", func() { + var ( + tmpdir string + systemState *system.SystemState + ml *model.ModelLoader + galleries []config.Gallery + ) + + BeforeEach(func() { + var err error + tmpdir, err = os.MkdirTemp("", "preload-install-disabled-*") + Expect(err).ToNot(HaveOccurred()) + systemState, err = system.GetSystemState( + system.WithModelPath(filepath.Join(tmpdir, "models")), + system.WithBackendPath(filepath.Join(tmpdir, "backends")), + system.WithDisableBackendInstall(true), + ) + Expect(err).ToNot(HaveOccurred()) + Expect(os.MkdirAll(systemState.Model.ModelsPath, 0o750)).To(Succeed()) + Expect(os.MkdirAll(systemState.Backend.BackendsPath, 0o750)).To(Succeed()) + ml = model.NewModelLoader(systemState) + + // The weights already sit in the models path, so the install touches + // nothing but the config file and the admission check. + Expect(os.WriteFile(filepath.Join(systemState.Model.ModelsPath, "weights.gguf"), []byte("weights"), 0o600)).To(Succeed()) + entries := []gallery.GalleryModel{{ + Metadata: gallery.Metadata{Name: "needs-vllm", Backend: "vllm"}, + ConfigFile: map[string]any{"backend": "vllm", "parameters": map[string]any{"model": "weights.gguf"}}, + }} + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + galleryPath := filepath.Join(systemState.Model.ModelsPath, "preload.yaml") + Expect(os.WriteFile(galleryPath, out, 0o600)).To(Succeed()) + galleries = []config.Gallery{{Name: "preload", URL: "file://" + galleryPath}} + }) + + AfterEach(func() { + os.RemoveAll(tmpdir) + }) + + preload := func(workersHave map[string]bool) error { + galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{SystemState: systemState, DisableBackendInstall: true}, ml) + galleryService.SetBackendManager(clusterBackendManager{available: workersHave}) + return InstallModels(context.Background(), galleryService, galleries, nil, systemState, ml, false, false, false, nil, "needs-vllm") + } + + It("installs a model whose backend a worker has, with no local backends", func() { + Expect(preload(map[string]bool{"vllm": true})).To(Succeed()) + Expect(filepath.Join(systemState.Model.ModelsPath, "needs-vllm.yaml")).To(BeAnExistingFile()) + }) + + It("refuses a model whose backend no worker has", func() { + err := preload(map[string]bool{}) + Expect(err).To(MatchError(gallery.ErrBackendInstallDisabled)) + Expect(filepath.Join(systemState.Model.ModelsPath, "needs-vllm.yaml")).NotTo(BeAnExistingFile()) + }) +}) diff --git a/docs/content/features/api-discovery.md b/docs/content/features/api-discovery.md index 555cd1ec49d2..1f6161b82b5b 100644 --- a/docs/content/features/api-discovery.md +++ b/docs/content/features/api-discovery.md @@ -63,6 +63,7 @@ Returns the instance version, all available endpoint URLs (flat and categorized) "config_patch": true, "vram_estimate": true, "model_storage": true, + "backend_install": true, "mcp": true, "agents": false, "p2p": false @@ -70,7 +71,7 @@ Returns the instance version, all available endpoint URLs (flat and categorized) } ``` -The `capabilities` object reflects the current runtime configuration - for example, `mcp` is only `true` if MCP is enabled, and `agents` is `true` only if the agent pool is running. +The `capabilities` object reflects the current runtime configuration - for example, `mcp` is only `true` if MCP is enabled, `agents` is `true` only if the agent pool is running, and `backend_install` is `false` when the server runs with `--disable-backend-install` (backend install and upgrade endpoints then answer 403). ## Instructions API diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 8817fb8a718a..3499b71a795d 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -209,6 +209,55 @@ the repository, so upgrade every node, workers included, before you rely on it: } ``` +### Disabling backend installation + +Some deployments must not acquire executable code at runtime: production +systems that run only the backend set they were tested with, locked-down or +air-gapped hosts, images that are built once and then promoted unchanged, and +hosts where backends come from a package manager (placed under +`--backends-system-path`, `LOCALAI_BACKENDS_SYSTEM_PATH`). For all of these, +`--disable-backend-install` (`LOCALAI_DISABLE_BACKEND_INSTALL=1`) tells +LocalAI never to install or upgrade a backend itself: + +```bash +export LOCALAI_DISABLE_BACKEND_INSTALL=1 +local-ai run +``` + +What is refused: + +- installs from the backend gallery, from OCI images, URLs or local paths — + from the Backends page, `POST /backends/apply`, the `/api/backends/...` + install routes, the LocalAI Assistant's `install_backend` tool and the + `LOCALAI_EXTERNAL_BACKENDS` boot-time list; +- upgrades — manual, "update all", the Assistant's `upgrade_backend` tool and + the background auto-upgrader, which is not started at all (upgrade checks + stop too, so no registry is polled for versions nobody can apply); +- installing a model whose backend is not present: refused before anything + is downloaded, naming the backend, on every install path (gallery, import, + preload, the Assistant's tools). + +What still works: backends already installed, backends provided under the +system path, listing and browsing the backend galleries, and deleting a +backend. In distributed mode a model counts as installable when a healthy +worker has its backend, under its own name or an alias — availability is +judged where inference runs, not on the frontend, which keeps no backends of +its own. + +How clients can tell: install and upgrade endpoints answer `403` with +`"code": "backend_install_disabled"`; installing a gallery model whose +backend is not present answers `409` with `"code": "backend_not_installed"` +and the backend's name. `GET /api/features` and the capabilities object in +`GET /.well-known/localai.json` report `"backend_install": false`, and the +Backends page shows a notice and offers no install or upgrade actions. + +The switch is read at start-up only; it is not a runtime setting. In +[distributed mode]({{% relref "features/distributed-mode#disabling-backend-installation-on-workers" %}}) +each worker enforces it for its own backends, so set it on every frontend +and every worker. The standalone `local-ai worker ...` subcommands honour +the same variable: they start with the backend already present under their +backends or system path and exit with an error instead of installing one. + ## Pre-installing Backends You can pre-install backends when starting LocalAI using the `LOCALAI_EXTERNAL_BACKENDS` environment variable: diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 111287ce5551..40a1ecc3b4f8 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -386,6 +386,22 @@ local-ai worker \ **HTTP file transfer:** Each worker also runs a small HTTP server for file transfer (model files, configs). By default it listens on the gRPC base port - 1 (e.g., if gRPC base is 50051, HTTP is on 50050). gRPC ports grow upward from the base port as additional models are loaded. Set `--advertise-http-addr` if the auto-detected address is not routable from the frontend. {{% /notice %}} +### Disabling backend installation on workers + +A worker installs a backend the first time the frontend schedules a model +that needs it. On workers that must run only pre-installed or system-path +backends, set `LOCALAI_DISABLE_BACKEND_INSTALL=1` on the worker: it then +refuses frontend install and upgrade requests for backends it does not have, +and a model that needs a missing backend fails to load on that node with a +clear error instead of pulling an image. Set the same variable on the +frontend so its own install and upgrade endpoints refuse as well — each +process enforces the switch for the backends it owns. Model installs on the +frontend are judged against the workers' backends, so a model whose backend +is installed on any healthy worker, under its own name or an alias, is still +accepted; a worker marked unhealthy does not count until its heartbeat +returns. See +[Disabling backend installation]({{% relref "features/backends#disabling-backend-installation" %}}). + ### Ephemeral request-input storage Workers reserve local capacity before accepting per-request audio, image, and other ephemeral inputs. The limit covers both direct HTTP staging and the worker's S3 download cache. A request is rejected before inference when accepting its input would exceed the byte limit or the configured free-space headroom. One request-scoped cleanup operation releases all exact input keys and their reservations after inference, while a one-hour recovery sweep removes abandoned files after crashes. The sweep runs at startup and every 15 minutes, preserves active requests, and considers the newest file in each request directory. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index 5d2cc669b658..0b2c2cd4c630 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -142,6 +142,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel | Parameter | Default | Description | Environment Variable | |-----------|---------|-------------|----------------------| | `--disable-predownload-scan` | `false` | If true, disables the best-effort security scanner before downloading any files | `$LOCALAI_DISABLE_PREDOWNLOAD_SCAN` | +| `--disable-backend-install` | `false` | If true, refuse to install or upgrade backends from galleries, OCI images, URLs or paths, including automatic installs, so the running backend set stays exactly what was deployed and tested; only backends already present or provided via `--backends-system-path` are used | `$LOCALAI_DISABLE_BACKEND_INSTALL`, `$DISABLE_BACKEND_INSTALL` | | `--opaque-errors` | `false` | If true, all error responses are replaced with blank 500 errors. This is intended only for hardening against information leaks and is normally not recommended | `$LOCALAI_OPAQUE_ERRORS` | | `--use-subtle-key-comparison` | `false` | If true, API Key validation comparisons will be performed using constant-time comparisons rather than simple equality. This trades off performance on each request for resilience against timing attacks | `$LOCALAI_SUBTLE_KEY_COMPARISON` | | `--disable-api-key-requirement-for-http-get` | `false` | If true, a valid API key is not required to issue GET requests to portions of the web UI. This should only be enabled in secure testing environments | `$LOCALAI_DISABLE_API_KEY_REQUIREMENT_FOR_HTTP_GET` | diff --git a/pkg/mcp/localaitools/inproc/client.go b/pkg/mcp/localaitools/inproc/client.go index b1900656bb43..297526c14de7 100644 --- a/pkg/mcp/localaitools/inproc/client.go +++ b/pkg/mcp/localaitools/inproc/client.go @@ -459,6 +459,9 @@ func (c *Client) InstallBackend(ctx context.Context, req localaitools.InstallBac if req.BackendName == "" { return "", errors.New("backend_name is required") } + if c.AppConfig.DisableBackendInstall { + return "", fmt.Errorf("%w: %s", gallery.ErrBackendInstallDisabled, req.BackendName) + } id, err := uuid.NewUUID() if err != nil { return "", fmt.Errorf("generate job id: %w", err) @@ -482,6 +485,9 @@ func (c *Client) UpgradeBackend(ctx context.Context, name string) (string, error if name == "" { return "", errors.New("name is required") } + if c.AppConfig.DisableBackendInstall { + return "", fmt.Errorf("%w: %s", gallery.ErrBackendInstallDisabled, name) + } id, err := uuid.NewUUID() if err != nil { return "", fmt.Errorf("generate job id: %w", err) diff --git a/pkg/mcp/localaitools/prompts/20_tools.md b/pkg/mcp/localaitools/prompts/20_tools.md index 591c49f51a40..c38adf302145 100644 --- a/pkg/mcp/localaitools/prompts/20_tools.md +++ b/pkg/mcp/localaitools/prompts/20_tools.md @@ -31,8 +31,8 @@ The MCP `tools/list` endpoint also exposes the full input schema for each of the - `install_model` — Install a model from a gallery. Returns a job id; poll with `get_job_status`. - `import_model_uri` — Install a model from an arbitrary URI (HuggingFace, OCI, http(s), file://). May return `ambiguous_backend` when several backends apply; call again with `backend_preference` to disambiguate. - `delete_model` — Delete an installed model. -- `install_backend` — Install a backend. -- `upgrade_backend` — Upgrade an installed backend by name. +- `install_backend` — Install a backend. Fails with "backend installation is disabled" on servers that manage backends outside LocalAI; report that rather than retrying. +- `upgrade_backend` — Upgrade an installed backend by name. Same refusal as `install_backend` on such servers. - `edit_model_config` — Patch (deep-merge) JSON into an installed model's config. - `reload_models` — Reload all model configs from disk. - `cancel_model_load` — Cancel one distributed load by the exact job_id from load-status. `stopping` means pending, not proof the work stopped. diff --git a/pkg/mcp/localaitools/prompts/skills/upgrade_backend.md b/pkg/mcp/localaitools/prompts/skills/upgrade_backend.md index 1e34d0d2aba3..ae19d6496029 100644 --- a/pkg/mcp/localaitools/prompts/skills/upgrade_backend.md +++ b/pkg/mcp/localaitools/prompts/skills/upgrade_backend.md @@ -6,5 +6,9 @@ Use this when the user asks to upgrade, refresh, or update a backend. 2. If the user asked generically ("upgrade the backends"), call `list_known_backends` and compare versions/tags to identify upgrade candidates. Present them as a numbered list and ask which to upgrade. 3. Summarise: "I'll upgrade **``** — confirm?" and wait. 4. On confirmation, call `upgrade_backend` with the canonical name. + If the call fails with "backend installation is disabled", the server is + configured never to install or upgrade backends itself (they are managed + outside LocalAI); tell the user that and stop — do not retry or try + `install_backend` instead. 5. Poll `get_job_status` until done. Report progress and the final outcome. 6. After success, recommend the user reload any model that was using the upgraded backend (or call `reload_models` if the user agrees). diff --git a/pkg/system/state.go b/pkg/system/state.go index 7139560f226c..d8bd91316c6a 100644 --- a/pkg/system/state.go +++ b/pkg/system/state.go @@ -42,6 +42,12 @@ type SystemState struct { // fetch reads it from here because the listing code it runs under never // sees the application config. RequireBackendIntegrity bool + + // DisableBackendInstall refuses every backend install and upgrade — + // gallery, OCI/URL/path, automatic — bound to --disable-backend-install / + // LOCALAI_DISABLE_BACKEND_INSTALL. Backends then come only from what is + // already in BackendsPath or from BackendsSystemPath (packaged installs). + DisableBackendInstall bool } type SystemStateOptions func(*SystemState) @@ -94,6 +100,12 @@ func WithRequireBackendIntegrity(require bool) SystemStateOptions { } } +func WithDisableBackendInstall(disable bool) SystemStateOptions { + return func(s *SystemState) { + s.DisableBackendInstall = disable + } +} + func WithPreferDevelopmentBackends(prefer bool) SystemStateOptions { return func(s *SystemState) { s.PreferDevelopmentBackends = prefer diff --git a/swagger/docs.go b/swagger/docs.go index 007d8f75d6fa..f7b51806dd8f 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -2949,6 +2949,20 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/schema.BackendResponse" } + }, + "403": { + "description": "Backend installation is disabled on this server (code backend_install_disabled)", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "409": { + "description": "Hardware-specific build in distributed mode (code concrete_backend_requires_target)", + "schema": { + "type": "object", + "additionalProperties": true + } } } } @@ -3091,6 +3105,13 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/schema.BackendResponse" } + }, + "403": { + "description": "Backend installation is disabled on this server (code backend_install_disabled)", + "schema": { + "type": "object", + "additionalProperties": true + } } } } @@ -3204,6 +3225,13 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/schema.GalleryResponse" } + }, + "409": { + "description": "The model's backend is not installed and backend installation is disabled (code backend_not_installed)", + "schema": { + "type": "object", + "additionalProperties": true + } } } } diff --git a/swagger/swagger.json b/swagger/swagger.json index 443e0e76791c..b7a5c8823fe1 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -2946,6 +2946,20 @@ "schema": { "$ref": "#/definitions/schema.BackendResponse" } + }, + "403": { + "description": "Backend installation is disabled on this server (code backend_install_disabled)", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "409": { + "description": "Hardware-specific build in distributed mode (code concrete_backend_requires_target)", + "schema": { + "type": "object", + "additionalProperties": true + } } } } @@ -3088,6 +3102,13 @@ "schema": { "$ref": "#/definitions/schema.BackendResponse" } + }, + "403": { + "description": "Backend installation is disabled on this server (code backend_install_disabled)", + "schema": { + "type": "object", + "additionalProperties": true + } } } } @@ -3201,6 +3222,13 @@ "schema": { "$ref": "#/definitions/schema.GalleryResponse" } + }, + "409": { + "description": "The model's backend is not installed and backend installation is disabled (code backend_not_installed)", + "schema": { + "type": "object", + "additionalProperties": true + } } } } diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index d990a3e72199..3b88fd2230f0 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -5914,6 +5914,16 @@ paths: description: Response schema: $ref: '#/definitions/schema.BackendResponse' + "403": + description: Backend installation is disabled on this server (code backend_install_disabled) + schema: + additionalProperties: true + type: object + "409": + description: Hardware-specific build in distributed mode (code concrete_backend_requires_target) + schema: + additionalProperties: true + type: object summary: Install backends to LocalAI. tags: - backends @@ -6004,6 +6014,11 @@ paths: description: Response schema: $ref: '#/definitions/schema.BackendResponse' + "403": + description: Backend installation is disabled on this server (code backend_install_disabled) + schema: + additionalProperties: true + type: object summary: Upgrade a backend tags: - backends @@ -6077,6 +6092,12 @@ paths: description: Response schema: $ref: '#/definitions/schema.GalleryResponse' + "409": + description: The model's backend is not installed and backend installation + is disabled (code backend_not_installed) + schema: + additionalProperties: true + type: object summary: Install models to LocalAI. tags: - models