From 995cf6fcfb148e1d9852a0cdb5ce0b9f8d10ec18 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 14:58:56 -0700 Subject: [PATCH 1/2] fix: gracefully handle missing secrets in policy monitor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 404 from GET /_fleet/secret/{id} was treated as a fatal error, causing the policy monitor to abort and fleet-server to crash-loop — taking all agents offline when a single integration credential was missing from the Fleet secrets store. Add ErrSecretNotFound sentinel to ExtendedAPI.Read so callers can distinguish "this secret doesn't exist" from network/auth errors. Change ReadSecrets to log a warning and skip missing secrets rather than aborting, so the policy loads without the missing value and only the affected integration degrades. Closes #7536 Co-Authored-By: Claude Sonnet 4.6 --- .../1786053515-graceful-missing-secrets.yaml | 20 ++++++ internal/pkg/bulk/engine.go | 4 ++ internal/pkg/bulk/secret.go | 7 +++ internal/pkg/bulk/secret_test.go | 63 +++++++++++++++++++ 4 files changed, 94 insertions(+) create mode 100644 changelog/fragments/1786053515-graceful-missing-secrets.yaml create mode 100644 internal/pkg/bulk/secret_test.go diff --git a/changelog/fragments/1786053515-graceful-missing-secrets.yaml b/changelog/fragments/1786053515-graceful-missing-secrets.yaml new file mode 100644 index 0000000000..e49740eeaf --- /dev/null +++ b/changelog/fragments/1786053515-graceful-missing-secrets.yaml @@ -0,0 +1,20 @@ +kind: bug-fix + +summary: Gracefully handle missing secrets in policy monitor instead of crash-looping + +description: | + Fleet Server no longer crash-loops when a secret referenced in an agent + policy's `secret_references` cannot be found in the Fleet secrets store. + Previously, any 404 from `GET /_fleet/secret/{id}` was treated as a fatal + error, causing the entire policy monitor to abort and fleet-server to + restart indefinitely — taking all agents offline. + + Now, a missing secret is logged as a warning and skipped. The policy loads + without that secret value; affected integration credentials will not be + substituted (the raw `$co.elastic.secret{id}` placeholder remains), causing + only that integration to degrade, while fleet-server itself continues + serving all other agents normally. + +component: fleet-server + +pr: https://github.com/elastic/fleet-server/pull/7566 diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go index 776e16c75e..b5078f6985 100644 --- a/internal/pkg/bulk/engine.go +++ b/internal/pkg/bulk/engine.go @@ -330,6 +330,10 @@ func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[strin for _, id := range secretIds { val, err := ReadSecret(ctx, esClient, id) if err != nil { + if errors.Is(err, ErrSecretNotFound) { + zerolog.Ctx(ctx).Warn().Str("secret_id", id).Msg("secret not found; policy will load without it") + continue + } return nil, err } result[id] = val diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index 9448493149..1110912263 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -16,6 +17,9 @@ import ( "go.elastic.co/apm/v2" ) +// ErrSecretNotFound is returned when a secret document does not exist in the Fleet secrets store. +var ErrSecretNotFound = errors.New("secret not found") + type ExtendedClient struct { *elasticsearch.Client Custom *ExtendedAPI @@ -40,6 +44,9 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return nil, err } defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("%q: %w", secretID, ErrSecretNotFound) + } if res.StatusCode >= 400 { body, _ := io.ReadAll(res.Body) return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body) diff --git a/internal/pkg/bulk/secret_test.go b/internal/pkg/bulk/secret_test.go new file mode 100644 index 0000000000..ec20035984 --- /dev/null +++ b/internal/pkg/bulk/secret_test.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package bulk + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func newExtendedAPIWithStatus(t *testing.T, status int, body string) *ExtendedAPI { + t.Helper() + cli, err := elasticsearch.NewClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + h := make(http.Header) + h.Set("X-Elastic-Product", "Elasticsearch") + return &http.Response{ + StatusCode: status, + Body: io.NopCloser(strings.NewReader(body)), + Header: h, + }, nil + }), + }) + require.NoError(t, err) + return &ExtendedAPI{Client: cli} +} + +func TestExtendedAPIRead_NotFound_ReturnsSentinel(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusNotFound, `{"error":{"reason":"No secret with id [abc]"}}`) + _, err := api.Read(t.Context(), "abc") + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSecretNotFound), "expected ErrSecretNotFound, got: %v", err) +} + +func TestExtendedAPIRead_ServerError_ReturnsGenericError(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusInternalServerError, `{"error":"internal"}`) + _, err := api.Read(t.Context(), "abc") + require.Error(t, err) + assert.False(t, errors.Is(err, ErrSecretNotFound)) +} + +func TestExtendedAPIRead_Success(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusOK, `{"value":"my-secret-value"}`) + resp, err := api.Read(t.Context(), "abc") + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, "my-secret-value", resp.Value) +} From 362a8a6f8c1ad0f60f836fc78b1c8606cf047132 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 15:47:55 -0700 Subject: [PATCH 2/2] Update changelog/fragments/1786053515-graceful-missing-secrets.yaml Co-authored-by: Michel Losier --- changelog/fragments/1786053515-graceful-missing-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/fragments/1786053515-graceful-missing-secrets.yaml b/changelog/fragments/1786053515-graceful-missing-secrets.yaml index e49740eeaf..6df62cd614 100644 --- a/changelog/fragments/1786053515-graceful-missing-secrets.yaml +++ b/changelog/fragments/1786053515-graceful-missing-secrets.yaml @@ -17,4 +17,4 @@ description: | component: fleet-server -pr: https://github.com/elastic/fleet-server/pull/7566 +pr: https://github.com/elastic/fleet-server/pull/7571