From 7144cfa3b396c2e2fa47f07d2e5b93ed2ac258cf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 16:52:20 -0700 Subject: [PATCH 1/3] fix: gracefully handle missing secrets in policy monitor (#7571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: gracefully handle missing secrets in policy monitor A 404 from GET /_fleet/secret/{id} was treated as a fatal error, causing the policy monitor to abort and fleet-server to crash-loop — taking all agents offline when a single integration credential was missing from the Fleet secrets store. Add ErrSecretNotFound sentinel to ExtendedAPI.Read so callers can distinguish "this secret doesn't exist" from network/auth errors. Change ReadSecrets to log a warning and skip missing secrets rather than aborting, so the policy loads without the missing value and only the affected integration degrades. Closes #7536 Co-Authored-By: Claude Sonnet 4.6 * Update changelog/fragments/1786053515-graceful-missing-secrets.yaml Co-authored-by: Michel Losier --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Michel Losier (cherry picked from commit 5ece0ba51fbdc9b0c711e705fb88630af77b763d) # Conflicts: # internal/pkg/bulk/secret.go --- .../1786053515-graceful-missing-secrets.yaml | 20 ++++++ internal/pkg/bulk/engine.go | 4 ++ internal/pkg/bulk/secret.go | 19 ++++++ internal/pkg/bulk/secret_test.go | 63 +++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 changelog/fragments/1786053515-graceful-missing-secrets.yaml create mode 100644 internal/pkg/bulk/secret_test.go diff --git a/changelog/fragments/1786053515-graceful-missing-secrets.yaml b/changelog/fragments/1786053515-graceful-missing-secrets.yaml new file mode 100644 index 0000000000..6df62cd614 --- /dev/null +++ b/changelog/fragments/1786053515-graceful-missing-secrets.yaml @@ -0,0 +1,20 @@ +kind: bug-fix + +summary: Gracefully handle missing secrets in policy monitor instead of crash-looping + +description: | + Fleet Server no longer crash-loops when a secret referenced in an agent + policy's `secret_references` cannot be found in the Fleet secrets store. + Previously, any 404 from `GET /_fleet/secret/{id}` was treated as a fatal + error, causing the entire policy monitor to abort and fleet-server to + restart indefinitely — taking all agents offline. + + Now, a missing secret is logged as a warning and skipped. The policy loads + without that secret value; affected integration credentials will not be + substituted (the raw `$co.elastic.secret{id}` placeholder remains), causing + only that integration to degrade, while fleet-server itself continues + serving all other agents normally. + +component: fleet-server + +pr: https://github.com/elastic/fleet-server/pull/7571 diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go index b5009ceedd..2225506442 100644 --- a/internal/pkg/bulk/engine.go +++ b/internal/pkg/bulk/engine.go @@ -297,6 +297,10 @@ func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[strin for _, id := range secretIds { val, err := ReadSecret(ctx, esClient, id) if err != nil { + if errors.Is(err, ErrSecretNotFound) { + zerolog.Ctx(ctx).Warn().Str("secret_id", id).Msg("secret not found; policy will load without it") + continue + } return nil, err } result[id] = val diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index acebe92130..9dc5a383b0 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -7,12 +7,21 @@ package bulk import ( "context" "encoding/json" +<<<<<<< HEAD +======= + "errors" + "fmt" + "io" +>>>>>>> 5ece0ba (fix: gracefully handle missing secrets in policy monitor (#7571)) "net/http" "github.com/elastic/go-elasticsearch/v8" "go.elastic.co/apm/v2" ) +// ErrSecretNotFound is returned when a secret document does not exist in the Fleet secrets store. +var ErrSecretNotFound = errors.New("secret not found") + type ExtendedClient struct { *elasticsearch.Client Custom *ExtendedAPI @@ -37,6 +46,16 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return nil, err } defer res.Body.Close() +<<<<<<< HEAD +======= + if res.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("%q: %w", secretID, ErrSecretNotFound) + } + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body) + } +>>>>>>> 5ece0ba (fix: gracefully handle missing secrets in policy monitor (#7571)) var secretResp SecretResponse err = json.NewDecoder(res.Body).Decode(&secretResp) diff --git a/internal/pkg/bulk/secret_test.go b/internal/pkg/bulk/secret_test.go new file mode 100644 index 0000000000..ec20035984 --- /dev/null +++ b/internal/pkg/bulk/secret_test.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package bulk + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func newExtendedAPIWithStatus(t *testing.T, status int, body string) *ExtendedAPI { + t.Helper() + cli, err := elasticsearch.NewClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + h := make(http.Header) + h.Set("X-Elastic-Product", "Elasticsearch") + return &http.Response{ + StatusCode: status, + Body: io.NopCloser(strings.NewReader(body)), + Header: h, + }, nil + }), + }) + require.NoError(t, err) + return &ExtendedAPI{Client: cli} +} + +func TestExtendedAPIRead_NotFound_ReturnsSentinel(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusNotFound, `{"error":{"reason":"No secret with id [abc]"}}`) + _, err := api.Read(t.Context(), "abc") + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSecretNotFound), "expected ErrSecretNotFound, got: %v", err) +} + +func TestExtendedAPIRead_ServerError_ReturnsGenericError(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusInternalServerError, `{"error":"internal"}`) + _, err := api.Read(t.Context(), "abc") + require.Error(t, err) + assert.False(t, errors.Is(err, ErrSecretNotFound)) +} + +func TestExtendedAPIRead_Success(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusOK, `{"value":"my-secret-value"}`) + resp, err := api.Read(t.Context(), "abc") + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, "my-secret-value", resp.Value) +} From d3807c7ccde436866d1b6a1740db63fdda0b960a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 17:02:16 -0700 Subject: [PATCH 2/3] fix: resolve merge conflict in backport of #7571 Cherry-pick left unresolved conflict markers in secret.go for the imports ("errors", "fmt", "io") and the status-code check blocks. Take both incoming additions as intended by the original fix. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/bulk/secret.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index 9dc5a383b0..9a071cbe58 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -7,12 +7,9 @@ package bulk import ( "context" "encoding/json" -<<<<<<< HEAD -======= "errors" "fmt" "io" ->>>>>>> 5ece0ba (fix: gracefully handle missing secrets in policy monitor (#7571)) "net/http" "github.com/elastic/go-elasticsearch/v8" @@ -46,8 +43,6 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return nil, err } defer res.Body.Close() -<<<<<<< HEAD -======= if res.StatusCode == http.StatusNotFound { return nil, fmt.Errorf("%q: %w", secretID, ErrSecretNotFound) } @@ -55,7 +50,6 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons body, _ := io.ReadAll(res.Body) return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body) } ->>>>>>> 5ece0ba (fix: gracefully handle missing secrets in policy monitor (#7571)) var secretResp SecretResponse err = json.NewDecoder(res.Body).Decode(&secretResp) From 3c7adc08e36f40ee91dc631f85913bdb1bb4ce49 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 17:47:01 -0700 Subject: [PATCH 3/3] fix: drop >= 400 block from backport of #7571 The >= 400 check was introduced on main in #7416 but never backported to these branches. The cherry-pick dragged it in as a side-effect. Remove it so non-404 errors fall through as they did before, keeping the backport minimal and matching the intent of #7571. Co-Authored-By: Claude Sonnet 4.6 --- internal/pkg/bulk/secret.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index 9a071cbe58..58a096ae2d 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "github.com/elastic/go-elasticsearch/v8" @@ -46,10 +45,6 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons if res.StatusCode == http.StatusNotFound { return nil, fmt.Errorf("%q: %w", secretID, ErrSecretNotFound) } - if res.StatusCode >= 400 { - body, _ := io.ReadAll(res.Body) - return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body) - } var secretResp SecretResponse err = json.NewDecoder(res.Body).Decode(&secretResp)