diff --git a/changelog/fragments/1786053515-graceful-missing-secrets.yaml b/changelog/fragments/1786053515-graceful-missing-secrets.yaml new file mode 100644 index 0000000000..6df62cd614 --- /dev/null +++ b/changelog/fragments/1786053515-graceful-missing-secrets.yaml @@ -0,0 +1,20 @@ +kind: bug-fix + +summary: Gracefully handle missing secrets in policy monitor instead of crash-looping + +description: | + Fleet Server no longer crash-loops when a secret referenced in an agent + policy's `secret_references` cannot be found in the Fleet secrets store. + Previously, any 404 from `GET /_fleet/secret/{id}` was treated as a fatal + error, causing the entire policy monitor to abort and fleet-server to + restart indefinitely — taking all agents offline. + + Now, a missing secret is logged as a warning and skipped. The policy loads + without that secret value; affected integration credentials will not be + substituted (the raw `$co.elastic.secret{id}` placeholder remains), causing + only that integration to degrade, while fleet-server itself continues + serving all other agents normally. + +component: fleet-server + +pr: https://github.com/elastic/fleet-server/pull/7571 diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go index b5009ceedd..2225506442 100644 --- a/internal/pkg/bulk/engine.go +++ b/internal/pkg/bulk/engine.go @@ -297,6 +297,10 @@ func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[strin for _, id := range secretIds { val, err := ReadSecret(ctx, esClient, id) if err != nil { + if errors.Is(err, ErrSecretNotFound) { + zerolog.Ctx(ctx).Warn().Str("secret_id", id).Msg("secret not found; policy will load without it") + continue + } return nil, err } result[id] = val diff --git a/internal/pkg/bulk/secret.go b/internal/pkg/bulk/secret.go index acebe92130..58a096ae2d 100644 --- a/internal/pkg/bulk/secret.go +++ b/internal/pkg/bulk/secret.go @@ -7,12 +7,17 @@ package bulk import ( "context" "encoding/json" + "errors" + "fmt" "net/http" "github.com/elastic/go-elasticsearch/v8" "go.elastic.co/apm/v2" ) +// ErrSecretNotFound is returned when a secret document does not exist in the Fleet secrets store. +var ErrSecretNotFound = errors.New("secret not found") + type ExtendedClient struct { *elasticsearch.Client Custom *ExtendedAPI @@ -37,6 +42,9 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons return nil, err } defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("%q: %w", secretID, ErrSecretNotFound) + } var secretResp SecretResponse err = json.NewDecoder(res.Body).Decode(&secretResp) diff --git a/internal/pkg/bulk/secret_test.go b/internal/pkg/bulk/secret_test.go new file mode 100644 index 0000000000..ec20035984 --- /dev/null +++ b/internal/pkg/bulk/secret_test.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package bulk + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func newExtendedAPIWithStatus(t *testing.T, status int, body string) *ExtendedAPI { + t.Helper() + cli, err := elasticsearch.NewClient(elasticsearch.Config{ + Addresses: []string{"http://localhost:9200"}, + Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + h := make(http.Header) + h.Set("X-Elastic-Product", "Elasticsearch") + return &http.Response{ + StatusCode: status, + Body: io.NopCloser(strings.NewReader(body)), + Header: h, + }, nil + }), + }) + require.NoError(t, err) + return &ExtendedAPI{Client: cli} +} + +func TestExtendedAPIRead_NotFound_ReturnsSentinel(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusNotFound, `{"error":{"reason":"No secret with id [abc]"}}`) + _, err := api.Read(t.Context(), "abc") + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSecretNotFound), "expected ErrSecretNotFound, got: %v", err) +} + +func TestExtendedAPIRead_ServerError_ReturnsGenericError(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusInternalServerError, `{"error":"internal"}`) + _, err := api.Read(t.Context(), "abc") + require.Error(t, err) + assert.False(t, errors.Is(err, ErrSecretNotFound)) +} + +func TestExtendedAPIRead_Success(t *testing.T) { + api := newExtendedAPIWithStatus(t, http.StatusOK, `{"value":"my-secret-value"}`) + resp, err := api.Read(t.Context(), "abc") + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, "my-secret-value", resp.Value) +}