fix: gracefully handle missing secrets in policy monitor

changelog/fragments/1786053515-graceful-missing-secrets.yaml

@@ -0,0 +1,20 @@
+kind: bug-fix
+
+summary: Gracefully handle missing secrets in policy monitor instead of crash-looping
+
+description: |
+  Fleet Server no longer crash-loops when a secret referenced in an agent
+  policy's `secret_references` cannot be found in the Fleet secrets store.
+  Previously, any 404 from `GET /_fleet/secret/{id}` was treated as a fatal
+  error, causing the entire policy monitor to abort and fleet-server to
+  restart indefinitely — taking all agents offline.
+
+  Now, a missing secret is logged as a warning and skipped. The policy loads
+  without that secret value; affected integration credentials will not be
+  substituted (the raw `$co.elastic.secret{id}` placeholder remains), causing
+  only that integration to degrade, while fleet-server itself continues
+  serving all other agents normally.
+
+component: fleet-server
+
+pr: https://github.com/elastic/fleet-server/pull/7571

internal/pkg/bulk/engine.go

@@ -330,6 +330,10 @@ func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[strin
 	for _, id := range secretIds {
 		val, err := ReadSecret(ctx, esClient, id)
 		if err != nil {
+			if errors.Is(err, ErrSecretNotFound) {
+				zerolog.Ctx(ctx).Warn().Str("secret_id", id).Msg("secret not found; policy will load without it")
+				continue
+			}
 			return nil, err
 		}
 		result[id] = val

internal/pkg/bulk/secret.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -16,6 +17,9 @@ import (
 	"go.elastic.co/apm/v2"
 )
 
+// ErrSecretNotFound is returned when a secret document does not exist in the Fleet secrets store.
+var ErrSecretNotFound = errors.New("secret not found")
+
 type ExtendedClient struct {
 	*elasticsearch.Client
 	Custom *ExtendedAPI
@@ -40,6 +44,9 @@ func (c *ExtendedAPI) Read(ctx context.Context, secretID string) (*SecretRespons
 		return nil, err
 	}
 	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return nil, fmt.Errorf("%q: %w", secretID, ErrSecretNotFound)
+	}
 	if res.StatusCode >= 400 {
 		body, _ := io.ReadAll(res.Body)
 		return nil, fmt.Errorf("unexpected status %d from fleet secret read: %s", res.StatusCode, body)

internal/pkg/bulk/secret_test.go

@@ -0,0 +1,63 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package bulk
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func newExtendedAPIWithStatus(t *testing.T, status int, body string) *ExtendedAPI {
+	t.Helper()
+	cli, err := elasticsearch.NewClient(elasticsearch.Config{
+		Addresses: []string{"http://localhost:9200"},
+		Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+			h := make(http.Header)
+			h.Set("X-Elastic-Product", "Elasticsearch")
+			return &http.Response{
+				StatusCode: status,
+				Body:       io.NopCloser(strings.NewReader(body)),
+				Header:     h,
+			}, nil
+		}),
+	})
+	require.NoError(t, err)
+	return &ExtendedAPI{Client: cli}
+}
+
+func TestExtendedAPIRead_NotFound_ReturnsSentinel(t *testing.T) {
+	api := newExtendedAPIWithStatus(t, http.StatusNotFound, `{"error":{"reason":"No secret with id [abc]"}}`)
+	_, err := api.Read(t.Context(), "abc")
+	require.Error(t, err)
+	assert.True(t, errors.Is(err, ErrSecretNotFound), "expected ErrSecretNotFound, got: %v", err)
+}
+
+func TestExtendedAPIRead_ServerError_ReturnsGenericError(t *testing.T) {
+	api := newExtendedAPIWithStatus(t, http.StatusInternalServerError, `{"error":"internal"}`)
+	_, err := api.Read(t.Context(), "abc")
+	require.Error(t, err)
+	assert.False(t, errors.Is(err, ErrSecretNotFound))
+}
+
+func TestExtendedAPIRead_Success(t *testing.T) {
+	api := newExtendedAPIWithStatus(t, http.StatusOK, `{"value":"my-secret-value"}`)
+	resp, err := api.Read(t.Context(), "abc")
+	require.NoError(t, err)
+	require.NotNil(t, resp)
+	assert.Equal(t, "my-secret-value", resp.Value)
+}