From 4db661ca3787433011705a43a9dccd99c6945fe6 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 10 Nov 2026 20:43:19 -0500 Subject: [PATCH 1/4] feat(pam): session recording test for postgres and rdp --- e2e/packages/infisical/compose.go | 3 +- e2e/pam/pam_helpers.go | 177 ++++++++++++++++++++++++++++++ e2e/pam/postgres_test.go | 48 ++++++++ e2e/pam/rdp_test.go | 26 ++++- 4 files changed, 252 insertions(+), 2 deletions(-) diff --git a/e2e/packages/infisical/compose.go b/e2e/packages/infisical/compose.go index de8c38a9e..b905a4aa0 100644 --- a/e2e/packages/infisical/compose.go +++ b/e2e/packages/infisical/compose.go @@ -299,7 +299,8 @@ func WithLocalStackService(buckets ...string) StackOption { } s.Project.Services["localstack"] = types.ServiceConfig{ Image: "localstack/localstack:4.4@sha256:b52c16663c70b7234f217cb993a339b46686e30a1a5d9279cb5feeb2202f837c", - Ports: []types.ServicePortConfig{{Published: "", Target: 4566}}, + // Published on 4566 so host processes, like the gateway uploading recordings, can use URLs presigned for .s3.localhost.localstack.cloud:4566 + Ports: []types.ServicePortConfig{{Published: "4566", Target: 4566}}, Environment: types.NewMappingWithEquals([]string{ "SERVICES=s3,sts", }), diff --git a/e2e/pam/pam_helpers.go b/e2e/pam/pam_helpers.go index 079df39c3..098ebc33b 100644 --- a/e2e/pam/pam_helpers.go +++ b/e2e/pam/pam_helpers.go @@ -3,6 +3,9 @@ package pam import ( "bytes" "context" + "crypto/aes" + "crypto/cipher" + "crypto/sha256" "encoding/base64" "encoding/json" "fmt" @@ -12,6 +15,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "testing" "time" @@ -49,6 +53,7 @@ type PAMTestInfra struct { GatewayCmd *helpers.Command ProvisionResult *client.ProvisionResult SharedHomeDir string + RecordingPath string } type SetupPAMOption func(svc *helpers.InfisicalService, extraEnv *[]string) @@ -158,6 +163,7 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P GatewayCmd: gatewayCmd, ProvisionResult: svc.ProvisionResult(), SharedHomeDir: sharedHomeDir, + RecordingPath: sessionRecordingPath, } } @@ -428,3 +434,174 @@ func LoginUser(t *testing.T, ctx context.Context, infra *PAMTestInfra) { }) require.Equal(t, helpers.WaitSuccess, result, "Login should succeed") } + +func getRaw(ctx context.Context, url, token string) (int, []byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, nil, err + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + return resp.StatusCode, body, err +} + +func getOK(t *testing.T, ctx context.Context, url, token string) []byte { + status, body, err := getRaw(ctx, url, token) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status, "GET %s: %s", url, string(body)) + return body +} + +// EndPamAccess interrupts `pam access` like Ctrl+C, which ends the session through the gateway; Stop kills it and leaves the session open. +func EndPamAccess(t *testing.T, infra *PAMTestInfra, pamCmd *helpers.Command) { + defer func() { + if t.Failed() { + infra.DumpOutput(pamCmd) + } + }() + require.NoError(t, pamCmd.Cmd().Process.Signal(os.Interrupt)) + result := helpers.WaitFor(t, helpers.WaitForOptions{ + Timeout: 60 * time.Second, + Interval: time.Second, + Condition: func() helpers.ConditionResult { + if pamCmd.IsRunning() { + return helpers.ConditionWait + } + return helpers.ConditionSuccess + }, + }) + require.Equal(t, helpers.WaitSuccess, result, "pam access should exit after an interrupt") +} + +// FindPamSessionId returns the ID of the one session opened against the named account. +func FindPamSessionId(t *testing.T, ctx context.Context, infra *PAMTestInfra, accountName string) string { + var list struct { + Sessions []struct { + Id string `json:"id"` + AccountName string `json:"accountName"` + } `json:"sessions"` + } + body := getOK(t, ctx, fmt.Sprintf("%s/api/v1/pam/sessions?limit=100", infra.Infisical.ApiUrl(t)), infra.ProvisionResult.Token) + require.NoError(t, json.Unmarshal(body, &list)) + + var ids []string + for _, s := range list.Sessions { + if s.AccountName == accountName { + ids = append(ids, s.Id) + } + } + require.Len(t, ids, 1, "expected one session for account %s", accountName) + return ids[0] +} + +type RecordingChunk struct { + ChunkIndex int `json:"chunkIndex"` + StorageBackend string `json:"storageBackend"` + ExternalChunkObjectKey *string `json:"externalChunkObjectKey"` + CiphertextSha256 string `json:"ciphertextSha256"` + CiphertextBytes int `json:"ciphertextBytes"` + Iv string `json:"iv"` + PresignedGetUrl *string `json:"presignedGetUrl"` +} + +type SessionPlayback struct { + SessionComplete bool `json:"sessionComplete"` + SessionKey string `json:"sessionKey"` + ProjectId string `json:"projectId"` + StorageBackend string `json:"storageBackend"` + Chunks []RecordingChunk `json:"chunks"` +} + +// WaitForSessionRecording waits for the session to end with recording chunks stored, and returns the playback the session viewer reads. +func WaitForSessionRecording(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string) SessionPlayback { + defer func() { + if t.Failed() { + infra.GatewayCmd.DumpOutput() + } + }() + playbackUrl := fmt.Sprintf("%s/api/v1/pam/sessions/%s/playback", infra.Infisical.ApiUrl(t), sessionId) + var playback SessionPlayback + var failure string + result := helpers.WaitFor(t, helpers.WaitForOptions{ + Timeout: 90 * time.Second, + Interval: 2 * time.Second, + Condition: func() helpers.ConditionResult { + status, body, err := getRaw(ctx, playbackUrl, infra.ProvisionResult.Token) + if err != nil || status != http.StatusOK { + failure = fmt.Sprintf("GET %s: status %d, error %v: %s", playbackUrl, status, err, string(body)) + return helpers.ConditionBreakEarly + } + + playback = SessionPlayback{} + if err := json.Unmarshal(body, &playback); err != nil { + failure = fmt.Sprintf("GET %s: %v", playbackUrl, err) + return helpers.ConditionBreakEarly + } + slog.Info("Waiting for session recording", "sessionId", sessionId, + "sessionComplete", playback.SessionComplete, "chunks", len(playback.Chunks)) + if playback.SessionComplete && len(playback.Chunks) > 0 { + return helpers.ConditionSuccess + } + return helpers.ConditionWait + }, + }) + require.Equal(t, helpers.WaitSuccess, result, failure) + return playback +} + +// ReadSessionRecording downloads, verifies and decrypts each chunk as the session viewer does, returning the recorded events in order. +func ReadSessionRecording[T any](t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string, playback SessionPlayback) []T { + sessionKey, err := base64.StdEncoding.DecodeString(playback.SessionKey) + require.NoError(t, err) + block, err := aes.NewCipher(sessionKey) + require.NoError(t, err) + gcm, err := cipher.NewGCM(block) + require.NoError(t, err) + + chunks := slices.Clone(playback.Chunks) + slices.SortFunc(chunks, func(a, b RecordingChunk) int { return a.ChunkIndex - b.ChunkIndex }) + + var events []T + for i, chunk := range chunks { + require.Equal(t, i, chunk.ChunkIndex, "recording chunks should be contiguous") + + var ciphertext []byte + switch chunk.StorageBackend { + case "postgres": + ciphertext = getOK(t, ctx, fmt.Sprintf("%s/api/v1/pam/sessions/%s/chunks/%d/ciphertext", + infra.Infisical.ApiUrl(t), sessionId, chunk.ChunkIndex), infra.ProvisionResult.Token) + case "aws-s3": + require.NotNil(t, chunk.ExternalChunkObjectKey, "chunk %d has no object key", chunk.ChunkIndex) + require.Equal(t, fmt.Sprintf("%s/%s/chunk-%06d.json.enc", playback.ProjectId, sessionId, chunk.ChunkIndex), + *chunk.ExternalChunkObjectKey) + require.NotNil(t, chunk.PresignedGetUrl, "chunk %d has no download URL", chunk.ChunkIndex) + ciphertext = getOK(t, ctx, *chunk.PresignedGetUrl, "") + default: + require.Failf(t, "unexpected storage backend", "chunk %d is stored in %q", chunk.ChunkIndex, chunk.StorageBackend) + } + + require.Equal(t, chunk.CiphertextBytes, len(ciphertext), "chunk %d size", chunk.ChunkIndex) + digest := sha256.Sum256(ciphertext) + require.Equal(t, chunk.CiphertextSha256, base64.StdEncoding.EncodeToString(digest[:]), "chunk %d digest", chunk.ChunkIndex) + + iv, err := base64.StdEncoding.DecodeString(chunk.Iv) + require.NoError(t, err) + aad := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%d|%s|v1", playback.ProjectId, sessionId, chunk.ChunkIndex, chunk.StorageBackend))) + plaintext, err := gcm.Open(nil, iv, ciphertext, aad[:]) + require.NoError(t, err, "chunk %d should decrypt with the session key", chunk.ChunkIndex) + + var chunkEvents []T + require.NoError(t, json.Unmarshal(plaintext, &chunkEvents), "chunk %d should hold a JSON array of events", chunk.ChunkIndex) + events = append(events, chunkEvents...) + } + return events +} diff --git a/e2e/pam/postgres_test.go b/e2e/pam/postgres_test.go index 0fdd3a9ac..b7c62cc2d 100644 --- a/e2e/pam/postgres_test.go +++ b/e2e/pam/postgres_test.go @@ -5,6 +5,8 @@ import ( "fmt" "log/slog" "net/http" + "os" + "path/filepath" "strings" "testing" "time" @@ -167,4 +169,50 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, proxyResult) slog.Info("SELECT 1 through PAM proxy succeeded") + + type recordedQuery struct { + Input string `json:"input"` + Output string `json:"output"` + } + expected := []recordedQuery{{Input: "SELECT 1", Output: "SELECT 1"}} + for _, query := range []recordedQuery{ + {Input: "CREATE TABLE pam_recording (note text)", Output: "CREATE TABLE"}, + {Input: "INSERT INTO pam_recording VALUES ('recorded through pam')", Output: "INSERT 0 1"}, + {Input: "SELECT note FROM pam_recording", Output: "SELECT 1"}, + } { + _, err := proxyConn.Exec(ctx, query.Input) + require.NoError(t, err) + expected = append(expected, query) + } + + sessionId := FindPamSessionId(t, ctx, infra, accountName) + + // The gateway keeps the encrypted recording on disk until the session ends and it is uploaded. + recordingGlob := filepath.Join(infra.RecordingPath, fmt.Sprintf("pam_session_%s_postgres_expires_*.enc", sessionId)) + fileResult := helpers.WaitFor(t, helpers.WaitForOptions{ + Timeout: 30 * time.Second, + Interval: time.Second, + Condition: func() helpers.ConditionResult { + files, _ := filepath.Glob(recordingGlob) + if len(files) != 1 { + return helpers.ConditionWait + } + if info, statErr := os.Stat(files[0]); statErr != nil || info.Size() == 0 { + return helpers.ConditionWait + } + return helpers.ConditionSuccess + }, + }) + require.Equal(t, helpers.WaitSuccess, fileResult, "gateway should write the session recording under %s", infra.RecordingPath) + + require.NoError(t, proxyConn.Close(ctx)) + EndPamAccess(t, infra, &pamCmd) + + playback := WaitForSessionRecording(t, ctx, infra, sessionId) + require.Equal(t, "postgres", playback.StorageBackend) + recorded := ReadSessionRecording[recordedQuery](t, ctx, infra, sessionId, playback) + for _, query := range expected { + require.Contains(t, recorded, query) + } + slog.Info("Postgres session recording verified", "sessionId", sessionId, "entries", len(recorded)) } diff --git a/e2e/pam/rdp_test.go b/e2e/pam/rdp_test.go index 1d64aaba2..3f4f9974d 100644 --- a/e2e/pam/rdp_test.go +++ b/e2e/pam/rdp_test.go @@ -3,6 +3,7 @@ package pam import ( "bytes" "context" + "encoding/json" "fmt" "log/slog" "net/http" @@ -205,11 +206,34 @@ func TestPAM_RDP(t *testing.T) { createRDPPamAccountAt(t, ctx, infra, folderId, templateId, accountName, resourceHost, rdpPort, rdpUser, rdpPassword) proxyPort := helpers.GetFreePort() - startRDPProxy(t, ctx, infra, folderName, accountName, "5m", proxyPort) + _, pamCmd := startRDPProxy(t, ctx, infra, folderName, accountName, "5m", proxyPort) err := connectFreeRDP(t, ctx, rdpBinary, "127.0.0.1", proxyPort, "testuser", "", 10*time.Second) require.NoError(t, err, "xfreerdp should connect through proxy") slog.Info("RDP connection test passed") + + sessionId := FindPamSessionId(t, ctx, infra, accountName) + EndPamAccess(t, infra, pamCmd) + playback := WaitForSessionRecording(t, ctx, infra, sessionId) + require.Equal(t, "aws-s3", playback.StorageBackend) + + type rdpEvent struct { + ChannelType string `json:"channelType"` + Data []byte `json:"data"` + } + frames := 0 + for _, event := range ReadSessionRecording[rdpEvent](t, ctx, infra, sessionId, playback) { + require.Equal(t, "rdp", event.ChannelType) + var envelope struct { + Type string `json:"type"` + } + require.NoError(t, json.Unmarshal(event.Data, &envelope)) + if envelope.Type == "target_frame" { + frames++ + } + } + require.NotZero(t, frames, "the recording should hold the frames the RDP target sent") + slog.Info("RDP session recording verified", "sessionId", sessionId, "frames", frames) }) t.Run("bad-credentials", func(t *testing.T) { From b32dff516f1f8faa5ce92785c955a1e8fd954df1 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 10 Nov 2026 21:14:40 -0500 Subject: [PATCH 2/4] addres bot comments --- e2e/pam/pam_helpers.go | 3 --- e2e/pam/postgres_test.go | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/e2e/pam/pam_helpers.go b/e2e/pam/pam_helpers.go index 098ebc33b..200f78592 100644 --- a/e2e/pam/pam_helpers.go +++ b/e2e/pam/pam_helpers.go @@ -482,7 +482,6 @@ func EndPamAccess(t *testing.T, infra *PAMTestInfra, pamCmd *helpers.Command) { require.Equal(t, helpers.WaitSuccess, result, "pam access should exit after an interrupt") } -// FindPamSessionId returns the ID of the one session opened against the named account. func FindPamSessionId(t *testing.T, ctx context.Context, infra *PAMTestInfra, accountName string) string { var list struct { Sessions []struct { @@ -521,7 +520,6 @@ type SessionPlayback struct { Chunks []RecordingChunk `json:"chunks"` } -// WaitForSessionRecording waits for the session to end with recording chunks stored, and returns the playback the session viewer reads. func WaitForSessionRecording(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string) SessionPlayback { defer func() { if t.Failed() { @@ -558,7 +556,6 @@ func WaitForSessionRecording(t *testing.T, ctx context.Context, infra *PAMTestIn return playback } -// ReadSessionRecording downloads, verifies and decrypts each chunk as the session viewer does, returning the recorded events in order. func ReadSessionRecording[T any](t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string, playback SessionPlayback) []T { sessionKey, err := base64.StdEncoding.DecodeString(playback.SessionKey) require.NoError(t, err) diff --git a/e2e/pam/postgres_test.go b/e2e/pam/postgres_test.go index b7c62cc2d..9f970ded2 100644 --- a/e2e/pam/postgres_test.go +++ b/e2e/pam/postgres_test.go @@ -211,8 +211,6 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { playback := WaitForSessionRecording(t, ctx, infra, sessionId) require.Equal(t, "postgres", playback.StorageBackend) recorded := ReadSessionRecording[recordedQuery](t, ctx, infra, sessionId, playback) - for _, query := range expected { - require.Contains(t, recorded, query) - } + require.Equal(t, expected, recorded) slog.Info("Postgres session recording verified", "sessionId", sessionId, "entries", len(recorded)) } From c68f001747c9d4ffd2235658b662ba77b74d3280 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 12 Nov 2026 15:37:04 -0500 Subject: [PATCH 3/4] feat(tests): pam recording survives gateway crash and rate-limited uploads --- e2e/pam/pam_helpers.go | 126 ++++++++++++++++++++++++-- e2e/pam/postgres_test.go | 189 +++++++++++++++++++++++++++++---------- 2 files changed, 261 insertions(+), 54 deletions(-) diff --git a/e2e/pam/pam_helpers.go b/e2e/pam/pam_helpers.go index 200f78592..8add0f928 100644 --- a/e2e/pam/pam_helpers.go +++ b/e2e/pam/pam_helpers.go @@ -11,11 +11,17 @@ import ( "fmt" "io" "log/slog" + "maps" "net" "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" "os" "path/filepath" + "regexp" "slices" + "sync/atomic" "testing" "time" @@ -51,36 +57,49 @@ type PAMTestInfra struct { GatewayId openapitypes.UUID RelayCmd *helpers.Command GatewayCmd *helpers.Command + GatewayAPI *GatewayAPIProxy ProvisionResult *client.ProvisionResult SharedHomeDir string RecordingPath string } -type SetupPAMOption func(svc *helpers.InfisicalService, extraEnv *[]string) +type setupPAMConfig struct { + backendEnv []string + gatewayAPIProxy bool +} + +type SetupPAMOption func(svc *helpers.InfisicalService, cfg *setupPAMConfig) const RecordingBucket = "e2e-recordings" func WithLocalStack() SetupPAMOption { - return func(svc *helpers.InfisicalService, extraEnv *[]string) { + return func(svc *helpers.InfisicalService, cfg *setupPAMConfig) { infisicalpkg.WithLocalStackService(RecordingBucket)(svc.Stack) - *extraEnv = append(*extraEnv, + cfg.backendEnv = append(cfg.backendEnv, "AWS_ENDPOINT_URL=http://localstack:4566", fmt.Sprintf("AWS_ENDPOINT_URL_S3=http://%s:4566", infisicalpkg.LocalStackS3Endpoint)) } } +// WithGatewayAPIProxy sends the gateway's API calls through infra.GatewayAPI. +func WithGatewayAPIProxy() SetupPAMOption { + return func(_ *helpers.InfisicalService, cfg *setupPAMConfig) { + cfg.gatewayAPIProxy = true + } +} + func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *PAMTestInfra { svc := helpers.NewInfisicalService() - var extraEnv []string + var cfg setupPAMConfig for _, opt := range opts { - opt(svc, &extraEnv) + opt(svc, &cfg) } env := append([]string{ "ALLOW_INTERNAL_IP_CONNECTIONS=true", "NODE_ENV=test", - }, extraEnv...) + }, cfg.backendEnv...) svc.WithBackendEnvironment(types.NewMappingWithEquals(env)).Up(t, ctx) @@ -118,16 +137,22 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P require.NoError(t, os.MkdirAll(sessionRecordingPath, 0755)) gatewayName := helpers.RandomSlug(2) gatewayId, enrollmentToken := CreateGatewayWithEnrollmentToken(t, ctx, svc, gatewayName) + gatewayApiUrl := svc.ApiUrl(t) + var gatewayAPI *GatewayAPIProxy + if cfg.gatewayAPIProxy { + gatewayAPI = startGatewayAPIProxy(t, gatewayApiUrl) + gatewayApiUrl = gatewayAPI.URL + } gatewayCmd := &helpers.Command{ Test: t, Args: []string{"gateway", "start", gatewayName, "--enroll-method", "token", "--token", enrollmentToken, - "--domain", svc.ApiUrl(t), + "--domain", gatewayApiUrl, fmt.Sprintf("--pam-session-recording-path=%s", sessionRecordingPath), }, Env: map[string]string{ - "INFISICAL_API_URL": svc.ApiUrl(t), + "INFISICAL_API_URL": gatewayApiUrl, }, } gatewayCmd.Start(ctx) @@ -161,12 +186,89 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P GatewayId: gatewayId, RelayCmd: relayCmd, GatewayCmd: gatewayCmd, + GatewayAPI: gatewayAPI, ProvisionResult: svc.ProvisionResult(), SharedHomeDir: sharedHomeDir, RecordingPath: sessionRecordingPath, } } +var chunkUploadPath = regexp.MustCompile(`^/api/v1/pam/sessions/[^/]+/chunks(/presigned-put)?$`) + +// GatewayAPIProxy forwards the gateway's API calls to the backend and, while throttled, answers its +// recording chunk uploads the way the backend's rate limiter does. +type GatewayAPIProxy struct { + URL string + throttled atomic.Bool + rejected atomic.Int32 +} + +func startGatewayAPIProxy(t *testing.T, apiUrl string) *GatewayAPIProxy { + target, err := url.Parse(apiUrl) + require.NoError(t, err) + forward := &httputil.ReverseProxy{Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(target) }} + + proxy := &GatewayAPIProxy{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !proxy.throttled.Load() || r.Method != http.MethodPost || !chunkUploadPath.MatchString(r.URL.Path) { + forward.ServeHTTP(w, r) + return + } + proxy.rejected.Add(1) + w.Header().Set("Content-Type", "application/json") + // The backend sends the seconds left in its one-minute window, so 60 is the longest it asks for. + w.Header().Set("Retry-After", "60") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, `{"statusCode":429,"message":"Rate limit exceeded. Please try again in 60 seconds","error":"RateLimitExceeded"}`) + })) + t.Cleanup(server.Close) + proxy.URL = server.URL + return proxy +} + +func (p *GatewayAPIProxy) ThrottleChunkUploads(throttled bool) { + p.throttled.Store(throttled) +} + +func (p *GatewayAPIProxy) RejectedChunkUploads() int { + return int(p.rejected.Load()) +} + +// CrashAndRestartGateway kills the gateway without letting it clean up, then starts it again the way a +// restarted host would, with its recordings and queued chunks still on disk. +func CrashAndRestartGateway(t *testing.T, ctx context.Context, infra *PAMTestInfra) { + crashed := infra.GatewayCmd + require.NoError(t, crashed.Cmd().Process.Kill()) + result := helpers.WaitFor(t, helpers.WaitForOptions{ + Timeout: 10 * time.Second, + Interval: 200 * time.Millisecond, + Condition: func() helpers.ConditionResult { + if crashed.IsRunning() { + return helpers.ConditionWait + } + return helpers.ConditionSuccess + }, + }) + require.Equal(t, helpers.WaitSuccess, result, "gateway should exit when killed") + + // Same HOME, because enrollment saved the gateway's access token there and the token can't enroll twice. + gatewayCmd := &helpers.Command{ + Test: t, + RunMethod: helpers.RunMethodSubprocess, + DisableTempHomeDir: true, + Args: crashed.Args, + Env: maps.Clone(crashed.Env), + } + gatewayCmd.Start(ctx) + t.Cleanup(gatewayCmd.Stop) + infra.GatewayCmd = gatewayCmd + result = helpers.WaitForStderr(t, helpers.WaitForStderrOptions{ + EnsureCmdRunning: gatewayCmd, + ExpectedString: "Gateway is reachable by Infisical", + }) + require.Equal(t, helpers.WaitSuccess, result, "gateway should come back after a restart") +} + // postJSON sends an authenticated JSON POST and decodes the response into out. Used for endpoints // missing from the generated client (see e2e/openapi-cfg.yaml). func postJSON(t *testing.T, ctx context.Context, url, token string, body, out interface{}) { @@ -521,6 +623,12 @@ type SessionPlayback struct { } func WaitForSessionRecording(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string) SessionPlayback { + return waitForPlayback(t, ctx, infra, sessionId, func(p SessionPlayback) bool { + return p.SessionComplete && len(p.Chunks) > 0 + }) +} + +func waitForPlayback(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string, ready func(SessionPlayback) bool) SessionPlayback { defer func() { if t.Failed() { infra.GatewayCmd.DumpOutput() @@ -546,7 +654,7 @@ func WaitForSessionRecording(t *testing.T, ctx context.Context, infra *PAMTestIn } slog.Info("Waiting for session recording", "sessionId", sessionId, "sessionComplete", playback.SessionComplete, "chunks", len(playback.Chunks)) - if playback.SessionComplete && len(playback.Chunks) > 0 { + if ready(playback) { return helpers.ConditionSuccess } return helpers.ConditionWait diff --git a/e2e/pam/postgres_test.go b/e2e/pam/postgres_test.go index 9f970ded2..fab8779eb 100644 --- a/e2e/pam/postgres_test.go +++ b/e2e/pam/postgres_test.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -18,19 +19,18 @@ import ( tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" ) -func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - infra := SetupPAMInfra(t, ctx) +const ( + pgUser = "pamuser" + pgPassword = "pampassword" + pgDatabase = "testdb" +) - const ( - pgUser = "pamuser" - pgPassword = "pampassword" - pgDatabase = "testdb" - ) +type recordedQuery struct { + Input string `json:"input"` + Output string `json:"output"` +} - // Start PAM-target Postgres via testcontainers +func startPostgresTarget(t *testing.T, ctx context.Context) (string, int) { pgContainer, err := tcpostgres.Run(ctx, "postgres:16", tcpostgres.WithDatabase(pgDatabase), tcpostgres.WithUsername(pgUser), @@ -56,17 +56,18 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { directConn.Close(ctx) slog.Info("Verified PAM Postgres is accessible directly") - // Get host/port for the PAM account pgHost, err := pgContainer.Host(ctx) require.NoError(t, err) pgPort, err := pgContainer.MappedPort(ctx, "5432") require.NoError(t, err) + return pgHost, pgPort.Int() +} - // Create folder + template + account (new PAM model) - folderName := "pg-folder" - accountName := "pg-account" +func createPostgresPamAccount(t *testing.T, ctx context.Context, infra *PAMTestInfra, prefix, pgHost string, pgPort int) (folderName, accountName string) { + folderName = prefix + "-folder" + accountName = prefix + "-account" folderId := CreatePamFolder(t, ctx, infra, folderName) - templateId := CreatePamTemplate(t, ctx, infra, "pg-template", client.CreatePamAccountTemplateJSONBodyTypePostgres) + templateId := CreatePamTemplate(t, ctx, infra, prefix+"-template", client.CreatePamAccountTemplateJSONBodyTypePostgres) gatewayId := infra.GatewayId password := pgPassword @@ -86,7 +87,7 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { SslRejectUnauthorized bool `json:"sslRejectUnauthorized"` }{ Host: pgHost, - Port: float32(pgPort.Int()), + Port: float32(pgPort), Database: pgDatabase, SslEnabled: false, SslRejectUnauthorized: false, @@ -103,13 +104,12 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { require.NoError(t, err) require.Equal(t, http.StatusOK, acctResp.StatusCode(), "create account: %s", string(acctResp.Body)) slog.Info("Created Postgres PAM account") + return folderName, accountName +} - // Login with provisioned admin user - LoginUser(t, ctx, infra) - - // Run pam access / +func startPostgresPamAccess(t *testing.T, ctx context.Context, infra *PAMTestInfra, folderName, accountName string) (*helpers.Command, int) { freePort := helpers.GetFreePort() - pamCmd := helpers.Command{ + pamCmd := &helpers.Command{ Test: t, RunMethod: helpers.RunMethodSubprocess, DisableTempHomeDir: true, @@ -128,7 +128,7 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { // Wait for proxy to be ready (printed to stdout via fmt.Printf) result := helpers.WaitFor(t, helpers.WaitForOptions{ - EnsureCmdRunning: &pamCmd, + EnsureCmdRunning: pamCmd, Condition: func() helpers.ConditionResult { if strings.Contains(pamCmd.Stdout(), "Proxy Session Started") { return helpers.ConditionSuccess @@ -137,15 +137,17 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { }, }) if result != helpers.WaitSuccess { - infra.DumpOutput(&pamCmd) + infra.DumpOutput(pamCmd) } require.Equal(t, helpers.WaitSuccess, result, "Database proxy should start successfully") + return pamCmd, freePort +} - // Connect via pgx to the proxy and run SELECT 1 - proxyConnStr := fmt.Sprintf("postgres://%s@127.0.0.1:%d/%s?sslmode=disable", pgUser, freePort, pgDatabase) +func connectPostgresProxy(t *testing.T, ctx context.Context, infra *PAMTestInfra, pamCmd *helpers.Command, port int) *pgx.Conn { + proxyConnStr := fmt.Sprintf("postgres://%s@127.0.0.1:%d/%s?sslmode=disable", pgUser, port, pgDatabase) var proxyConn *pgx.Conn connectResult := helpers.WaitFor(t, helpers.WaitForOptions{ - EnsureCmdRunning: &pamCmd, + EnsureCmdRunning: pamCmd, Interval: 2 * time.Second, Timeout: 30 * time.Second, Condition: func() helpers.ConditionResult { @@ -159,31 +161,51 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { }, }) if connectResult != helpers.WaitSuccess { - infra.DumpOutput(&pamCmd) + infra.DumpOutput(pamCmd) } require.Equal(t, helpers.WaitSuccess, connectResult, "Should connect to database through proxy") + return proxyConn +} + +func runQueries(t *testing.T, ctx context.Context, conn *pgx.Conn, queries []recordedQuery) { + for _, query := range queries { + _, err := conn.Exec(ctx, query.Input) + require.NoError(t, err) + } +} + +func requirePostgresRecording(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string, expected []recordedQuery) { + playback := WaitForSessionRecording(t, ctx, infra, sessionId) + require.Equal(t, "postgres", playback.StorageBackend) + recorded := ReadSessionRecording[recordedQuery](t, ctx, infra, sessionId, playback) + require.Equal(t, expected, recorded) + slog.Info("Postgres session recording verified", "sessionId", sessionId, "entries", len(recorded)) +} + +func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + infra := SetupPAMInfra(t, ctx) + pgHost, pgPort := startPostgresTarget(t, ctx) + folderName, accountName := createPostgresPamAccount(t, ctx, infra, "pg", pgHost, pgPort) + LoginUser(t, ctx, infra) + pamCmd, proxyPort := startPostgresPamAccess(t, ctx, infra, folderName, accountName) + proxyConn := connectPostgresProxy(t, ctx, infra, pamCmd, proxyPort) defer proxyConn.Close(ctx) var proxyResult int - err = proxyConn.QueryRow(ctx, "SELECT 1").Scan(&proxyResult) + err := proxyConn.QueryRow(ctx, "SELECT 1").Scan(&proxyResult) require.NoError(t, err) require.Equal(t, 1, proxyResult) slog.Info("SELECT 1 through PAM proxy succeeded") - type recordedQuery struct { - Input string `json:"input"` - Output string `json:"output"` - } - expected := []recordedQuery{{Input: "SELECT 1", Output: "SELECT 1"}} - for _, query := range []recordedQuery{ + queries := []recordedQuery{ {Input: "CREATE TABLE pam_recording (note text)", Output: "CREATE TABLE"}, {Input: "INSERT INTO pam_recording VALUES ('recorded through pam')", Output: "INSERT 0 1"}, {Input: "SELECT note FROM pam_recording", Output: "SELECT 1"}, - } { - _, err := proxyConn.Exec(ctx, query.Input) - require.NoError(t, err) - expected = append(expected, query) } + runQueries(t, ctx, proxyConn, queries) sessionId := FindPamSessionId(t, ctx, infra, accountName) @@ -206,11 +228,88 @@ func TestPAM_Postgres_ConnectToDatabase(t *testing.T) { require.Equal(t, helpers.WaitSuccess, fileResult, "gateway should write the session recording under %s", infra.RecordingPath) require.NoError(t, proxyConn.Close(ctx)) - EndPamAccess(t, infra, &pamCmd) + EndPamAccess(t, infra, pamCmd) - playback := WaitForSessionRecording(t, ctx, infra, sessionId) - require.Equal(t, "postgres", playback.StorageBackend) - recorded := ReadSessionRecording[recordedQuery](t, ctx, infra, sessionId, playback) - require.Equal(t, expected, recorded) - slog.Info("Postgres session recording verified", "sessionId", sessionId, "entries", len(recorded)) + requirePostgresRecording(t, ctx, infra, sessionId, + append([]recordedQuery{{Input: "SELECT 1", Output: "SELECT 1"}}, queries...)) +} + +func TestPAM_Postgres_RecordingSurvivesGatewayCrash(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + infra := SetupPAMInfra(t, ctx) + pgHost, pgPort := startPostgresTarget(t, ctx) + folderName, accountName := createPostgresPamAccount(t, ctx, infra, "pg-crash", pgHost, pgPort) + LoginUser(t, ctx, infra) + pamCmd, proxyPort := startPostgresPamAccess(t, ctx, infra, folderName, accountName) + proxyConn := connectPostgresProxy(t, ctx, infra, pamCmd, proxyPort) + sessionId := FindPamSessionId(t, ctx, infra, accountName) + + uploaded := []recordedQuery{ + {Input: "CREATE TABLE crash_recording (note text)", Output: "CREATE TABLE"}, + {Input: "INSERT INTO crash_recording VALUES ('uploaded before the crash')", Output: "INSERT 0 1"}, + } + runQueries(t, ctx, proxyConn, uploaded) + waitForPlayback(t, ctx, infra, sessionId, func(p SessionPlayback) bool { return len(p.Chunks) > 0 }) + + // Written to disk but not uploaded when the gateway dies, so the restarted gateway has to send it. + onDisk := []recordedQuery{ + {Input: "INSERT INTO crash_recording VALUES ('on disk when the gateway died')", Output: "INSERT 0 1"}, + } + runQueries(t, ctx, proxyConn, onDisk) + + CrashAndRestartGateway(t, ctx, infra) + _ = proxyConn.Close(ctx) // the crash already dropped it + + proxyConn = connectPostgresProxy(t, ctx, infra, pamCmd, proxyPort) + afterRestart := []recordedQuery{ + {Input: "SELECT note FROM crash_recording", Output: "SELECT 2"}, + } + runQueries(t, ctx, proxyConn, afterRestart) + require.NoError(t, proxyConn.Close(ctx)) + EndPamAccess(t, infra, pamCmd) + + requirePostgresRecording(t, ctx, infra, sessionId, slices.Concat(uploaded, onDisk, afterRestart)) +} + +func TestPAM_Postgres_RecordingSurvivesRateLimiting(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + infra := SetupPAMInfra(t, ctx, WithGatewayAPIProxy()) + pgHost, pgPort := startPostgresTarget(t, ctx) + folderName, accountName := createPostgresPamAccount(t, ctx, infra, "pg-throttle", pgHost, pgPort) + LoginUser(t, ctx, infra) + pamCmd, proxyPort := startPostgresPamAccess(t, ctx, infra, folderName, accountName) + proxyConn := connectPostgresProxy(t, ctx, infra, pamCmd, proxyPort) + sessionId := FindPamSessionId(t, ctx, infra, accountName) + + infra.GatewayAPI.ThrottleChunkUploads(true) + throttled := []recordedQuery{ + {Input: "CREATE TABLE throttled_recording (note text)", Output: "CREATE TABLE"}, + {Input: "INSERT INTO throttled_recording VALUES ('first upload was rate limited')", Output: "INSERT 0 1"}, + } + runQueries(t, ctx, proxyConn, throttled) + result := helpers.WaitFor(t, helpers.WaitForOptions{ + Timeout: 30 * time.Second, + Interval: time.Second, + Condition: func() helpers.ConditionResult { + if infra.GatewayAPI.RejectedChunkUploads() == 0 { + return helpers.ConditionWait + } + return helpers.ConditionSuccess + }, + }) + require.Equal(t, helpers.WaitSuccess, result, "gateway should try to upload the recording while the session is open") + infra.GatewayAPI.ThrottleChunkUploads(false) + + afterThrottle := []recordedQuery{ + {Input: "SELECT note FROM throttled_recording", Output: "SELECT 1"}, + } + runQueries(t, ctx, proxyConn, afterThrottle) + require.NoError(t, proxyConn.Close(ctx)) + EndPamAccess(t, infra, pamCmd) + + requirePostgresRecording(t, ctx, infra, sessionId, slices.Concat(throttled, afterThrottle)) } From 2698a1e4f544a0bbded7ff0cdf23e920ca953731 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 12 Nov 2026 15:44:10 -0500 Subject: [PATCH 4/4] address grep comments --- e2e/pam/pam_helpers.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/e2e/pam/pam_helpers.go b/e2e/pam/pam_helpers.go index 8add0f928..3cfc421ae 100644 --- a/e2e/pam/pam_helpers.go +++ b/e2e/pam/pam_helpers.go @@ -81,7 +81,6 @@ func WithLocalStack() SetupPAMOption { } } -// WithGatewayAPIProxy sends the gateway's API calls through infra.GatewayAPI. func WithGatewayAPIProxy() SetupPAMOption { return func(_ *helpers.InfisicalService, cfg *setupPAMConfig) { cfg.gatewayAPIProxy = true @@ -144,7 +143,8 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P gatewayApiUrl = gatewayAPI.URL } gatewayCmd := &helpers.Command{ - Test: t, + Test: t, + RunMethod: helpers.RunMethodSubprocess, Args: []string{"gateway", "start", gatewayName, "--enroll-method", "token", "--token", enrollmentToken, @@ -195,8 +195,7 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P var chunkUploadPath = regexp.MustCompile(`^/api/v1/pam/sessions/[^/]+/chunks(/presigned-put)?$`) -// GatewayAPIProxy forwards the gateway's API calls to the backend and, while throttled, answers its -// recording chunk uploads the way the backend's rate limiter does. +// Tests can't lower the backend's write limit, which comes from the license plan, so this answers chunk uploads with the 429 it would send. type GatewayAPIProxy struct { URL string throttled atomic.Bool @@ -234,8 +233,6 @@ func (p *GatewayAPIProxy) RejectedChunkUploads() int { return int(p.rejected.Load()) } -// CrashAndRestartGateway kills the gateway without letting it clean up, then starts it again the way a -// restarted host would, with its recordings and queued chunks still on disk. func CrashAndRestartGateway(t *testing.T, ctx context.Context, infra *PAMTestInfra) { crashed := infra.GatewayCmd require.NoError(t, crashed.Cmd().Process.Kill())