Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion e2e/packages/infisical/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,8 @@ func WithLocalStackService(buckets ...string) StackOption {
}
s.Project.Services["localstack"] = types.ServiceConfig{
Image: "localstack/localstack:4.4@sha256:b52c16663c70b7234f217cb993a339b46686e30a1a5d9279cb5feeb2202f837c",
Ports: []types.ServicePortConfig{{Published: "", Target: 4566}},
// Published on 4566 so host processes, like the gateway uploading recordings, can use URLs presigned for <bucket>.s3.localhost.localstack.cloud:4566
Ports: []types.ServicePortConfig{{Published: "4566", Target: 4566}},
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
Environment: types.NewMappingWithEquals([]string{
"SERVICES=s3,sts",
}),
Expand Down
297 changes: 288 additions & 9 deletions e2e/pam/pam_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,25 @@ package pam
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log/slog"
"maps"
"net"
"net/http"
"net/http/httptest"
"net/http/httputil"
"net/url"
"os"
"path/filepath"
"regexp"
"slices"
"sync/atomic"
"testing"
"time"

Expand Down Expand Up @@ -47,35 +57,48 @@ type PAMTestInfra struct {
GatewayId openapitypes.UUID
RelayCmd *helpers.Command
GatewayCmd *helpers.Command
GatewayAPI *GatewayAPIProxy
ProvisionResult *client.ProvisionResult
SharedHomeDir string
RecordingPath string
}

type SetupPAMOption func(svc *helpers.InfisicalService, extraEnv *[]string)
type setupPAMConfig struct {
backendEnv []string
gatewayAPIProxy bool
}

type SetupPAMOption func(svc *helpers.InfisicalService, cfg *setupPAMConfig)

const RecordingBucket = "e2e-recordings"

func WithLocalStack() SetupPAMOption {
return func(svc *helpers.InfisicalService, extraEnv *[]string) {
return func(svc *helpers.InfisicalService, cfg *setupPAMConfig) {
infisicalpkg.WithLocalStackService(RecordingBucket)(svc.Stack)
*extraEnv = append(*extraEnv,
cfg.backendEnv = append(cfg.backendEnv,
"AWS_ENDPOINT_URL=http://localstack:4566",
fmt.Sprintf("AWS_ENDPOINT_URL_S3=http://%s:4566", infisicalpkg.LocalStackS3Endpoint))
}
}

func WithGatewayAPIProxy() SetupPAMOption {
return func(_ *helpers.InfisicalService, cfg *setupPAMConfig) {
cfg.gatewayAPIProxy = true
}
}

func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *PAMTestInfra {
svc := helpers.NewInfisicalService()

var extraEnv []string
var cfg setupPAMConfig
for _, opt := range opts {
opt(svc, &extraEnv)
opt(svc, &cfg)
}

env := append([]string{
"ALLOW_INTERNAL_IP_CONNECTIONS=true",
"NODE_ENV=test",
}, extraEnv...)
}, cfg.backendEnv...)

svc.WithBackendEnvironment(types.NewMappingWithEquals(env)).Up(t, ctx)

Expand Down Expand Up @@ -113,16 +136,23 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P
require.NoError(t, os.MkdirAll(sessionRecordingPath, 0755))
gatewayName := helpers.RandomSlug(2)
gatewayId, enrollmentToken := CreateGatewayWithEnrollmentToken(t, ctx, svc, gatewayName)
gatewayApiUrl := svc.ApiUrl(t)
var gatewayAPI *GatewayAPIProxy
if cfg.gatewayAPIProxy {
gatewayAPI = startGatewayAPIProxy(t, gatewayApiUrl)
gatewayApiUrl = gatewayAPI.URL
}
gatewayCmd := &helpers.Command{
Test: t,
Test: t,
RunMethod: helpers.RunMethodSubprocess,
Args: []string{"gateway", "start", gatewayName,
"--enroll-method", "token",
"--token", enrollmentToken,
"--domain", svc.ApiUrl(t),
"--domain", gatewayApiUrl,
fmt.Sprintf("--pam-session-recording-path=%s", sessionRecordingPath),
},
Env: map[string]string{
"INFISICAL_API_URL": svc.ApiUrl(t),
"INFISICAL_API_URL": gatewayApiUrl,
},
}
gatewayCmd.Start(ctx)
Expand Down Expand Up @@ -156,9 +186,84 @@ func SetupPAMInfra(t *testing.T, ctx context.Context, opts ...SetupPAMOption) *P
GatewayId: gatewayId,
RelayCmd: relayCmd,
GatewayCmd: gatewayCmd,
GatewayAPI: gatewayAPI,
ProvisionResult: svc.ProvisionResult(),
SharedHomeDir: sharedHomeDir,
RecordingPath: sessionRecordingPath,
}
}

var chunkUploadPath = regexp.MustCompile(`^/api/v1/pam/sessions/[^/]+/chunks(/presigned-put)?$`)

// Tests can't lower the backend's write limit, which comes from the license plan, so this answers chunk uploads with the 429 it would send.
type GatewayAPIProxy struct {
URL string
throttled atomic.Bool
rejected atomic.Int32
}

func startGatewayAPIProxy(t *testing.T, apiUrl string) *GatewayAPIProxy {
target, err := url.Parse(apiUrl)
require.NoError(t, err)
forward := &httputil.ReverseProxy{Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(target) }}

proxy := &GatewayAPIProxy{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !proxy.throttled.Load() || r.Method != http.MethodPost || !chunkUploadPath.MatchString(r.URL.Path) {
forward.ServeHTTP(w, r)
return
}
proxy.rejected.Add(1)
w.Header().Set("Content-Type", "application/json")
// The backend sends the seconds left in its one-minute window, so 60 is the longest it asks for.
w.Header().Set("Retry-After", "60")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = io.WriteString(w, `{"statusCode":429,"message":"Rate limit exceeded. Please try again in 60 seconds","error":"RateLimitExceeded"}`)
}))
t.Cleanup(server.Close)
proxy.URL = server.URL
return proxy
}

func (p *GatewayAPIProxy) ThrottleChunkUploads(throttled bool) {
p.throttled.Store(throttled)
}

func (p *GatewayAPIProxy) RejectedChunkUploads() int {
return int(p.rejected.Load())
}

func CrashAndRestartGateway(t *testing.T, ctx context.Context, infra *PAMTestInfra) {
crashed := infra.GatewayCmd
require.NoError(t, crashed.Cmd().Process.Kill())
Comment thread
lb-vn marked this conversation as resolved.
Resolved
result := helpers.WaitFor(t, helpers.WaitForOptions{
Timeout: 10 * time.Second,
Interval: 200 * time.Millisecond,
Condition: func() helpers.ConditionResult {
if crashed.IsRunning() {
return helpers.ConditionWait
}
return helpers.ConditionSuccess
},
})
require.Equal(t, helpers.WaitSuccess, result, "gateway should exit when killed")

// Same HOME, because enrollment saved the gateway's access token there and the token can't enroll twice.
gatewayCmd := &helpers.Command{
Test: t,
RunMethod: helpers.RunMethodSubprocess,
DisableTempHomeDir: true,
Args: crashed.Args,
Env: maps.Clone(crashed.Env),
}
gatewayCmd.Start(ctx)
t.Cleanup(gatewayCmd.Stop)
infra.GatewayCmd = gatewayCmd
result = helpers.WaitForStderr(t, helpers.WaitForStderrOptions{
EnsureCmdRunning: gatewayCmd,
ExpectedString: "Gateway is reachable by Infisical",
})
require.Equal(t, helpers.WaitSuccess, result, "gateway should come back after a restart")
}

// postJSON sends an authenticated JSON POST and decodes the response into out. Used for endpoints
Expand Down Expand Up @@ -428,3 +533,177 @@ func LoginUser(t *testing.T, ctx context.Context, infra *PAMTestInfra) {
})
require.Equal(t, helpers.WaitSuccess, result, "Login should succeed")
}

func getRaw(ctx context.Context, url, token string) (int, []byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return 0, nil, err
}
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}

resp, err := http.DefaultClient.Do(req)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()

body, err := io.ReadAll(resp.Body)
return resp.StatusCode, body, err
}

func getOK(t *testing.T, ctx context.Context, url, token string) []byte {
status, body, err := getRaw(ctx, url, token)
require.NoError(t, err)
require.Equal(t, http.StatusOK, status, "GET %s: %s", url, string(body))
return body
}

// EndPamAccess interrupts `pam access` like Ctrl+C, which ends the session through the gateway; Stop kills it and leaves the session open.
func EndPamAccess(t *testing.T, infra *PAMTestInfra, pamCmd *helpers.Command) {
defer func() {
if t.Failed() {
infra.DumpOutput(pamCmd)
}
}()
require.NoError(t, pamCmd.Cmd().Process.Signal(os.Interrupt))
result := helpers.WaitFor(t, helpers.WaitForOptions{
Timeout: 60 * time.Second,
Interval: time.Second,
Condition: func() helpers.ConditionResult {
if pamCmd.IsRunning() {
return helpers.ConditionWait
}
return helpers.ConditionSuccess
},
})
require.Equal(t, helpers.WaitSuccess, result, "pam access should exit after an interrupt")
}

func FindPamSessionId(t *testing.T, ctx context.Context, infra *PAMTestInfra, accountName string) string {
var list struct {
Sessions []struct {
Id string `json:"id"`
AccountName string `json:"accountName"`
} `json:"sessions"`
}
body := getOK(t, ctx, fmt.Sprintf("%s/api/v1/pam/sessions?limit=100", infra.Infisical.ApiUrl(t)), infra.ProvisionResult.Token)
require.NoError(t, json.Unmarshal(body, &list))

var ids []string
for _, s := range list.Sessions {
if s.AccountName == accountName {
ids = append(ids, s.Id)
}
}
require.Len(t, ids, 1, "expected one session for account %s", accountName)
return ids[0]
}

type RecordingChunk struct {
ChunkIndex int `json:"chunkIndex"`
StorageBackend string `json:"storageBackend"`
ExternalChunkObjectKey *string `json:"externalChunkObjectKey"`
CiphertextSha256 string `json:"ciphertextSha256"`
CiphertextBytes int `json:"ciphertextBytes"`
Iv string `json:"iv"`
PresignedGetUrl *string `json:"presignedGetUrl"`
}

type SessionPlayback struct {
SessionComplete bool `json:"sessionComplete"`
SessionKey string `json:"sessionKey"`
ProjectId string `json:"projectId"`
StorageBackend string `json:"storageBackend"`
Chunks []RecordingChunk `json:"chunks"`
}

func WaitForSessionRecording(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string) SessionPlayback {
return waitForPlayback(t, ctx, infra, sessionId, func(p SessionPlayback) bool {
return p.SessionComplete && len(p.Chunks) > 0
})
}

func waitForPlayback(t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string, ready func(SessionPlayback) bool) SessionPlayback {
defer func() {
if t.Failed() {
infra.GatewayCmd.DumpOutput()
}
}()
playbackUrl := fmt.Sprintf("%s/api/v1/pam/sessions/%s/playback", infra.Infisical.ApiUrl(t), sessionId)
var playback SessionPlayback
var failure string
result := helpers.WaitFor(t, helpers.WaitForOptions{
Timeout: 90 * time.Second,
Interval: 2 * time.Second,
Condition: func() helpers.ConditionResult {
status, body, err := getRaw(ctx, playbackUrl, infra.ProvisionResult.Token)
if err != nil || status != http.StatusOK {
failure = fmt.Sprintf("GET %s: status %d, error %v: %s", playbackUrl, status, err, string(body))
return helpers.ConditionBreakEarly
}

playback = SessionPlayback{}
if err := json.Unmarshal(body, &playback); err != nil {
failure = fmt.Sprintf("GET %s: %v", playbackUrl, err)
return helpers.ConditionBreakEarly
}
slog.Info("Waiting for session recording", "sessionId", sessionId,
"sessionComplete", playback.SessionComplete, "chunks", len(playback.Chunks))
if ready(playback) {
return helpers.ConditionSuccess
}
return helpers.ConditionWait
},
})
require.Equal(t, helpers.WaitSuccess, result, failure)
return playback
}

func ReadSessionRecording[T any](t *testing.T, ctx context.Context, infra *PAMTestInfra, sessionId string, playback SessionPlayback) []T {
sessionKey, err := base64.StdEncoding.DecodeString(playback.SessionKey)
require.NoError(t, err)
block, err := aes.NewCipher(sessionKey)
require.NoError(t, err)
gcm, err := cipher.NewGCM(block)
require.NoError(t, err)

chunks := slices.Clone(playback.Chunks)
slices.SortFunc(chunks, func(a, b RecordingChunk) int { return a.ChunkIndex - b.ChunkIndex })

var events []T
for i, chunk := range chunks {
require.Equal(t, i, chunk.ChunkIndex, "recording chunks should be contiguous")

var ciphertext []byte
switch chunk.StorageBackend {
case "postgres":
ciphertext = getOK(t, ctx, fmt.Sprintf("%s/api/v1/pam/sessions/%s/chunks/%d/ciphertext",
infra.Infisical.ApiUrl(t), sessionId, chunk.ChunkIndex), infra.ProvisionResult.Token)
case "aws-s3":
require.NotNil(t, chunk.ExternalChunkObjectKey, "chunk %d has no object key", chunk.ChunkIndex)
require.Equal(t, fmt.Sprintf("%s/%s/chunk-%06d.json.enc", playback.ProjectId, sessionId, chunk.ChunkIndex),
*chunk.ExternalChunkObjectKey)
require.NotNil(t, chunk.PresignedGetUrl, "chunk %d has no download URL", chunk.ChunkIndex)
ciphertext = getOK(t, ctx, *chunk.PresignedGetUrl, "")
default:
require.Failf(t, "unexpected storage backend", "chunk %d is stored in %q", chunk.ChunkIndex, chunk.StorageBackend)
}

require.Equal(t, chunk.CiphertextBytes, len(ciphertext), "chunk %d size", chunk.ChunkIndex)
digest := sha256.Sum256(ciphertext)
require.Equal(t, chunk.CiphertextSha256, base64.StdEncoding.EncodeToString(digest[:]), "chunk %d digest", chunk.ChunkIndex)

iv, err := base64.StdEncoding.DecodeString(chunk.Iv)
require.NoError(t, err)
aad := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%d|%s|v1", playback.ProjectId, sessionId, chunk.ChunkIndex, chunk.StorageBackend)))
plaintext, err := gcm.Open(nil, iv, ciphertext, aad[:])
require.NoError(t, err, "chunk %d should decrypt with the session key", chunk.ChunkIndex)

var chunkEvents []T
require.NoError(t, json.Unmarshal(plaintext, &chunkEvents), "chunk %d should hold a JSON array of events", chunk.ChunkIndex)
events = append(events, chunkEvents...)
}
return events
}
Loading
Loading