diff --git a/packages/api/model.go b/packages/api/model.go index 8b317be82..1e511c5a9 100644 --- a/packages/api/model.go +++ b/packages/api/model.go @@ -1104,8 +1104,15 @@ type PAMSessionLogMaskingConfig struct { BuiltInDetection bool `json:"builtInDetection"` } +type PAMOutputMaskingConfig struct { + Patterns []string `json:"patterns"` + AccountCredentials bool `json:"accountCredentials"` + CreditCards bool `json:"creditCards"` +} + type PAMPolicyRules struct { CommandBlocking *PAMPolicyRuleConfig `json:"command-blocking,omitempty"` + OutputMasking *PAMOutputMaskingConfig `json:"output-masking,omitempty"` SessionLogMasking *PAMSessionLogMaskingConfig `json:"session-log-masking,omitempty"` } diff --git a/packages/gateway-v2/capabilities.go b/packages/gateway-v2/capabilities.go index 8b2b73825..22b645d6a 100644 --- a/packages/gateway-v2/capabilities.go +++ b/packages/gateway-v2/capabilities.go @@ -8,6 +8,8 @@ const CapabilitySupportedAccountTypes = "supported_account_types" const CapabilityClickhouseNativeProtocol = "clickhouseNativeProtocol" +const CapabilityOutputMasking = "outputMasking" + const CapabilityCertificateScan = "certificateScan" const CapabilityWindowsCertificateScan = "windowsCertificateScan" diff --git a/packages/gateway-v2/gateway.go b/packages/gateway-v2/gateway.go index f48b9596f..23cc04ea1 100644 --- a/packages/gateway-v2/gateway.go +++ b/packages/gateway-v2/gateway.go @@ -471,6 +471,7 @@ func (g *Gateway) registerHeartBeat(ctx context.Context, errCh chan error) { } capabilities[CapabilitySupportedAccountTypes] = pam.GetSupportedResourceTypes() capabilities[CapabilityClickhouseNativeProtocol] = true + capabilities[CapabilityOutputMasking] = true capabilities[CapabilityCertificateScan] = true capabilities[CapabilityWindowsCertificateScan] = true req := api.GatewayHeartbeatRequest{Capabilities: capabilities} diff --git a/packages/pam/handlers/postgres.go b/packages/pam/handlers/postgres.go index d38fde81f..03db50d7c 100644 --- a/packages/pam/handlers/postgres.go +++ b/packages/pam/handlers/postgres.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "encoding/base64" "encoding/hex" + "errors" "fmt" "io" "net" @@ -17,7 +18,9 @@ import ( "time" session "github.com/Infisical/infisical-merge/packages/pam/session" + "github.com/Infisical/infisical-merge/packages/pam/session/masking" "github.com/jackc/pgx/v5/pgproto3" + "github.com/jackc/pgx/v5/pgtype" "github.com/rs/zerolog/log" "golang.org/x/crypto/pbkdf2" // TODO: Remove this once we update to go 1.25.1 or later where it's already in the standard library ) @@ -31,6 +34,7 @@ type PostgresProxyConfig struct { TLSConfig *tls.Config SessionID string SessionLogger session.SessionLogger + OutputMasker masking.Masker } type PostgresProxy struct { @@ -39,6 +43,12 @@ type PostgresProxy struct { pendingRequests map[string]*PendingRequest preparedStatements map[string]string activePortal string + + describes []describeSource + rowColumns []pgColumnFormat + typeMap *pgtype.Map + discarding bool + closeAfterReply bool } type PendingRequest struct { @@ -679,6 +689,11 @@ func (p *PostgresProxy) proxyClientToServer(clientBackend *pgproto3.Backend, ser p.trackClientMessage(msg) + if p.masking() { + if describe := p.trackDescribe(msg); describe != nil { + serverFrontend.Send(describe) + } + } serverFrontend.Send(msg) err = serverFrontend.Flush() if err != nil { @@ -696,6 +711,12 @@ func (p *PostgresProxy) proxyServerToClient(serverFrontend *pgproto3.Frontend, c return } + if p.masking() { + if msg = p.maskServerMessage(msg); msg == nil { + continue + } + } + p.trackServerMessage(msg) clientBackend.Send(msg) @@ -704,6 +725,10 @@ func (p *PostgresProxy) proxyServerToClient(serverFrontend *pgproto3.Frontend, c errChan <- err return } + if p.closeAfterReply { + errChan <- errors.New("replication refused because the account masks output") + return + } } } diff --git a/packages/pam/handlers/postgres_masking.go b/packages/pam/handlers/postgres_masking.go new file mode 100644 index 000000000..3f174b592 --- /dev/null +++ b/packages/pam/handlers/postgres_masking.go @@ -0,0 +1,255 @@ +package handlers + +import ( + "bytes" + "encoding/json" + + "github.com/Infisical/infisical-merge/packages/pam/session/masking" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/jackc/pgx/v5/pgtype" +) + +var pgTextTypes = map[uint32]bool{ + pgtype.TextOID: true, + pgtype.VarcharOID: true, + pgtype.BPCharOID: true, + pgtype.NameOID: true, + pgtype.JSONOID: true, + pgtype.JSONBOID: true, + pgtype.XMLOID: true, + pgtype.UnknownOID: true, +} + +type pgColumnFormat struct { + typeOID uint32 + binary bool +} + +// The server answers Describes in request order, so a queue of these says whose answer each one is. +type describeSource int + +const ( + describeForClient describeSource = iota + describeForMasking + syncPoint +) + +func (p *PostgresProxy) masking() bool { + return p.config.OutputMasker != nil +} + +// Each Execute gets a portal Describe sent ahead of it, so its rows always follow a RowDescription. +func (p *PostgresProxy) trackDescribe(msg pgproto3.FrontendMessage) *pgproto3.Describe { + p.mutex.Lock() + defer p.mutex.Unlock() + + switch m := msg.(type) { + case *pgproto3.Describe: + p.describes = append(p.describes, describeForClient) + case *pgproto3.Execute: + p.describes = append(p.describes, describeForMasking) + return &pgproto3.Describe{ObjectType: 'P', Name: m.Portal} + case *pgproto3.Sync, *pgproto3.Query, *pgproto3.FunctionCall: + p.describes = append(p.describes, syncPoint) + } + return nil +} + +func (p *PostgresProxy) maskServerMessage(msg pgproto3.BackendMessage) pgproto3.BackendMessage { + if _, ok := msg.(*pgproto3.ReadyForQuery); ok { + p.discarding = false + p.popThroughSync() + return msg + } + if p.discarding { + return nil + } + + masker := p.config.OutputMasker + switch m := msg.(type) { + case *pgproto3.RowDescription: + p.rowColumns = p.rowColumns[:0] + for i, field := range m.Fields { + p.rowColumns = append(p.rowColumns, pgColumnFormat{field.DataTypeOID, field.Format == pgtype.BinaryFormatCode}) + // Dynamic SQL behind a cursor can name a column after a value. + m.Fields[i].Name = masker.Mask(field.Name) + } + return p.answerDescribe(msg) + case *pgproto3.NoData: + p.rowColumns = p.rowColumns[:0] + return p.answerDescribe(msg) + case *pgproto3.DataRow: + for i, value := range m.Values { + if value == nil { + continue + } + var column pgColumnFormat + if i < len(p.rowColumns) { + column = p.rowColumns[i] + } + isText := pgTextTypes[column.typeOID] + text := p.matchText(column, value) + masked := masker.Mask(text) + if bytes.Equal(masked, text) { + continue + } + if isText && keepsValidJSON(column, masked) { + m.Values[i] = masked + } else { + // A placeholder would not decode as this column's type. + m.Values[i] = nil + } + } + case *pgproto3.CopyOutResponse: + // COPY escapes its fields, so patterns can miss values in it. + p.discarding = true + return &pgproto3.ErrorResponse{ + Severity: "ERROR", + Code: "42501", + Message: "COPY TO STDOUT is blocked because this account masks output, use SELECT instead", + } + case *pgproto3.CopyBothResponse: + // A replication stream never reaches ReadyForQuery, so only ending the connection stops it. + p.closeAfterReply = true + return &pgproto3.ErrorResponse{ + Severity: "FATAL", + Code: "42501", + Message: "Replication is blocked because this account masks output", + } + case *pgproto3.FunctionCallResponse: + if masked := masker.Mask(m.Result); !bytes.Equal(masked, m.Result) { + m.Result = nil + } + case *pgproto3.NotificationResponse: + maskStrings(masker, &m.Channel, &m.Payload) + case *pgproto3.ParameterStatus: + // The one reported parameter a query can set to arbitrary text, through set_config. + if m.Name == "application_name" { + m.Value = masker.MaskString(m.Value) + } + case *pgproto3.ErrorResponse: + maskErrorFields(masker, m) + case *pgproto3.NoticeResponse: + maskErrorFields(masker, (*pgproto3.ErrorResponse)(m)) + } + return msg +} + +// Matches the content a client reads out of a value, so binary encoding or bytea's hex can't hide a match. +func (p *PostgresProxy) matchText(column pgColumnFormat, value []byte) (text []byte) { + // pgx panics on some malformed values, and a panic here would take down the whole gateway. + defer func() { + if recover() != nil { + text = value + } + }() + + isBytea := column.typeOID == pgtype.ByteaOID || column.typeOID == pgtype.ByteaArrayOID + if pgTextTypes[column.typeOID] || (!column.binary && !isBytea) { + return value + } + if p.typeMap == nil { + p.typeMap = pgtype.NewMap() + } + dataType, ok := p.typeMap.TypeForOID(column.typeOID) + if !ok { + return value + } + format := int16(pgtype.TextFormatCode) + if column.binary { + format = pgtype.BinaryFormatCode + } + decoded, err := dataType.Codec.DecodeValue(p.typeMap, column.typeOID, format, value) + if err != nil || decoded == nil { + return value + } + if content, ok := appendContent(nil, decoded); ok { + return content + } + if encoded, err := p.typeMap.Encode(column.typeOID, pgtype.TextFormatCode, decoded, nil); err == nil { + return encoded + } + return value +} + +func appendContent(buf []byte, value any) ([]byte, bool) { + switch v := value.(type) { + case []byte: + return append(buf, v...), true + case string: + return append(buf, v...), true + case []any: + for i, element := range v { + if i > 0 { + buf = append(buf, '\n') + } + if element == nil { + continue + } + var ok bool + if buf, ok = appendContent(buf, element); !ok { + return nil, false + } + } + return buf, true + } + return nil, false +} + +func keepsValidJSON(column pgColumnFormat, masked []byte) bool { + switch column.typeOID { + case pgtype.JSONOID: + return json.Valid(masked) + case pgtype.JSONBOID: + if column.binary && len(masked) > 0 { + masked = masked[1:] // binary jsonb leads with a version byte + } + return json.Valid(masked) + } + return true +} + +// Severity, code and source location come from the server, and clients act on them. +func maskErrorFields(masker masking.Masker, e *pgproto3.ErrorResponse) { + maskStrings(masker, &e.Message, &e.Detail, &e.Hint, &e.Where, &e.InternalQuery, &e.SchemaName, &e.TableName, + &e.ColumnName, &e.DataTypeName, &e.ConstraintName) + for key, value := range e.UnknownFields { + e.UnknownFields[key] = masker.MaskString(value) + } +} + +func maskStrings(masker masking.Masker, fields ...*string) { + for _, field := range fields { + *field = masker.MaskString(*field) + } +} + +// Outside an extended batch a RowDescription answers a simple query, so it goes to the client. +func (p *PostgresProxy) answerDescribe(msg pgproto3.BackendMessage) pgproto3.BackendMessage { + p.mutex.Lock() + defer p.mutex.Unlock() + + if len(p.describes) == 0 || p.describes[0] == syncPoint { + return msg + } + source := p.describes[0] + p.describes = p.describes[1:] + if source == describeForMasking { + return nil + } + return msg +} + +// After an error the server skips the rest of the batch, so its pending answers never arrive. +func (p *PostgresProxy) popThroughSync() { + p.mutex.Lock() + defer p.mutex.Unlock() + + for len(p.describes) > 0 { + source := p.describes[0] + p.describes = p.describes[1:] + if source == syncPoint { + return + } + } +} diff --git a/packages/pam/handlers/postgres_masking_test.go b/packages/pam/handlers/postgres_masking_test.go new file mode 100644 index 000000000..a2695dcc0 --- /dev/null +++ b/packages/pam/handlers/postgres_masking_test.go @@ -0,0 +1,308 @@ +package handlers + +import ( + "encoding/binary" + "regexp" + "testing" + + "github.com/Infisical/infisical-merge/packages/pam/session/masking" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/jackc/pgx/v5/pgtype" +) + +func newMaskingProxy() *PostgresProxy { + return NewPostgresProxy(PostgresProxyConfig{ + OutputMasker: masking.New([]*regexp.Regexp{regexp.MustCompile(`secret`), regexp.MustCompile(`\b\d{16}\b`)}, false, nil, "test"), + }) +} + +func relay(p *PostgresProxy, client []pgproto3.FrontendMessage, server []pgproto3.BackendMessage) (int, []pgproto3.BackendMessage) { + injected := 0 + for _, msg := range client { + if p.trackDescribe(msg) != nil { + injected++ + } + } + var forwarded []pgproto3.BackendMessage + for _, msg := range server { + if out := p.maskServerMessage(msg); out != nil { + forwarded = append(forwarded, out) + } + } + return injected, forwarded +} + +func rowDescription(format int16, oids ...uint32) *pgproto3.RowDescription { + desc := &pgproto3.RowDescription{} + for _, oid := range oids { + desc.Fields = append(desc.Fields, pgproto3.FieldDescription{Name: []byte("c"), DataTypeOID: oid, Format: format}) + } + return desc +} + +func dataRow(values ...[]byte) *pgproto3.DataRow { + return &pgproto3.DataRow{Values: values} +} + +func messageTypes(msgs []pgproto3.BackendMessage) []string { + var types []string + for _, msg := range msgs { + switch msg.(type) { + case *pgproto3.ParameterDescription: + types = append(types, "ParameterDescription") + case *pgproto3.RowDescription: + types = append(types, "RowDescription") + case *pgproto3.DataRow: + types = append(types, "DataRow") + case *pgproto3.PortalSuspended: + types = append(types, "PortalSuspended") + case *pgproto3.CommandComplete: + types = append(types, "CommandComplete") + case *pgproto3.ErrorResponse: + types = append(types, "ErrorResponse") + case *pgproto3.ReadyForQuery: + types = append(types, "ReadyForQuery") + default: + types = append(types, "other") + } + } + return types +} + +func assertTypes(t *testing.T, got []pgproto3.BackendMessage, want ...string) { + t.Helper() + gotTypes := messageTypes(got) + if len(gotTypes) != len(want) { + t.Fatalf("forwarded %v, want %v", gotTypes, want) + } + for i := range want { + if gotTypes[i] != want[i] { + t.Fatalf("forwarded %v, want %v", gotTypes, want) + } + } +} + +func TestMaskingDropsOnlyTheInsertedDescribe(t *testing.T) { + p := newMaskingProxy() + injected, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Describe{ObjectType: 'S'}, &pgproto3.Execute{}, &pgproto3.Sync{}}, + []pgproto3.BackendMessage{ + &pgproto3.ParameterDescription{}, + rowDescription(0, pgtype.TextOID), + rowDescription(0, pgtype.TextOID), + dataRow([]byte("a secret")), + &pgproto3.CommandComplete{CommandTag: []byte("SELECT 1")}, + &pgproto3.ReadyForQuery{TxStatus: 'I'}, + }) + if injected != 1 { + t.Fatalf("injected %d Describes, want 1", injected) + } + assertTypes(t, forwarded, "ParameterDescription", "RowDescription", "DataRow", "CommandComplete", "ReadyForQuery") + if got := string(forwarded[2].(*pgproto3.DataRow).Values[0]); got != "a [MASKED]" { + t.Fatalf("row value %q, want %q", got, "a [MASKED]") + } +} + +func TestMaskingClearsABatchThatFailedBeforeItsReply(t *testing.T) { + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{ + &pgproto3.Parse{}, &pgproto3.Bind{}, &pgproto3.Describe{ObjectType: 'P'}, &pgproto3.Execute{}, &pgproto3.Sync{}, + &pgproto3.Bind{}, &pgproto3.Execute{}, &pgproto3.Sync{}, + }, + []pgproto3.BackendMessage{ + &pgproto3.ErrorResponse{Severity: "ERROR", Message: "syntax error"}, + &pgproto3.ReadyForQuery{TxStatus: 'I'}, + rowDescription(0, pgtype.TextOID), + dataRow([]byte("secret")), + &pgproto3.CommandComplete{CommandTag: []byte("SELECT 1")}, + &pgproto3.ReadyForQuery{TxStatus: 'I'}, + }) + assertTypes(t, forwarded, "ErrorResponse", "ReadyForQuery", "DataRow", "CommandComplete", "ReadyForQuery") +} + +func TestMaskingDescribesAResumedPortalAgain(t *testing.T) { + p := newMaskingProxy() + injected, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Execute{MaxRows: 1}, &pgproto3.Execute{MaxRows: 1}, &pgproto3.Sync{}}, + []pgproto3.BackendMessage{ + rowDescription(0, pgtype.TextOID), + dataRow([]byte("one secret")), + &pgproto3.PortalSuspended{}, + rowDescription(0, pgtype.TextOID), + dataRow([]byte("two secret")), + &pgproto3.CommandComplete{CommandTag: []byte("SELECT 2")}, + &pgproto3.ReadyForQuery{TxStatus: 'I'}, + }) + if injected != 2 { + t.Fatalf("injected %d Describes, want 2", injected) + } + assertTypes(t, forwarded, "DataRow", "PortalSuspended", "DataRow", "CommandComplete", "ReadyForQuery") +} + +func TestMaskingForwardsASimpleQueryRowDescription(t *testing.T) { + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Query{String: "SELECT note FROM notes"}}, + []pgproto3.BackendMessage{ + rowDescription(0, pgtype.TextOID), + dataRow([]byte("secret")), + &pgproto3.CommandComplete{CommandTag: []byte("SELECT 1")}, + &pgproto3.ReadyForQuery{TxStatus: 'I'}, + }) + assertTypes(t, forwarded, "RowDescription", "DataRow", "CommandComplete", "ReadyForQuery") +} + +func TestMaskingMatchesBinaryValuesAsText(t *testing.T) { + typeMap := pgtype.NewMap() + var card pgtype.Numeric + if err := card.Scan("4111111111111111"); err != nil { + t.Fatal(err) + } + binaryCard, err := typeMap.Encode(pgtype.NumericOID, pgtype.BinaryFormatCode, card, nil) + if err != nil { + t.Fatal(err) + } + + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Execute{}, &pgproto3.Sync{}}, + []pgproto3.BackendMessage{rowDescription(1, pgtype.NumericOID), dataRow(binaryCard)}) + if value := forwarded[0].(*pgproto3.DataRow).Values[0]; value != nil { + t.Fatalf("binary numeric card number was forwarded as %v, want NULL", value) + } +} + +func TestMaskingKeepsJSONParseable(t *testing.T) { + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Query{String: "SELECT doc FROM docs"}}, + []pgproto3.BackendMessage{ + rowDescription(0, pgtype.JSONOID, pgtype.JSONOID), + dataRow([]byte(`{"note":"secret"}`), []byte(`{"card":4111111111111111}`)), + }) + row := forwarded[1].(*pgproto3.DataRow) + if got := string(row.Values[0]); got != `{"note":"[MASKED]"}` { + t.Fatalf("string match gave %q", got) + } + if row.Values[1] != nil { + t.Fatalf("number match gave %q, want NULL", row.Values[1]) + } +} + +func TestMaskingRefusesCopyAndReplication(t *testing.T) { + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Query{String: "COPY notes TO STDOUT"}}, + []pgproto3.BackendMessage{ + &pgproto3.CopyOutResponse{}, + &pgproto3.CopyData{Data: []byte("secret\n")}, + &pgproto3.CopyDone{}, + &pgproto3.CommandComplete{CommandTag: []byte("COPY 1")}, + &pgproto3.ReadyForQuery{TxStatus: 'I'}, + }) + assertTypes(t, forwarded, "ErrorResponse", "ReadyForQuery") + + _, forwarded = relay(p, nil, []pgproto3.BackendMessage{&pgproto3.CopyBothResponse{}}) + if err, ok := forwarded[0].(*pgproto3.ErrorResponse); !ok || err.Severity != "FATAL" || !p.closeAfterReply { + t.Fatalf("replication reply %+v, closeAfterReply %v", forwarded[0], p.closeAfterReply) + } +} + +func TestMaskingCoversProtocolMetadata(t *testing.T) { + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Query{String: "SELECT set_config('application_name', card::text, false) FROM cards"}}, + []pgproto3.BackendMessage{ + &pgproto3.RowDescription{Fields: []pgproto3.FieldDescription{{Name: []byte("4111111111111111"), DataTypeOID: pgtype.TextOID}}}, + &pgproto3.ParameterStatus{Name: "application_name", Value: "4111111111111111"}, + &pgproto3.ParameterStatus{Name: "server_version", Value: "16.4"}, + &pgproto3.ErrorResponse{Severity: "ERROR", Code: "P0001", Message: "raised", ColumnName: "secret", ConstraintName: "secret"}, + &pgproto3.NotificationResponse{Channel: "secret", Payload: "secret"}, + }) + if name := string(forwarded[0].(*pgproto3.RowDescription).Fields[0].Name); name != "[MASKED]" { + t.Fatalf("column name %q", name) + } + if value := forwarded[1].(*pgproto3.ParameterStatus).Value; value != "[MASKED]" { + t.Fatalf("application_name %q", value) + } + if value := forwarded[2].(*pgproto3.ParameterStatus).Value; value != "16.4" { + t.Fatalf("server_version %q", value) + } + if e := forwarded[3].(*pgproto3.ErrorResponse); e.ColumnName != "[MASKED]" || e.ConstraintName != "[MASKED]" || e.Code != "P0001" { + t.Fatalf("error fields %+v", e) + } + if n := forwarded[4].(*pgproto3.NotificationResponse); n.Channel != "[MASKED]" || n.Payload != "[MASKED]" { + t.Fatalf("notification %+v", n) + } +} + +func TestMaskingMatchesByteaContent(t *testing.T) { + typeMap := pgtype.NewMap() + encode := func(oid uint32, format int16, value any) []byte { + encoded, err := typeMap.Encode(oid, format, value, nil) + if err != nil { + t.Fatal(err) + } + return encoded + } + + for _, c := range []struct { + name string + oid uint32 + format int16 + value []byte + }{ + {"binary bytea", pgtype.ByteaOID, 1, encode(pgtype.ByteaOID, 1, []byte("a secret"))}, + {"hex bytea", pgtype.ByteaOID, 0, encode(pgtype.ByteaOID, 0, []byte("a secret"))}, + {"binary bytea array", pgtype.ByteaArrayOID, 1, encode(pgtype.ByteaArrayOID, 1, [][]byte{[]byte("x"), []byte("secret")})}, + {"text bytea array", pgtype.ByteaArrayOID, 0, encode(pgtype.ByteaArrayOID, 0, [][]byte{[]byte("secret")})}, + {"binary text array", pgtype.TextArrayOID, 1, encode(pgtype.TextArrayOID, 1, []string{"x", "secret"})}, + {"binary bigint array", pgtype.Int8ArrayOID, 1, encode(pgtype.Int8ArrayOID, 1, []int64{4111111111111111})}, + } { + t.Run(c.name, func(t *testing.T) { + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Execute{}, &pgproto3.Sync{}}, + []pgproto3.BackendMessage{rowDescription(c.format, c.oid), dataRow(c.value)}) + if value := forwarded[0].(*pgproto3.DataRow).Values[0]; value != nil { + t.Fatalf("forwarded %q, want NULL", value) + } + }) + } + + p := newMaskingProxy() + plain := encode(pgtype.ByteaOID, 0, []byte("nothing to hide")) + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Execute{}, &pgproto3.Sync{}}, + []pgproto3.BackendMessage{rowDescription(0, pgtype.ByteaOID), dataRow(plain)}) + if value := forwarded[0].(*pgproto3.DataRow).Values[0]; string(value) != string(plain) { + t.Fatalf("unmatched bytea forwarded as %q, want %q", value, plain) + } +} + +func TestMaskingChecksFunctionCallResults(t *testing.T) { + p := newMaskingProxy() + for _, result := range [][]byte{[]byte("a secret"), []byte("card 4111111111111111")} { + _, forwarded := relay(p, nil, []pgproto3.BackendMessage{&pgproto3.FunctionCallResponse{Result: result}}) + if got := forwarded[0].(*pgproto3.FunctionCallResponse).Result; got != nil { + t.Fatalf("function result %q forwarded as %q, want NULL", result, got) + } + } +} + +func TestMaskingSurvivesAMalformedBinaryArray(t *testing.T) { + header := make([]byte, 20) + binary.BigEndian.PutUint32(header[0:], 1) + binary.BigEndian.PutUint32(header[8:], pgtype.Int4OID) + binary.BigEndian.PutUint32(header[12:], 1) + binary.BigEndian.PutUint32(header[16:], 1) + + p := newMaskingProxy() + _, forwarded := relay(p, + []pgproto3.FrontendMessage{&pgproto3.Execute{}, &pgproto3.Sync{}}, + []pgproto3.BackendMessage{rowDescription(1, pgtype.Int4ArrayOID), dataRow(header)}) + if len(forwarded) != 1 { + t.Fatalf("forwarded %d messages, want the row", len(forwarded)) + } +} diff --git a/packages/pam/handlers/ssh/output_masking.go b/packages/pam/handlers/ssh/output_masking.go new file mode 100644 index 000000000..1ca2e34fb --- /dev/null +++ b/packages/pam/handlers/ssh/output_masking.go @@ -0,0 +1,160 @@ +package ssh + +import ( + "bytes" + "io" + "sort" + "time" + + "github.com/Infisical/infisical-merge/packages/pam/session/masking" +) + +func (p *SSHProxy) masksOutput() bool { + return len(p.config.OutputMaskingPatterns) > 0 +} + +const maskedOutputQuietPeriod = 25 * time.Millisecond + +const maxHeldOutput = 64 * 1024 + +// Masks a line at a time so a value split across reads still matches; a partial line waits until output goes quiet. +func (p *SSHProxy) proxyMaskedOutput(src io.Reader, dst io.Writer, sessionID string, chState *channelState) error { + defer p.flushOutputBuffer(sessionID, chState) + + type chunk struct { + data []byte + err error + } + reads := make(chan chunk) + done := make(chan struct{}) + defer close(done) + go func() { + for { + buf := make([]byte, 32*1024) + n, err := src.Read(buf) + select { + case reads <- chunk{buf[:n], err}: + case <-done: + return + } + if err != nil { + return + } + } + }() + + emit := func(data []byte) error { + if len(data) == 0 { + return nil + } + masked := maskTerminalOutput(p.config.OutputMaskingPatterns, data) + chState.mutex.Lock() + channelType := chState.channelType + chState.mutex.Unlock() + p.bufferOutput(masked, sessionID, channelType, chState) + _, err := dst.Write(masked) + return err + } + + var pending []byte + for { + var quiet <-chan time.Time + if len(pending) > 0 { + quiet = time.After(maskedOutputQuietPeriod) + } + select { + case c := <-reads: + if len(c.data) > 0 && p.config.OnActivity != nil { + p.config.OnActivity() + } + pending = append(pending, c.data...) + cut := bytes.LastIndexAny(pending, "\r\n") + 1 + if c.err != nil || len(pending)-cut > maxHeldOutput { + cut = len(pending) + } + if err := emit(pending[:cut]); err != nil { + return err + } + pending = append(pending[:0], pending[cut:]...) + if c.err == io.EOF { + return nil + } + if c.err != nil { + return c.err + } + case <-quiet: + if err := emit(pending); err != nil { + return err + } + pending = pending[:0] + } + } +} + +// Skips escape sequences when matching so color can't split a value, and keeps them after the placeholder. +func maskTerminalOutput(patterns []masking.Matcher, data []byte) []byte { + plain := make([]byte, 0, len(data)) + offsets := make([]int, 0, len(data)) + for i := 0; i < len(data); { + if data[i] == 0x1b { + i += escapeLength(data, i) + continue + } + plain = append(plain, data[i]) + offsets = append(offsets, i) + i++ + } + + var spans [][2]int + for _, pattern := range patterns { + for _, match := range pattern.FindAllIndex(plain, -1) { + if match[1] > match[0] { + spans = append(spans, [2]int{match[0], match[1]}) + } + } + } + if len(spans) == 0 { + return data + } + sort.Slice(spans, func(a, b int) bool { return spans[a][0] < spans[b][0] }) + + out := make([]byte, 0, len(data)) + last := 0 + for i := 0; i < len(spans); { + start, end := spans[i][0], spans[i][1] + for i++; i < len(spans) && spans[i][0] <= end; i++ { + end = max(end, spans[i][1]) + } + from, to := offsets[start], offsets[end-1]+1 + out = append(out, data[last:from]...) + out = append(out, masking.Placeholder...) + for j := from; j < to; { + if data[j] == 0x1b { + n := escapeLength(data, j) + out = append(out, data[j:j+n]...) + j += n + continue + } + j++ + } + last = to + } + return append(out, data[last:]...) +} + +// Only CSI sequences (color, cursor) are skipped whole, so a title's or link's text is still matched. +func escapeLength(data []byte, i int) int { + j := i + 1 + if j >= len(data) { + return 1 + } + if data[j] != '[' { + return 2 + } + for j++; j < len(data) && data[j] >= 0x20 && data[j] <= 0x3f; j++ { + } + if j < len(data) && data[j] >= 0x40 && data[j] <= 0x7e { + j++ + } + return j - i +} diff --git a/packages/pam/handlers/ssh/output_masking_test.go b/packages/pam/handlers/ssh/output_masking_test.go new file mode 100644 index 000000000..c2a23f8e0 --- /dev/null +++ b/packages/pam/handlers/ssh/output_masking_test.go @@ -0,0 +1,110 @@ +package ssh + +import ( + "bytes" + "io" + "regexp" + "sync" + "testing" + "time" + + "github.com/Infisical/infisical-merge/packages/pam/session/masking" +) + +type lockedBuffer struct { + mutex sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.buf.String() +} + +func newOutputMaskingProxy() *SSHProxy { + return NewSSHProxy(SSHProxyConfig{ + SessionID: "sid", + SessionLogger: &recordingLogger{}, + OutputMaskingPatterns: []masking.Matcher{ + regexp.MustCompile(`token=\w+`), + regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}`), + }, + }) +} + +func TestMaskedOutputMatchesAValueSplitAcrossReads(t *testing.T) { + src, w := io.Pipe() + out := &lockedBuffer{} + go func() { + _, _ = w.Write([]byte("value token=ab")) + time.Sleep(maskedOutputQuietPeriod / 5) + _, _ = w.Write([]byte("cd123 end\nprompt$ ")) + _ = w.Close() + }() + + if err := newOutputMaskingProxy().proxyData(src, out, "server→client", "sid", false, newChannelState()); err != nil { + t.Fatal(err) + } + if got := out.String(); got != "value [MASKED] end\nprompt$ " { + t.Fatalf("client saw %q", got) + } +} + +func TestMaskedOutputSendsAPartialLineOnceQuiet(t *testing.T) { + src, w := io.Pipe() + out := &lockedBuffer{} + done := make(chan error, 1) + go func() { + done <- newOutputMaskingProxy().proxyData(src, out, "server→client", "sid", false, newChannelState()) + }() + + _, _ = w.Write([]byte("prompt$ ")) + time.Sleep(4 * maskedOutputQuietPeriod) + if got := out.String(); got != "prompt$ " { + t.Fatalf("client saw %q before closing, want the prompt", got) + } + _ = w.Close() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestMaskedOutputMatchesThroughColorCodes(t *testing.T) { + line := "1,Jane,jane\x1b[01;31m\x1b[K@\x1b[m\x1b[Kacme.com,123\n" + src, w := io.Pipe() + out := &lockedBuffer{} + go func() { + _, _ = w.Write([]byte(line)) + _ = w.Close() + }() + + if err := newOutputMaskingProxy().proxyData(src, out, "server→client", "sid", false, newChannelState()); err != nil { + t.Fatal(err) + } + if got, want := out.String(), "1,Jane,[MASKED]\x1b[01;31m\x1b[K\x1b[m\x1b[K,123\n"; got != want { + t.Fatalf("client saw %q, want %q", got, want) + } +} + +func TestMaskedOutputMatchesInsideTerminalTitles(t *testing.T) { + src, w := io.Pipe() + out := &lockedBuffer{} + go func() { + _, _ = w.Write([]byte("\x1b]0;token=abcd123\x07\n")) + _ = w.Close() + }() + + if err := newOutputMaskingProxy().proxyData(src, out, "server→client", "sid", false, newChannelState()); err != nil { + t.Fatal(err) + } + if got, want := out.String(), "\x1b]0;[MASKED]\x07\n"; got != want { + t.Fatalf("client saw %q, want %q", got, want) + } +} diff --git a/packages/pam/handlers/ssh/proxy.go b/packages/pam/handlers/ssh/proxy.go index 4ead932f4..3a9325639 100644 --- a/packages/pam/handlers/ssh/proxy.go +++ b/packages/pam/handlers/ssh/proxy.go @@ -12,6 +12,7 @@ import ( "time" "github.com/Infisical/infisical-merge/packages/pam/session" + "github.com/Infisical/infisical-merge/packages/pam/session/masking" "github.com/Infisical/infisical-merge/packages/util" "github.com/rs/zerolog/log" "golang.org/x/crypto/ssh" @@ -27,8 +28,9 @@ type SSHProxyConfig struct { InjectCertificate string SessionID string SessionLogger session.SessionLogger - BlockedCommandPatterns []*regexp.Regexp // Regex patterns for command blocking (nil = no blocking) - OnActivity func() // Called when channel data flows + BlockedCommandPatterns []*regexp.Regexp // Regex patterns for command blocking (nil = no blocking) + OutputMaskingPatterns []masking.Matcher // Matchers for output masking (nil = no masking) + OnActivity func() // Called when channel data flows } // SSHProxy handles proxying SSH connections with credential injection @@ -255,6 +257,12 @@ func (p *SSHProxy) handleChannel(ctx context.Context, newChannel ssh.NewChannel, Str("channelType", channelType). Msg("← CLIENT new channel request") + // A forwarded connection carries another protocol, which masking would corrupt rather than redact. + if p.masksOutput() && channelType != "session" { + newChannel.Reject(ssh.Prohibited, "port forwarding is blocked because this account masks output") + return + } + // Open corresponding channel on server serverChannel, serverRequests, err := serverConn.OpenChannel(channelType, newChannel.ExtraData()) if err != nil { @@ -395,6 +403,10 @@ func (p *SSHProxy) handleChannelRequests(requests <-chan *ssh.Request, targetCha // Determine the type of operation isSCP := strings.HasPrefix(command, "scp ") + if isSCP && p.masksOutput() { + p.rejectUnmaskableTransfer(req, sessionID) + continue + } chState.mutex.Lock() if isSCP { // Mark this channel as binary so we don't log the raw file data @@ -460,6 +472,11 @@ func (p *SSHProxy) handleChannelRequests(requests <-chan *ssh.Request, targetCha Str("subsystem", subsystem). Msg("SSH subsystem requested") + if subsystem == "sftp" && p.masksOutput() { + p.rejectUnmaskableTransfer(req, sessionID) + continue + } + // Log SFTP sessions and set up SFTP parser for file operation logging if subsystem == "sftp" { chState.mutex.Lock() @@ -517,6 +534,10 @@ func (p *SSHProxy) handleChannelRequests(requests <-chan *ssh.Request, targetCha // proxyData proxies data between channels with optional logging func (p *SSHProxy) proxyData(src io.Reader, dst io.Writer, direction string, sessionID string, logInput bool, chState *channelState) error { + if !logInput && p.masksOutput() { + return p.proxyMaskedOutput(src, dst, sessionID, chState) + } + buf := make([]byte, 32*1024) // 32KB buffer // Flush any remaining buffers on exit @@ -738,6 +759,14 @@ func (p *SSHProxy) logOutputLines(sessionID string, chState *channelState, lines } } +// File transfers carry raw bytes that masking would corrupt rather than redact. +func (p *SSHProxy) rejectUnmaskableTransfer(req *ssh.Request, sessionID string) { + log.Warn().Str("sessionID", sessionID).Str("requestType", req.Type).Msg("Blocked file transfer on an account with output masking") + if req.WantReply { + req.Reply(false, nil) + } +} + // matchBlockedCommand checks if a command matches any blocked pattern. func (p *SSHProxy) matchBlockedCommand(command string) bool { command = strings.TrimSpace(command) diff --git a/packages/pam/pam-proxy.go b/packages/pam/pam-proxy.go index d2156b6ae..9f7de19db 100644 --- a/packages/pam/pam-proxy.go +++ b/packages/pam/pam-proxy.go @@ -271,6 +271,31 @@ func HandlePAMProxy(ctx context.Context, conn *tls.Conn, pamConfig *GatewayPAMCo ) } + var outputMatchers []masking.Matcher + var outputMasker masking.Masker + if credentials.PolicyRules != nil && credentials.PolicyRules.OutputMasking != nil { + rule := credentials.PolicyRules.OutputMasking + for i, pattern := range rule.Patterns { + re, err := regexp.Compile(pattern) + if err != nil { + // Fail closed, and keep the pattern out of the error since it can hold the value it masks. + return fmt.Errorf("output masking pattern %d does not compile", i+1) + } + outputMatchers = append(outputMatchers, re) + } + if rule.AccountCredentials { + for _, secret := range masking.CredentialSecrets(credentialValues(credentials)) { + outputMatchers = append(outputMatchers, regexp.MustCompile(regexp.QuoteMeta(secret))) + } + } + if rule.CreditCards { + outputMatchers = append(outputMatchers, masking.CardNumbers()) + } + if len(outputMatchers) > 0 { + outputMasker = masking.NewMatcherMasker(outputMatchers) + } + } + sessionLogger, err := session.NewSessionLogger(pamConfig.SessionId, encryptionKey, pamConfig.ExpiryTime, pamConfig.ResourceType, masker) if err != nil { return fmt.Errorf("failed to create session logger: %w", err) @@ -334,6 +359,7 @@ func HandlePAMProxy(ctx context.Context, conn *tls.Conn, pamConfig *GatewayPAMCo TLSConfig: tlsConfig, SessionID: pamConfig.SessionId, SessionLogger: sessionLogger, + OutputMasker: outputMasker, } proxy := handlers.NewPostgresProxy(proxyConfig) log.Info(). @@ -420,6 +446,7 @@ func HandlePAMProxy(ctx context.Context, conn *tls.Conn, pamConfig *GatewayPAMCo SessionID: pamConfig.SessionId, SessionLogger: sessionLogger, BlockedCommandPatterns: blockedCommandPatterns, + OutputMaskingPatterns: outputMatchers, OnActivity: pamConfig.OnActivity, } proxy := ssh.NewSSHProxy(sshConfig) diff --git a/packages/pam/session/masking/credentials.go b/packages/pam/session/masking/credentials.go index ae119aeab..0ff40e81c 100644 --- a/packages/pam/session/masking/credentials.go +++ b/packages/pam/session/masking/credentials.go @@ -56,6 +56,14 @@ func (m *credentialMasker) Mask(data []byte) []byte { return []byte(masked) } +// CredentialSecrets is what newCredentialMasker would redact, for callers that match with patterns. +func CredentialSecrets(values []string) []string { + if m := newCredentialMasker(values); m != nil { + return m.secrets + } + return nil +} + // newCredentialMasker returns nil when nothing is worth redacting, so callers can skip the stage. func newCredentialMasker(values []string) *credentialMasker { seen := make(map[string]struct{}, len(values)) diff --git a/packages/pam/session/masking/output.go b/packages/pam/session/masking/output.go new file mode 100644 index 000000000..7484c3de2 --- /dev/null +++ b/packages/pam/session/masking/output.go @@ -0,0 +1,90 @@ +package masking + +import ( + "slices" + "regexp" + "sort" +) + +// Matcher finds the spans to mask. *regexp.Regexp is one. +type Matcher interface { + FindAllIndex(b []byte, n int) [][]int +} + +var cardNumber = regexp.MustCompile(`\b(?:\d{4}[ -]?){3}\d{4}\b|\b3[47]\d{2}[ -]?\d{6}[ -]?\d{5}\b`) + +type cardMatcher struct{} + +// CardNumbers matches card-shaped numbers that pass the Luhn checksum, so other long digit runs stay visible. +func CardNumbers() Matcher { + return cardMatcher{} +} + +func (cardMatcher) FindAllIndex(b []byte, n int) [][]int { + var valid [][]int + for _, loc := range cardNumber.FindAllIndex(b, n) { + if luhnValid(b[loc[0]:loc[1]]) { + valid = append(valid, loc) + } + } + return valid +} + +func luhnValid(number []byte) bool { + sum, digits := 0, 0 + for _, n := range slices.Backward(number) { + if n < '0' || n > '9' { + continue + } + digit := int(n - '0') + if digits%2 == 1 { + if digit *= 2; digit > 9 { + digit -= 9 + } + } + sum += digit + digits++ + } + return sum%10 == 0 +} + +type matcherMasker struct { + matchers []Matcher +} + +// NewMatcherMasker masks every span any matcher finds, merging overlaps, so the order of matchers doesn't matter. +func NewMatcherMasker(matchers []Matcher) Masker { + return matcherMasker{matchers: matchers} +} + +func (m matcherMasker) Mask(data []byte) []byte { + var spans [][2]int + for _, matcher := range m.matchers { + for _, loc := range matcher.FindAllIndex(data, -1) { + if loc[1] > loc[0] { + spans = append(spans, [2]int{loc[0], loc[1]}) + } + } + } + if len(spans) == 0 { + return data + } + sort.Slice(spans, func(a, b int) bool { return spans[a][0] < spans[b][0] }) + + out := make([]byte, 0, len(data)) + last := 0 + for i := 0; i < len(spans); { + start, end := spans[i][0], spans[i][1] + for i++; i < len(spans) && spans[i][0] <= end; i++ { + end = max(end, spans[i][1]) + } + out = append(out, data[last:start]...) + out = append(out, Placeholder...) + last = end + } + return append(out, data[last:]...) +} + +func (m matcherMasker) MaskString(s string) string { + return string(m.Mask([]byte(s))) +} diff --git a/packages/pam/session/masking/output_test.go b/packages/pam/session/masking/output_test.go new file mode 100644 index 000000000..ba6dc9931 --- /dev/null +++ b/packages/pam/session/masking/output_test.go @@ -0,0 +1,31 @@ +package masking + +import ( + "regexp" + "testing" +) + +func TestCardNumbersNeedAValidChecksum(t *testing.T) { + masker := NewMatcherMasker([]Matcher{CardNumbers()}) + for input, want := range map[string]string{ + "card 4111 1111 1111 1111": "card [MASKED]", + "card 4111-1111-1111-1111": "card [MASKED]", + "amex 3782 822463 10005": "amex [MASKED]", + "order 4111111111111112": "order 4111111111111112", + "order 1234 5678 9012 3456": "order 1234 5678 9012 3456", + } { + if got := masker.MaskString(input); got != want { + t.Errorf("MaskString(%q) = %q, want %q", input, got, want) + } + } +} + +func TestMatcherMaskerMergesOverlappingMatches(t *testing.T) { + masker := NewMatcherMasker([]Matcher{ + regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s:/@]*:\S*@`), + regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`), + }) + if got := masker.MaskString("postgres://svc:hunter2@10.0.4.12:5432/app"); got != "[MASKED]:5432/app" { + t.Fatalf("got %q", got) + } +}