diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 62da49ee0..f20848eac 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -31,6 +31,7 @@ jobs: licenses: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.licenses == 'true' }} gh_credentials: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.gh_credentials == 'true' }} provider_update: ${{ github.event_name != 'pull_request' || steps.filter.outputs.full == 'true' || steps.filter.outputs.provider_update == 'true' }} + shared_logging: ${{ github.event_name != 'pull_request' || steps.filter.outputs.shared_logging == 'true' }} mode: ${{ github.event_name == 'pull_request' && 'pr' || 'full' }} changed_files: ${{ steps.filter.outputs.all_files || '[]' }} steps: @@ -62,6 +63,10 @@ jobs: - 'hack/licenses/**' - 'hack/podman_ci/**' - '.golangci.yaml' + shared_logging: + - 'logging/**' + - '.github/workflows/pr-ci.yml' + - '.golangci.yaml' agent_skill: - '.agents/skills/devsy/**' - 'hack/validate_agent_skill/**' @@ -171,6 +176,40 @@ jobs: - name: Validate skill and installer compatibility run: go run ./hack/validate_agent_skill + shared-logging: + name: Shared Logging + needs: [changes] + if: needs.changes.outputs.shared_logging == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + defaults: + run: + working-directory: logging + env: + GOWORK: "off" + GOTOOLCHAIN: local + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + go-version: 1.23.x + cache-dependency-path: logging/go.mod + - name: Test Go floor + run: go test -race ./... && go vet ./... && test -z "$(gofmt -l .)" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + go-version-file: go.mod + cache-dependency-path: logging/go.mod + - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a + with: + version: v2.14.0 + working-directory: logging + args: --config ../.golangci.yaml + - name: Check Go formatting + run: golangci-lint fmt --diff --config ../.golangci.yaml + lint: name: Lint needs: [changes] @@ -979,6 +1018,7 @@ jobs: - plan - precommit - lint + - shared-logging - agent-skill - unit-tests - github-credential-host-tests diff --git a/hack/ci/success.go b/hack/ci/success.go index c6b5370a5..f685007c2 100644 --- a/hack/ci/success.go +++ b/hack/ci/success.go @@ -193,6 +193,8 @@ func domainOutputForJob(job string) (string, bool) { return "gh_credentials", true case "provider-update-windows": return "provider_update", true + case "shared-logging": + return "shared_logging", true default: return "", false } diff --git a/hack/ci/success_test.go b/hack/ci/success_test.go index 56ffd586e..0d7983208 100644 --- a/hack/ci/success_test.go +++ b/hack/ci/success_test.go @@ -257,6 +257,7 @@ func (s *successSuite) TestPerDomainSkipsRequireExactFalseOutput() { {job: "licenses", field: "licenses"}, {job: "github-credential-host-tests", field: "gh_credentials"}, {job: "provider-update-windows", field: "provider_update"}, + {job: "shared-logging", field: "shared_logging"}, } for _, tt := range tests { s.Run(tt.job, func() { @@ -281,6 +282,12 @@ func (s *successSuite) TestPerDomainSkipsRequireExactFalseOutput() { } }) } + s.Run("shared-logging missing output", func() { + needs := needsObject(entry("changes", goTrue), entry("shared-logging", skippedJob)) + res := s.runSuccess(needs) + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::shared-logging ended with result=skipped unexpectedly\n") + }) } func (s *successSuite) TestMatrixSkipsUseTheirOwnPlanOutput() { diff --git a/logging/README.md b/logging/README.md new file mode 100644 index 000000000..04544c0bd --- /dev/null +++ b/logging/README.md @@ -0,0 +1,44 @@ +# Shared logging + +This module provides structured logging and credential masking for Go producers. It uses only the standard library and supports Go 1.23. It can be used independently of the Core and Runtime SDK modules. Use an approved merged commit or published submodule version when adding a dependency. Do not depend on an unpublished local checkout. + +## Configure a producer + +Inject the diagnostic writer, environment, and known secret values. The caller owns configuration errors, warnings, output streams, and logger lifetime. + +```go +import ( + "errors" + "io" + "log/slog" + + "github.com/devsy-org/devsy/logging" + "github.com/devsy-org/devsy/logging/redact" +) + +func newLogger(out io.Writer, level, format string, environment, secrets []string) (*slog.Logger, error) { + config, invalid := logging.ResolveProvider(level, format) + if invalid.Level || invalid.Format { + return nil, errors.New("invalid logging configuration") + } + redactor := redact.New(redact.Options{Environment: environment, Values: secrets}) + handler := logging.NewHandler(out, logging.HandlerOptions{ + Level: config.Level, + Format: config.Format, + Redactor: redactor, + }) + return slog.New(handler), nil +} +``` + +`ResolveProvider` returns info and JSON for empty or invalid settings and reports invalid fields separately. The caller decides whether to reject invalid settings or warn and use the fallback. Values are case insensitive and are not trimmed. Accepted formats are `auto`, `json`, `text`, `plain`, and `logfmt`. `auto` resolves to JSON and `plain` selects text. The caller chooses any terminal-specific format or presentation before constructing the handler. + +JSON records use `ts`, `level`, and `msg` with structured attributes. The handler writes only to the supplied writer. Send diagnostics to stderr or another dedicated sink. Keep stdout and transported command bytes reserved for their protocols. The handler does not configure process globals or terminate the process for fatal severity. + +## Protect diagnostic data + +The redactor masks supplied secret values, values from credential-like environment names, URL userinfo, and Authorization header credentials. It also processes diagnostic messages, keys, groups, errors, and nested values. Supply secrets the environment policy cannot identify. A nil handler redactor still protects recognized credential formats. Redaction cannot discover every secret. Avoid logging sensitive input and retain host-side protection where needed. + +`redact.NewStream` supports chunked diagnostic streams. Call `RedactChunk` for each chunk and `Flush` at the end. An unresolved generic URL authority has a 512-byte lookahead including its scheme. If it remains ambiguous beyond that limit, the stream conservatively masks the authority and suppresses it until `@`, a slash, `?`, `#`, or whitespace. This can mask an overlong ordinary authority split across calls. A complete ordinary URL remains unchanged when its delimiter makes it unambiguous in the supplied input. + +Known secrets across chunks use an append-backed pending buffer. Long unresolved match churn can defer safe output until later input or `Flush`. Complete unambiguous diagnostics are emitted immediately. There is no wall-clock emission bound. After each `RedactChunk`, pending byte length is bounded by `max(N + floor(N/2), max(N, 512) + 9)`, where `N` is the longest possible proper prefix of a known secret, or zero if there are none. Do not use this stream to rewrite protocol or raw business output. diff --git a/logging/config.go b/logging/config.go new file mode 100644 index 000000000..1d063f563 --- /dev/null +++ b/logging/config.go @@ -0,0 +1,113 @@ +package logging + +import ( + "log/slog" + "strings" +) + +const ( + LevelTrace slog.Level = -8 + LevelFatal slog.Level = 12 +) + +const ( + levelTraceName = "trace" + levelDebugName = "debug" + levelInfoName = "info" + levelWarnName = "warn" + levelErrorName = "error" +) + +type Format string + +const ( + FormatAuto Format = "auto" + FormatJSON Format = "json" + FormatText Format = "text" + FormatLogfmt Format = "logfmt" +) + +type ProviderConfig struct { + Level slog.Level + Format Format +} + +type InvalidConfig struct { + Level bool + Format bool +} + +func ParseLevel(value string) (slog.Level, bool) { + switch value { + case levelTraceName: + return LevelTrace, true + case levelDebugName: + return slog.LevelDebug, true + case levelInfoName: + return slog.LevelInfo, true + case levelWarnName: + return slog.LevelWarn, true + case levelErrorName: + return slog.LevelError, true + default: + return slog.LevelInfo, false + } +} + +func LevelNames() []string { + return []string{levelErrorName, levelWarnName, levelInfoName, levelDebugName, levelTraceName} +} + +func LevelName(level slog.Level) string { + switch { + case level < slog.LevelDebug: + return levelTraceName + case level < slog.LevelInfo: + return levelDebugName + case level < slog.LevelWarn: + return levelInfoName + case level < slog.LevelError: + return levelWarnName + case level < LevelFatal: + return levelErrorName + default: + return "fatal" + } +} + +func ParseFormat(value string) (Format, bool) { + switch Format(value) { + case FormatAuto, FormatJSON, FormatText, FormatLogfmt: + return Format(value), true + case "plain": + return FormatText, true + default: + return "", false + } +} + +func ResolveProvider(level, format string) (ProviderConfig, InvalidConfig) { + level = strings.ToLower(level) + format = strings.ToLower(format) + if level == "" { + level = levelInfoName + } + parsedLevel, levelOK := ParseLevel(level) + parsedFormat, formatOK := resolveProviderFormat(format) + return ProviderConfig{Level: parsedLevel, Format: parsedFormat}, + InvalidConfig{Level: !levelOK, Format: !formatOK} +} + +func resolveProviderFormat(value string) (Format, bool) { + if value == "" { + return FormatJSON, true + } + format, ok := ParseFormat(value) + if !ok { + return FormatJSON, false + } + if format == FormatAuto { + return FormatJSON, true + } + return format, true +} diff --git a/logging/go.mod b/logging/go.mod new file mode 100644 index 000000000..485eac383 --- /dev/null +++ b/logging/go.mod @@ -0,0 +1,3 @@ +module github.com/devsy-org/devsy/logging + +go 1.23.0 diff --git a/logging/handler.go b/logging/handler.go new file mode 100644 index 000000000..fd9986c76 --- /dev/null +++ b/logging/handler.go @@ -0,0 +1,448 @@ +package logging + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "strings" + "time" + + "github.com/devsy-org/devsy/logging/redact" +) + +type TextStyle int + +const ( + TextLowercase TextStyle = iota + TextUppercase + TextColored +) + +type ReservedKeys int + +const ( + DropReserved ReservedKeys = iota + EscapeReserved +) + +type HandlerOptions struct { + Level slog.Level + Format Format + TextStyle TextStyle + Redactor *redact.Redactor + ReservedKeys ReservedKeys + TrustedAttrs []slog.Attr +} + +const timestampLayout = "2006-01-02T15:04:05.000Z0700" + +type sanitizer struct { + redactor *redact.Redactor + policy ReservedKeys + reserved map[string]bool +} + +type handler struct { + sanitizer + inner slog.Handler + grouped bool + blocked bool +} + +func NewHandler(w io.Writer, options HandlerOptions) slog.Handler { + style := options.TextStyle + if options.Format == FormatJSON || options.Format == FormatLogfmt { + style = TextLowercase + } + if style == TextColored { + w = &colorWriter{writer: w} + } + opts := &slog.HandlerOptions{ + Level: options.Level, + ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + return replaceEnvelopeAttr(groups, attr, style) + }, + } + var inner slog.Handler + if options.Format == FormatJSON { + inner = slog.NewJSONHandler(w, opts) + } else { + inner = slog.NewTextHandler(w, opts) + } + return ProtectHandler(inner, options.Redactor, options.ReservedKeys, options.TrustedAttrs...) +} + +func replaceEnvelopeAttr(groups []string, attr slog.Attr, style TextStyle) slog.Attr { + if len(groups) != 0 { + return attr + } + switch attr.Key { + case slog.TimeKey: + return replaceTimeAttr(attr) + case slog.LevelKey: + return replaceLevelAttr(attr, style) + default: + return attr + } +} + +func replaceTimeAttr(attr slog.Attr) slog.Attr { + if attr.Value.Kind() == slog.KindTime { + return slog.String("ts", attr.Value.Time().Format(timestampLayout)) + } + return attr +} + +func replaceLevelAttr(attr slog.Attr, style TextStyle) slog.Attr { + level, ok := attr.Value.Any().(slog.Level) + if !ok { + return attr + } + name := LevelName(level) + if style != TextLowercase { + name = strings.ToUpper(name) + } + + return slog.String("level", name) +} + +type colorWriter struct { + writer io.Writer +} + +func (w *colorWriter) Write(p []byte) (int, error) { + token, ok := canonicalLevelToken(p) + if !ok { + return w.writer.Write(p) + } + name := string(p[token.start:token.end]) + colored := colorLevel(token.level, name) + output := make([]byte, 0, len(p)+len(colored)-len(name)) + output = append(output, p[:token.start]...) + output = append(output, colored...) + output = append(output, p[token.end:]...) + n, err := w.writer.Write(output) + return originalWriteCount(n, token.start, token.end, len(p)), err +} + +type levelToken struct { + start int + end int + level slog.Level +} + +func canonicalLevelToken(p []byte) (levelToken, bool) { + start := 0 + if bytes.HasPrefix(p, []byte("ts=")) { + space := bytes.IndexByte(p, ' ') + if space < 0 { + return levelToken{}, false + } + start = space + 1 + } + if !bytes.HasPrefix(p[start:], []byte("level=")) { + return levelToken{}, false + } + start += len("level=") + end := len(p) + if offset := bytes.IndexAny(p[start:], " \n"); offset >= 0 { + end = start + offset + } + level, ok := canonicalColorLevel(string(p[start:end])) + return levelToken{start: start, end: end, level: level}, ok +} + +func canonicalColorLevel(name string) (slog.Level, bool) { + switch name { + case "TRACE", "DEBUG", "INFO": + return slog.LevelInfo, true + case "WARN": + return slog.LevelWarn, true + case "ERROR", "FATAL": + return slog.LevelError, true + default: + return 0, false + } +} + +func originalWriteCount(n, start, end, inputLen int) int { + const prefix = len("\x1b[36m") + const suffix = len("\x1b[0m") + switch { + case n <= start: + return n + case n <= start+prefix: + return start + case n < end+prefix: + return n - prefix + case n <= end+prefix+suffix: + return end + case n >= inputLen+prefix+suffix: + return inputLen + default: + return n - prefix - suffix + } +} + +func newSanitizer( + redactor *redact.Redactor, + policy ReservedKeys, + trustedAttrs []slog.Attr, +) sanitizer { + if redactor == nil { + redactor = redact.New(redact.Options{}) + } + reserved := map[string]bool{ + "ts": true, + "time": true, + "level": true, + "msg": true, + "message": true, + } + reserveTrustedKeys(reserved, trustedAttrs) + return sanitizer{redactor: redactor, policy: policy, reserved: reserved} +} + +func reserveTrustedKeys(reserved map[string]bool, attrs []slog.Attr) { + for _, attr := range attrs { + if attr.Key != "" { + reserved[attr.Key] = true + } else if attr.Value.Kind() == slog.KindGroup { + reserveTrustedKeys(reserved, attr.Value.Group()) + } + } +} + +func resolveTrustedAttrs(attrs []slog.Attr) []slog.Attr { + resolved := make([]slog.Attr, 0, len(attrs)) + for _, attr := range attrs { + attr.Value = attr.Value.Resolve() + if attr.Value.Kind() == slog.KindGroup { + attr.Value = slog.GroupValue(resolveTrustedAttrs(attr.Value.Group())...) + } + resolved = append(resolved, attr) + } + return resolved +} + +func SanitizeAttrs(redactor *redact.Redactor, attrs []slog.Attr) []slog.Attr { + cleaner := newSanitizer(redactor, DropReserved, nil) + return cleaner.cleanAttrs(attrs, true) +} + +func ProtectHandler( + next slog.Handler, + redactor *redact.Redactor, + policy ReservedKeys, + trustedAttrs ...slog.Attr, +) slog.Handler { + trustedAttrs = resolveTrustedAttrs(trustedAttrs) + h := &handler{inner: next, sanitizer: newSanitizer(redactor, policy, trustedAttrs)} + if len(trustedAttrs) != 0 { + cleaned := make([]slog.Attr, 0, len(trustedAttrs)) + for _, attr := range trustedAttrs { + cleaned = append(cleaned, h.cleanTrusted(attr)) + } + h.inner = next.WithAttrs(cleaned) + } + return h +} + +func colorLevel(level slog.Level, name string) string { + color := "36" + switch { + case level >= slog.LevelError: + color = "31" + case level >= slog.LevelWarn: + color = "33" + } + return "\x1b[" + color + "m" + name + "\x1b[0m" +} + +func (h *handler) Enabled(ctx context.Context, level slog.Level) bool { + return h.inner.Enabled(ctx, level) +} + +func (h *handler) WithAttrs(attrs []slog.Attr) slog.Handler { + if h.blocked { + return h + } + cleaned := h.cleanAttrs(attrs, !h.grouped) + copyHandler := *h + copyHandler.inner = h.inner.WithAttrs(cleaned) + return ©Handler +} + +func (h *handler) WithGroup(name string) slog.Handler { + if name == "" || h.blocked { + return h + } + group, ok := h.clean(slog.Attr{Key: name, Value: slog.GroupValue()}, !h.grouped) + copyHandler := *h + if !ok { + copyHandler.blocked = true + } else if group.Key != "" { + copyHandler.inner = h.inner.WithGroup(group.Key) + copyHandler.grouped = true + } + return ©Handler +} + +func (h *handler) Handle(ctx context.Context, record slog.Record) error { + out := slog.NewRecord(record.Time, record.Level, h.redactor.Redact(record.Message), record.PC) + if !h.blocked { + record.Attrs(func(attr slog.Attr) bool { + if cleaned, ok := h.clean(attr, !h.grouped); ok { + out.AddAttrs(cleaned) + } + return true + }) + } + return h.inner.Handle(ctx, out) +} + +func (h *sanitizer) cleanTrusted(attr slog.Attr) slog.Attr { + if attr.Key == "" && attr.Value.Kind() == slog.KindGroup { + children := attr.Value.Group() + cleaned := make([]slog.Attr, 0, len(children)) + for _, child := range children { + cleaned = append(cleaned, h.cleanTrusted(child)) + } + return slog.Attr{Value: slog.GroupValue(cleaned...)} + } + key := attr.Key + cleaned, _ := h.clean(attr, false) + cleaned.Key = key + return cleaned +} + +func (h *sanitizer) cleanAttrs(attrs []slog.Attr, root bool) []slog.Attr { + cleaned := make([]slog.Attr, 0, len(attrs)) + for _, attr := range attrs { + if value, ok := h.clean(attr, root); ok { + cleaned = append(cleaned, value) + } + } + return cleaned +} + +func (h *sanitizer) clean(attr slog.Attr, root bool) (slog.Attr, bool) { + key := h.redactor.Redact(attr.Key) + if root && key != "" { + var ok bool + key, ok = h.rootKey(attr.Key, key) + if !ok { + return slog.Attr{}, false + } + } + attr.Value = attr.Value.Resolve() + if attr.Value.Kind() == slog.KindGroup { + children := attr.Value.Group() + cleaned := make([]slog.Attr, 0, len(children)) + for _, child := range children { + if value, ok := h.clean(child, root && key == ""); ok { + cleaned = append(cleaned, value) + } + } + return slog.Attr{Key: key, Value: slog.GroupValue(cleaned...)}, true + } + return slog.Attr{Key: key, Value: h.cleanValue(attr.Key, attr.Value)}, true +} + +func (h *sanitizer) rootKey(original, key string) (string, bool) { + count := h.reservedAncestors(key) + if !h.isReservedRoot(original, key, count) { + return key, true + } + if h.policy != EscapeReserved { + return "", false + } + if count == 0 { + count = 1 + } + for range count { + key = "user_" + key + } + for h.reserved[key] { + key = "user_" + key + } + return key, true +} + +func (h *sanitizer) isReservedRoot(original, key string, count int) bool { + return h.reserved[key] || h.reserved[original] || (h.policy == EscapeReserved && count > 0) +} + +func (h *sanitizer) reservedAncestors(key string) int { + count := 0 + for { + if h.reserved[key] { + count++ + } + if !strings.HasPrefix(key, "user_") { + return count + } + key = strings.TrimPrefix(key, "user_") + } +} + +func (h *sanitizer) cleanValue(key string, value slog.Value) slog.Value { + switch value.Kind() { + case slog.KindString: + return slog.StringValue(h.redactor.RedactField(key, value.String())) + case slog.KindAny: + return slog.AnyValue(h.cleanAny(key, value.Any())) + default: + return value + } +} + +func (h *sanitizer) cleanAny(key string, value any) (result any) { + defer func() { + if recover() != nil { + result = "[invalid value]" + } + }() + _, marshalsJSON := value.(json.Marshaler) + if err, ok := value.(error); ok && !marshalsJSON { + return h.redactor.RedactField(key, err.Error()) + } + data, err := json.Marshal(value) + if err != nil { + return "[unsupported value]" + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var decoded any + if decoder.Decode(&decoded) != nil { + return "[invalid value]" + } + return h.cleanJSON(key, decoded) +} + +func (h *sanitizer) cleanJSON(key string, value any) any { + switch v := value.(type) { + case string: + return h.redactor.RedactField(key, v) + case []any: + for i := range v { + v[i] = h.cleanJSON(key, v[i]) + } + case map[string]any: + out := make(map[string]any, len(v)) + for childKey, item := range v { + out[h.redactor.Redact(childKey)] = h.cleanJSON(childKey, item) + } + return out + } + return value +} + +func ParseTimestamp(value string) (time.Time, error) { + if parsed, err := time.Parse(timestampLayout, value); err == nil { + return parsed, nil + } + return time.Parse(time.RFC3339Nano, value) +} diff --git a/logging/handler_test.go b/logging/handler_test.go new file mode 100644 index 000000000..ac8f8778c --- /dev/null +++ b/logging/handler_test.go @@ -0,0 +1,526 @@ +package logging + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/logging/redact" +) + +const ( + testSecret = "secret-value" + nestedKey = "nested" + testDebug = "DEBUG" +) + +func TestProviderConfiguration(t *testing.T) { + for _, tc := range []struct { + name, level, format string + want ProviderConfig + invalid InvalidConfig + }{ + {"defaults", "", "", ProviderConfig{slog.LevelInfo, FormatJSON}, InvalidConfig{}}, + {"case and plain", testDebug, "PLAIN", ProviderConfig{slog.LevelDebug, FormatText}, InvalidConfig{}}, + {"auto", "INFO", "AUTO", ProviderConfig{slog.LevelInfo, FormatJSON}, InvalidConfig{}}, + {"logfmt", "trace", "LOGFMT", ProviderConfig{LevelTrace, FormatLogfmt}, InvalidConfig{}}, + {"json", "warn", "JSON", ProviderConfig{slog.LevelWarn, FormatJSON}, InvalidConfig{}}, + {"text", "error", "TEXT", ProviderConfig{slog.LevelError, FormatText}, InvalidConfig{}}, + { + "invalid level", " debug", "logfmt", + ProviderConfig{slog.LevelInfo, FormatLogfmt}, + InvalidConfig{Level: true}, + }, + { + "invalid format", testDebug, " text", + ProviderConfig{slog.LevelDebug, FormatJSON}, + InvalidConfig{Format: true}, + }, + { + "both invalid", "bad", "bad", + ProviderConfig{slog.LevelInfo, FormatJSON}, + InvalidConfig{Level: true, Format: true}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + config, invalid := ResolveProvider(tc.level, tc.format) + if config != tc.want || invalid != tc.invalid { + t.Fatalf("resolution = %+v %+v, want %+v %+v", config, invalid, tc.want, tc.invalid) + } + }) + } +} + +func TestHandlerRedactionAndReservedFields(t *testing.T) { + var output bytes.Buffer + r := redact.New(redact.Options{Values: []string{testSecret}}) + h := NewHandler( + &output, + HandlerOptions{ + Format: FormatJSON, + Redactor: r, + ReservedKeys: EscapeReserved, + TrustedAttrs: []slog.Attr{slog.String("provider", "trusted")}, + }, + ) + h = h.WithAttrs( + []slog.Attr{ + slog.Group("", slog.String(slog.LevelKey, "fake")), + slog.String("user_level", "second"), + }, + ) + if output.Len() != 0 { + t.Fatal("handler wrote before Handle") + } + record := slog.NewRecord( + time.Date(2024, 1, 2, 3, 4, 5, 6000000, time.UTC), + slog.LevelInfo, + testSecret, + 0, + ) + record.AddAttrs( + slog.String("provider", "forged"), + slog.Any("payload", map[string]any{testSecret: []any{testSecret}}), + ) + if err := h.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), testSecret) { + t.Fatalf("secret leaked: %s", output.String()) + } + var got map[string]any + if err := json.Unmarshal(output.Bytes(), &got); err != nil { + t.Fatal(err) + } + for key, want := range map[string]string{ + slog.LevelKey: "info", + "user_level": "fake", + "user_user_level": "second", + "provider": "trusted", + "user_provider": "forged", + "ts": "2024-01-02T03:04:05.006Z", + } { + if got[key] != want { + t.Errorf("%s = %v, want %q", key, got[key], want) + } + } +} + +func TestGroupAndTimestamp(t *testing.T) { + var output bytes.Buffer + h := NewHandler(&output, HandlerOptions{Format: FormatJSON}).WithGroup(slog.LevelKey) + r := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + r.AddAttrs(slog.String("key", "value")) + if err := h.Handle(context.Background(), r); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), `"key"`) { + t.Fatalf("reserved root group was not dropped: %s", output.String()) + } + for _, value := range []string{ + "2024-01-02T03:04:05.006Z", + "2024-01-02T03:04:05.123456789+01:00", + "2024-01-02T03:04:05.006+0100", + } { + if _, err := ParseTimestamp(value); err != nil { + t.Fatalf("timestamp %q: %v", value, err) + } + } +} + +func TestBoundAndRecordAttributesShareGroup(t *testing.T) { + var output bytes.Buffer + h := NewHandler(&output, HandlerOptions{Format: FormatJSON}).WithGroup("group") + h = h.WithAttrs([]slog.Attr{slog.String("first", "one")}) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + record.AddAttrs(slog.String("second", "two")) + if err := h.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + var got struct { + Group map[string]string `json:"group"` + } + if err := json.Unmarshal(output.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Group["first"] != "one" || got.Group["second"] != "two" { + t.Fatalf("group lost attributes: %s", output.String()) + } +} + +func TestAutoResolutionProducesJSON(t *testing.T) { + config, invalid := ResolveProvider("", "auto") + if invalid != (InvalidConfig{}) || config.Format != FormatJSON { + t.Fatalf("unexpected auto resolution: %+v %+v", config, invalid) + } + var output bytes.Buffer + handler := NewHandler(&output, HandlerOptions{Level: config.Level, Format: config.Format}) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(output.Bytes(), &got); err != nil || got["msg"] != "message" { + t.Fatalf("expected JSON record, got %q: %v", output.String(), err) + } +} + +func TestColorLevel(t *testing.T) { + for _, tc := range []struct { + level slog.Level + name string + want string + }{ + {LevelTrace, "TRACE", "\x1b[36mTRACE\x1b[0m"}, + {slog.LevelDebug, testDebug, "\x1b[36mDEBUG\x1b[0m"}, + {slog.LevelInfo, "INFO", "\x1b[36mINFO\x1b[0m"}, + {slog.LevelWarn, "WARN", "\x1b[33mWARN\x1b[0m"}, + {slog.LevelError, "ERROR", "\x1b[31mERROR\x1b[0m"}, + {LevelFatal, "FATAL", "\x1b[31mFATAL\x1b[0m"}, + } { + if got := colorLevel(tc.level, tc.name); got != tc.want { + t.Errorf("level %s: got %q, want %q", tc.name, got, tc.want) + } + } +} + +type countingValue struct { + calls *int + value map[string]any +} + +func (v countingValue) LogValue() slog.Value { + *v.calls++ + return slog.AnyValue(v.value) +} + +func TestWithAttrsSnapshotsOnceAcrossHandlers(t *testing.T) { + var output bytes.Buffer + calls := 0 + value := map[string]any{nestedKey: []any{"initial"}} + root := NewHandler(&output, HandlerOptions{Format: FormatJSON}) + bound := root.WithAttrs( + []slog.Attr{slog.Any("payload", countingValue{calls: &calls, value: value})}, + ) + if calls != 1 || output.Len() != 0 { + t.Fatalf("binding calls=%d output=%q", calls, output.String()) + } + value[nestedKey].([]any)[0] = "changed" + branches := []slog.Handler{bound, bound.WithGroup("group")} + for _, branch := range branches { + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + if err := branch.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + } + if calls != 1 || strings.Contains(output.String(), "changed") || + strings.Count(output.String(), "initial") != 2 { + t.Fatalf("binding was not a snapshot: calls=%d output=%s", calls, output.String()) + } +} + +func TestSanitizeAttrs(t *testing.T) { + input := []slog.Attr{ + slog.String(slog.LevelKey, "forged"), + slog.Group("", slog.String("msg", "forged"), slog.String("safe", testSecret)), + slog.Any("", map[string]any{nestedKey: []any{testSecret, int64(9007199254740993)}}), + } + redactor := redact.New(redact.Options{Values: []string{testSecret}}) + cleaned := SanitizeAttrs(redactor, input) + checkSanitizedAttrs(t, cleaned) + checkSanitizerInput(t, input) +} + +func checkSanitizedAttrs(t *testing.T, cleaned []slog.Attr) { + t.Helper() + if len(cleaned) != 2 { + t.Fatalf("unexpected sanitized attrs: %#v", cleaned) + } + if cleaned[0].Key != "" || cleaned[1].Key != "" { + t.Fatalf("unexpected sanitized keys: %#v", cleaned) + } + checkSanitizedGroup(t, cleaned[0].Value.Group()) + nested := cleaned[1].Value.Any().(map[string]any)[nestedKey].([]any) + if nested[0] != "***" || nested[1] != json.Number("9007199254740993") { + t.Fatalf("unnamed nested value was not preserved: %#v", nested) + } +} + +func checkSanitizedGroup(t *testing.T, group []slog.Attr) { + t.Helper() + if len(group) != 1 { + t.Fatalf("inline group was not sanitized: %#v", group) + } + if group[0].Key != "safe" || group[0].Value.String() != "***" { + t.Fatalf("inline group was not sanitized: %#v", group) + } +} + +func checkSanitizerInput(t *testing.T, input []slog.Attr) { + t.Helper() + original := input[2].Value.Any().(map[string]any)[nestedKey].([]any) + if input[0].Key != slog.LevelKey || len(input[1].Value.Group()) != 2 { + t.Fatalf("input attrs were mutated: %#v", input) + } + if original[0] != testSecret || original[1] != int64(9007199254740993) { + t.Fatalf("input value was mutated: %#v", input) + } +} + +func TestSanitizeAttrsNilRedactor(t *testing.T) { + cleaned := SanitizeAttrs( + nil, + []slog.Attr{slog.String("url", "https://user:password@example.com/path")}, + ) + if len(cleaned) != 1 || strings.Contains(cleaned[0].Value.String(), "password") { + t.Fatalf("credential was not masked: %#v", cleaned) + } +} + +func TestReservedValuesAreResolvedOnlyWhenRetained(t *testing.T) { + for _, tc := range []struct { + name string + key string + nested bool + policy ReservedKeys + want int + }{ + {name: "drop", key: slog.LevelKey, want: 0}, + {name: "accepted", key: "ordinary", want: 1}, + {name: nestedKey, key: slog.LevelKey, nested: true, want: 1}, + {name: "escape", key: slog.LevelKey, policy: EscapeReserved, want: 1}, + } { + for _, bound := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/bound=%t", tc.name, bound), func(t *testing.T) { + var output bytes.Buffer + calls := 0 + value := countingValue{calls: &calls, value: map[string]any{"value": "safe"}} + attr := slog.Any(tc.key, value) + if tc.nested { + attr = slog.Group("outer", attr) + } + handler := NewHandler(&output, HandlerOptions{ + Format: FormatJSON, ReservedKeys: tc.policy, + }) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + if bound { + handler = handler.WithAttrs([]slog.Attr{attr}) + } else { + record.AddAttrs(attr) + } + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + if calls != tc.want { + t.Fatalf("LogValue calls = %d, want %d", calls, tc.want) + } + }) + } + } +} + +func TestColoredTextWireOutput(t *testing.T) { + for _, tc := range []struct { + level slog.Level + want string + }{ + {LevelTrace, "level=\x1b[36mTRACE\x1b[0m"}, + {slog.LevelDebug, "level=\x1b[36mDEBUG\x1b[0m"}, + {slog.LevelInfo, "level=\x1b[36mINFO\x1b[0m"}, + {slog.LevelWarn, "level=\x1b[33mWARN\x1b[0m"}, + {slog.LevelError, "level=\x1b[31mERROR\x1b[0m"}, + {LevelFatal, "level=\x1b[31mFATAL\x1b[0m"}, + } { + for _, timestamp := range []time.Time{ + {}, + time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC), + } { + checkColoredText(t, tc.level, tc.want, timestamp) + } + } +} + +func checkColoredText(t *testing.T, level slog.Level, want string, timestamp time.Time) { + t.Helper() + var output bytes.Buffer + handler := NewHandler(&output, HandlerOptions{ + Level: LevelTrace, Format: FormatText, TextStyle: TextColored, + }) + record := slog.NewRecord(timestamp, level, "user \x1b[31m", 0) + record.AddAttrs(slog.String("value", "\x1b[32m")) + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + got := output.String() + if !strings.Contains(got, want) || strings.Count(got, "\x1b") != 2 { + t.Fatalf("unexpected terminal output %q", got) + } + if !strings.Contains(got, `\x1b[31m`) || !strings.Contains(got, `\x1b[32m`) { + t.Fatalf("user escapes were not quoted: %q", got) + } +} + +type limitedColorWriter struct { + limit int + err error +} + +func (w limitedColorWriter) Write(p []byte) (int, error) { + return min(w.limit, len(p)), w.err +} + +func TestColorWriterCountsAndErrors(t *testing.T) { + input := []byte("level=INFO msg=hello\n") + failure := fmt.Errorf("sink failure") + for _, tc := range []struct{ limit, want int }{ + {2, 2}, {8, 6}, {13, 8}, {17, 10}, {100, len(input)}, + } { + writer := colorWriter{writer: limitedColorWriter{limit: tc.limit, err: failure}} + n, err := writer.Write(input) + if n != tc.want || err != failure { + t.Fatalf("limit %d: count %d, error %v", tc.limit, n, err) + } + } +} + +type trustedGroupValue struct { + calls *int +} + +func (v trustedGroupValue) LogValue() slog.Value { + *v.calls++ + return slog.GroupValue(slog.String("provider", "trusted")) +} + +func TestInlineTrustedGroupsReserveRootKeys(t *testing.T) { + for _, policy := range []ReservedKeys{DropReserved, EscapeReserved} { + var output bytes.Buffer + calls := 0 + handler := NewHandler(&output, HandlerOptions{ + Format: FormatJSON, ReservedKeys: policy, + TrustedAttrs: []slog.Attr{ + slog.Group("", slog.Any("", trustedGroupValue{calls: &calls})), + slog.Group("scope", slog.String("scoped", "trusted")), + }, + }) + handler = handler.WithAttrs([]slog.Attr{slog.String("provider", "bound-forged")}) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + record.AddAttrs(slog.String("provider", "record-forged"), slog.String("scoped", "root")) + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + got := output.String() + checkTrustedIdentity(t, got, calls) + checkTrustedPolicy(t, got, policy) + } +} + +func checkTrustedIdentity(t *testing.T, got string, calls int) { + t.Helper() + if calls != 1 || strings.Count(got, `"provider":`) != 1 { + t.Fatalf("trusted key or resolution failed: calls=%d output=%s", calls, got) + } + if !strings.Contains(got, `"provider":"trusted"`) || !strings.Contains(got, `"scoped":"root"`) { + t.Fatalf("trusted or scoped attribute lost: %s", got) + } +} + +func checkTrustedPolicy(t *testing.T, got string, policy ReservedKeys) { + t.Helper() + if policy == DropReserved && strings.Contains(got, "forged") { + t.Fatalf("reserved attribute leaked: %s", got) + } + if policy == EscapeReserved && !strings.Contains(got, `"user_provider":"record-forged"`) { + t.Fatalf("reserved attribute was not escaped: %s", got) + } +} + +func TestStructuredAuthorizationRedaction(t *testing.T) { + var output bytes.Buffer + handler := NewHandler(&output, HandlerOptions{Format: FormatJSON}) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + record.AddAttrs( + slog.String("Authorization", " bAsIc\tfixture, safe"), + slog.Any("nested", map[string]any{"authorization": []any{"Bearer opaque", "Basic token"}}), + slog.Group("headers", slog.String("Authorization", "Bearer group-secret")), + ) + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + got := output.String() + for _, secret := range []string{"fixture", "opaque", "token", "group-secret"} { + if strings.Contains(got, secret) { + t.Fatalf("structured credential leaked: %s", got) + } + } + for _, expected := range []string{`bAsIc\t***, safe`, "Bearer ***", "Basic ***"} { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in %s", expected, got) + } + } +} + +func TestStructuredAuthorizationKnownOverlap(t *testing.T) { + var output bytes.Buffer + handler := NewHandler(&output, HandlerOptions{ + Format: FormatJSON, Redactor: redact.New(redact.Options{Values: []string{"Bearer"}}), + }) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + record.AddAttrs(slog.String("Authorization", "Bearer opaque")) + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), `"Authorization":"*** ***"`) { + t.Fatalf("overlapping redaction failed: %s", output.String()) + } +} + +type structuredDiagnosticError struct{} + +func (structuredDiagnosticError) Error() string { + return "fallback secret-value" +} + +func (structuredDiagnosticError) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]any{ + "code": "backend_failure", + "message": "failed secret-value", + "Authorization": "Bearer fresh-token", + }) +} + +func TestStructuredErrorJSONAndPlainErrorRedaction(t *testing.T) { + var output bytes.Buffer + handler := NewHandler(&output, HandlerOptions{ + Format: FormatJSON, Redactor: redact.New(redact.Options{Values: []string{testSecret}}), + }) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "message", 0) + record.AddAttrs( + slog.Any("structured", structuredDiagnosticError{}), + slog.Any("plain", fmt.Errorf("plain %s", testSecret)), + ) + if err := handler.Handle(context.Background(), record); err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(output.Bytes(), &got); err != nil { + t.Fatal(err) + } + structured, ok := got["structured"].(map[string]any) + if !ok { + t.Fatalf("structured error was not an object: %s", output.String()) + } + if structured["code"] != "backend_failure" || + structured["message"] != "failed ***" || + structured["Authorization"] != "Bearer ***" { + t.Fatalf("unexpected structured error: %s", output.String()) + } + if got["plain"] != "plain ***" { + t.Fatalf("plain error was not redacted: %s", output.String()) + } +} diff --git a/logging/redact/authorization.go b/logging/redact/authorization.go new file mode 100644 index 000000000..d4d4788fb --- /dev/null +++ b/logging/redact/authorization.go @@ -0,0 +1,197 @@ +package redact + +const ( + authName = iota + authAfterName + authBeforeScheme + authScheme + authLongS + authNeedSpace + authBeforeToken + authToken +) + +const authorizationName = "authorization" + +var authorizationFailure = [...]int{0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0} + +type authorizationState struct { + phase int + position int + namePosition int + scheme string +} + +func (s *authorizationState) restart(_ byte) { + s.phase = authName + s.position = 0 + s.scheme = "" +} + +func asciiLower(value byte) byte { + if value >= 'A' && value <= 'Z' { + return value + ('a' - 'A') + } + return value +} + +func (s *authorizationState) step(value byte) bool { + if s.phase != authToken && s.nameByte(value) { + s.phase = authAfterName + s.position = 0 + s.scheme = "" + return false + } + switch s.phase { + case authName: + case authAfterName: + s.separatorByte(value) + case authBeforeScheme: + s.schemeStart(value) + case authScheme: + s.schemeByte(value) + default: + return s.tokenByte(value) + } + return false +} + +func (s *authorizationState) nameByte(value byte) bool { + value = asciiLower(value) + for s.namePosition > 0 && authorizationName[s.namePosition] != value { + s.namePosition = authorizationFailure[s.namePosition-1] + } + if authorizationName[s.namePosition] == value { + s.namePosition++ + } + if s.namePosition != len(authorizationName) { + return false + } + s.namePosition = authorizationFailure[len(authorizationName)-1] + return true +} + +func (s *authorizationState) separatorByte(value byte) { + if formatSpace(value) { + return + } + if value == ':' || value == '=' { + s.phase = authBeforeScheme + } else { + s.restart(value) + } +} + +func (s *authorizationState) schemeStart(value byte) { + if formatSpace(value) { + return + } + if asciiLower(value) == 'b' { + s.phase = authScheme + s.position = 1 + } else { + s.restart(value) + } +} + +func (s *authorizationState) tokenByte(value byte) bool { + switch s.phase { + case authLongS: + s.longSByte(value) + case authNeedSpace: + if formatSpace(value) { + s.phase = authBeforeToken + } else { + s.restart(value) + } + case authBeforeToken: + return s.tokenStart(value) + case authToken: + if authorizationEnd(value) { + s.restart(value) + return false + } + return true + } + return false +} + +func (s *authorizationState) tokenStart(value byte) bool { + if formatSpace(value) { + return false + } + if value == ',' { + s.restart(value) + return false + } + s.phase = authToken + s.namePosition = 0 + return true +} + +func (s *authorizationState) longSByte(value byte) { + if value != 0xbf { + s.restart(value) + return + } + s.phase = authScheme + s.position++ + if s.position == len(s.scheme) { + s.phase = authNeedSpace + } +} + +func (s *authorizationState) schemeByte(value byte) { + if s.scheme == "" && !s.selectScheme(value) { + return + } + if s.scheme[s.position] == 's' && value == 0xc5 { + s.phase = authLongS + return + } + if asciiLower(value) != s.scheme[s.position] { + s.restart(value) + return + } + s.position++ + if s.position == len(s.scheme) { + s.phase = authNeedSpace + } +} + +func (s *authorizationState) selectScheme(value byte) bool { + switch asciiLower(value) { + case 'e': + s.scheme = "bearer" + case 'a': + s.scheme = "basic" + default: + s.restart(value) + return false + } + return true +} + +func scanAuthorization( + value string, + limit int, + initial authorizationState, +) [][]int { + state := initial + var matches [][]int + start := -1 + for i := range limit { + if state.step(value[i]) { + if start < 0 { + start = i + } + } else if start >= 0 { + matches = append(matches, []int{start, i, start, start}) + start = -1 + } + } + if start >= 0 { + matches = append(matches, []int{start, limit, start, start}) + } + return matches +} diff --git a/logging/redact/matcher.go b/logging/redact/matcher.go new file mode 100644 index 000000000..b419e223a --- /dev/null +++ b/logging/redact/matcher.go @@ -0,0 +1,116 @@ +package redact + +type byteMatcher struct { + forward *matchNode + reverse *matchNode +} + +type matchNode struct { + children map[byte]*matchNode + failure *matchNode + depth int + terminal int + longest int + proper int +} + +func newByteMatcher(values []string) *byteMatcher { + return &byteMatcher{forward: newAutomaton(values, false), reverse: newAutomaton(values, true)} +} + +func newAutomaton(values []string, reverse bool) *matchNode { + root := &matchNode{} + for _, value := range values { + insertSecret(root, value, reverse) + } + var queue []*matchNode + for _, child := range root.children { + child.failure = root + child.updateOutputs() + queue = append(queue, child) + } + for index := 0; index < len(queue); index++ { + current := queue[index] + for key, child := range current.children { + child.failure = transition(root, current.failure, key) + child.updateOutputs() + queue = append(queue, child) + } + } + return root +} + +func insertSecret(root *matchNode, value string, reverse bool) { + current := root + for i := range len(value) { + index := i + if reverse { + index = len(value) - 1 - i + } + key := value[index] + if current.children == nil { + current.children = make(map[byte]*matchNode) + } + next := current.children[key] + if next == nil { + next = &matchNode{depth: current.depth + 1} + current.children[key] = next + } + current = next + } + current.terminal = len(value) +} + +func (n *matchNode) updateOutputs() { + n.longest = max(n.terminal, n.failure.longest) + n.proper = n.failure.proper + if len(n.children) != 0 { + n.proper = n.depth + } +} + +func transition(root, state *matchNode, value byte) *matchNode { + for state != root && state.children[value] == nil { + state = state.failure + } + if next := state.children[value]; next != nil { + return next + } + return root +} + +func (r *Redactor) matchLengths(value string, cut int) []int { + if r == nil || r.matcher == nil || cut == 0 { + return nil + } + root := r.matcher.reverse + state := root + var lengths []int + for index := len(value) - 1; index >= 0; index-- { + state = transition(root, state, value[index]) + if index < cut && state.longest != 0 { + lengths = recordMatchLength(lengths, index, state.longest) + } + } + return lengths +} + +func recordMatchLength(lengths []int, index, length int) []int { + if lengths == nil { + lengths = make([]int, index+1) + } + lengths[index] = length + return lengths +} + +func (r *Redactor) secretPrefixBoundary(value string) int { + if r == nil || r.matcher == nil { + return len(value) + } + root := r.matcher.forward + state := root + for i := max(0, len(value)-r.maxLength+1); i < len(value); i++ { + state = transition(root, state, value[i]) + } + return len(value) - state.proper +} diff --git a/logging/redact/redact.go b/logging/redact/redact.go new file mode 100644 index 000000000..3cccb28d9 --- /dev/null +++ b/logging/redact/redact.go @@ -0,0 +1,267 @@ +package redact + +import ( + "regexp" + "strings" +) + +const ( + defaultMask = "***" + authorizationField = "Authorization" +) + +type Options struct { + Values []string + Environment []string + AdditionalEnvironmentKeys []string + Mask string + MinExplicitBytes int + SplitExplicitLines bool +} + +type Redactor struct { + values []string + matcher *byteMatcher + maxLength int + mask string +} + +var ( + credentialURL = regexp.MustCompile(`(?i)(https?://)[^\s/@?#]+@`) + authorization = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*(?:bearer|basic)\s+)[^\s,]+`) +) + +func New(opts Options) *Redactor { + mask := opts.Mask + if mask == "" { + mask = defaultMask + } + collector := valueCollector{ + values: make([]string, 0, len(opts.Values)+len(opts.Environment)), + seen: make(map[string]struct{}), + } + collector.explicit(opts.Values, max(opts.MinExplicitBytes, 1), opts.SplitExplicitLines) + collector.environment(opts.Environment, opts.AdditionalEnvironmentKeys) + return newRedactor(collector.values, mask) +} + +type valueCollector struct { + values []string + seen map[string]struct{} +} + +func (c *valueCollector) add(value string, minimum int) { + if len(value) < minimum { + return + } + if _, ok := c.seen[value]; ok { + return + } + c.seen[value] = struct{}{} + c.values = append(c.values, strings.Clone(value)) +} + +func (c *valueCollector) explicit(values []string, minimum int, splitLines bool) { + for _, value := range values { + c.add(value, minimum) + if splitLines { + for _, line := range strings.Split(value, "\n") { + c.add(line, minimum) + } + } + } +} + +func (c *valueCollector) environment(entries, additionalKeys []string) { + for _, entry := range entries { + key, value, ok := strings.Cut(entry, "=") + if ok && sensitiveKey(key, additionalKeys) { + c.add(value, 1) + } + } +} + +func FromEntries(entries []string) *Redactor { + values := make([]string, 0, len(entries)) + for _, entry := range entries { + _, value, ok := strings.Cut(entry, "=") + if ok && value != "" { + values = append(values, value) + } + } + return New(Options{Values: values}) +} + +func Combine(redactors ...*Redactor) *Redactor { + var values []string + for _, redactor := range redactors { + if redactor != nil { + values = append(values, redactor.values...) + } + } + return New(Options{Values: values}) +} + +func (r *Redactor) Redact(value string) string { + urls, headers := formatMatches(value, 0) + value, _, _ = r.known(value, matchState{cut: len(value), urls: urls, headers: headers}) + return value +} + +func (r *Redactor) RedactField(key, value string) string { + if !strings.EqualFold(key, authorizationField) { + return r.Redact(value) + } + urls, headers := formatMatches(value, 0) + fieldHeaders := scanAuthorization( + value, + len(value), + authorizationState{phase: authBeforeScheme}, + ) + headers = mergeHeaderMatches(headers, fieldHeaders) + value, _, _ = r.known(value, matchState{cut: len(value), urls: urls, headers: headers}) + return value +} + +func mergeHeaderMatches(first, second [][]int) [][]int { + if len(first) == 0 { + return second + } + if len(second) == 0 { + return first + } + result := make([][]int, 0, len(first)+len(second)) + i, j := 0, 0 + for i < len(first) && j < len(second) { + if first[i][3] <= second[j][3] { + result = append(result, first[i]) + i++ + } else { + result = append(result, second[j]) + j++ + } + } + result = append(result, first[i:]...) + return append(result, second[j:]...) +} + +func newRedactor(values []string, mask string) *Redactor { + r := &Redactor{values: values, mask: mask} + if len(values) == 0 { + return r + } + r.matcher = newByteMatcher(values) + for _, value := range values { + r.maxLength = max(r.maxLength, len(value)) + } + return r +} + +func sensitiveKey(key string, additional []string) bool { + key = strings.ToUpper(key) + for _, marker := range []string{ + "PASSWORD", "PASSWD", "TOKEN", "SECRET", "API_KEY", "APIKEY", + "AUTH", "CREDENTIAL", "PRIVATE_KEY", "ACCESS_KEY", + } { + if strings.Contains(key, marker) { + return true + } + } + for _, extra := range additional { + if strings.EqualFold(key, extra) { + return true + } + } + return false +} + +func (r *Redactor) maskText() string { + if r == nil || r.mask == "" { + return defaultMask + } + return r.mask +} + +type matchState struct { + cut int + covered int + markerEnd int + urls [][]int + headers [][]int +} + +type formatCursor struct { + matches [][]int + index int + trim int +} + +type maskWriter struct { + result strings.Builder + mask string + cut int + covered int + markerEnd int + last int + changed bool +} + +func (r *Redactor) known(value string, state matchState) (string, int, int) { + output := maskWriter{ + mask: r.maskText(), cut: state.cut, covered: state.covered, + markerEnd: state.markerEnd, changed: state.covered > 0, + } + if output.changed { + output.result.Grow(state.cut) + } + urls := formatCursor{matches: state.urls, trim: 1} + headers := formatCursor{matches: state.headers} + lengths := r.matchLengths(value, state.cut) + for i := range state.cut { + length := 0 + if i < len(lengths) { + length = lengths[i] + } + length = max(length, urls.matchLength(i), headers.matchLength(i)) + output.consume(value, i, length) + } + if !output.changed { + return value[:state.cut], 0, 0 + } + output.result.WriteString(value[output.last:state.cut]) + return output.result.String(), max( + 0, + output.covered-state.cut, + ), max( + 0, + output.markerEnd-state.cut, + ) +} + +func (w *maskWriter) consume(value string, index, length int) { + if length != 0 { + if index >= w.markerEnd { + if !w.changed { + w.result.Grow(w.cut) + w.changed = true + } + w.result.WriteString(value[w.last:index]) + w.result.WriteString(w.mask) + w.markerEnd = index + length + } + w.covered = max(w.covered, index+length) + } + if index < w.covered { + w.last = index + 1 + } +} + +func (c *formatCursor) matchLength(index int) int { + length := 0 + for c.index < len(c.matches) && c.matches[c.index][3] == index { + end := c.matches[c.index][1] - c.trim + length = max(length, end-index) + c.index++ + } + return length +} diff --git a/logging/redact/redact_test.go b/logging/redact/redact_test.go new file mode 100644 index 000000000..c3a17e56b --- /dev/null +++ b/logging/redact/redact_test.go @@ -0,0 +1,253 @@ +package redact + +import ( + "strings" + "testing" +) + +const ( + testShortSecret = "abc" + testSelfOverlap = "aba" + testOverlapReverse = "bab" + testCrossingSecret = "abc-secret-part" + testCrossingTail = "part-tail" + testRepeatedSecret = "abcabc" + testBearer = "Bearer" + testBearerValue = "Bearer fresh-token" + testOrdinary = "ordinary" + testMaskedBearer = testBearer + " " + testProviderMask + testProviderMask = "[REDACTED]" +) + +func testCredentialURL() string { + return strings.Join([]string{"https://user", "pass@example.com"}, ":") +} + +func TestRedactValuesAndOverlap(t *testing.T) { + tests := []struct { + name, input, want string + values []string + }{ + {"contained", "a secret b", "a *** b", []string{"sec", "secret"}}, + { + "crossing", + "safe abc-secret-part-tail visible", + "safe [REDACTED] visible", + []string{testShortSecret, testCrossingSecret, testCrossingTail}, + }, + {"self overlap", "x ababa y", "x *** y", []string{testSelfOverlap}}, + {"repeated", "abcabcabcabc", "******", []string{testRepeatedSecret}}, + {"binary", "a\x00\xffba\x00\xffb", "******", []string{"a\x00\xffb"}}, + {testOrdinary, "ordinary output", "ordinary output", []string{"private"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mask := "" + if test.name == "crossing" { + mask = testProviderMask + } + r := New(Options{Values: test.values, Mask: mask}) + if got := r.Redact(test.input); got != test.want { + t.Fatalf("Redact = %q, want %q", got, test.want) + } + }) + } +} + +func TestEnvironmentAndExplicitPolicy(t *testing.T) { + opts := Options{ + Values: []string{testShortSecret, "whole-secret\nline-secret\nxyz"}, + Environment: []string{ + "PATH=/usr/local/bin", "API_TOKEN=canary-token", "COMMAND=secret-command", "AUTH=q", + }, + AdditionalEnvironmentKeys: []string{"command"}, + Mask: testProviderMask, MinExplicitBytes: 4, SplitExplicitLines: true, + } + r := New(opts) + opts.Values[0] = testOrdinary + opts.Environment[1] = "API_TOKEN=changed" + input := "abc xyz whole-secret line-secret /usr/local/bin canary-token secret-command q" + want := "abc xyz [REDACTED] [REDACTED] /usr/local/bin [REDACTED] [REDACTED] [REDACTED]" + if got := r.Redact(input); got != want { + t.Fatalf("Redact = %q, want %q", got, want) + } +} + +func TestAdditionalEnvironmentKeysAreExact(t *testing.T) { + r := New(Options{ + Environment: []string{ + "command=hidden-command", + "COMMANDLINE=visible-commandline", + "PRIVATE_KEY=hidden-private", + "NORMAL=visible-normal", + }, + AdditionalEnvironmentKeys: []string{"COMMAND"}, + }) + input := "hidden-command visible-commandline hidden-private visible-normal" + want := "*** visible-commandline *** visible-normal" + if got := r.Redact(input); got != want { + t.Fatalf("environment Redact = %q, want %q", got, want) + } +} + +func TestEntriesAndCombine(t *testing.T) { + r := Combine(FromEntries([]string{"A=alpha=secret", "empty=", "invalid"}), nil, + New(Options{Values: []string{"beta-secret"}, Mask: testProviderMask})) + if got := r.Redact("alpha=secret beta-secret"); got != "*** ***" { + t.Fatalf("combined Redact = %q", got) + } +} + +func TestFormatCredentialsAndNil(t *testing.T) { + var r *Redactor + input := testCredentialURL() + " Authorization: Bearer api-token, Basic" + want := "https://***@example.com Authorization: Bearer ***, Basic" + if got := r.Redact(input); got != want { + t.Fatalf("nil Redact = %q, want %q", got, want) + } + long := "https://" + strings.Repeat("x", 1024) + "@example.com" + if got := r.Redact(long); got != "https://***@example.com" { + t.Fatalf("long complete URL leaked: %q", got) + } +} + +func TestRedactFieldAuthorization(t *testing.T) { + r := New(Options{Values: []string{"known-secret"}, Mask: testProviderMask}) + for _, test := range []struct { + key, value, want string + }{ + {authorizationField, testBearerValue, testMaskedBearer}, + {"authorization", " Basic\tfresh-token, safe", " Basic\t[REDACTED], safe"}, + {"AUTHORIZATION", "Bearer known-secret", testMaskedBearer}, + {authorizationField, "Bearer " + testCredentialURL(), testMaskedBearer}, + {authorizationField, "prefix Authorization: Basic fresh-token", "prefix Authorization: Basic [REDACTED]"}, + { + authorizationField, + "Bearer first, Authorization: Basic second", + "Bearer [REDACTED], Authorization: Basic [REDACTED]", + }, + {testOrdinary, "Authorization: Bearer fresh-token", "Authorization: Bearer [REDACTED]"}, + {"X-Authorization", testBearerValue, testBearerValue}, + {authorizationField, testBearer, testBearer}, + {authorizationField, "Bearer ", "Bearer "}, + } { + if got := r.RedactField(test.key, test.value); got != test.want { + t.Fatalf("key %q value %q: got %q, want %q", test.key, test.value, got, test.want) + } + } + var nilRedactor *Redactor + if got := nilRedactor.RedactField(authorizationField, "Basic fresh-token"); got != "Basic ***" { + t.Fatalf("nil field redaction = %q", got) + } +} + +func TestRedactFieldUsesOriginalSpans(t *testing.T) { + r := New(Options{Values: []string{testBearer}}) + if got := r.RedactField(authorizationField, testBearerValue); got != "*** ***" { + t.Fatalf("overlapping authorization field = %q", got) + } + if got := r.RedactField(testOrdinary, testBearerValue); got != "*** fresh-token" { + t.Fatalf("ordinary field = %q", got) + } + whole := New(Options{Values: []string{testBearerValue}}) + if got := whole.RedactField(authorizationField, testBearerValue); got != defaultMask { + t.Fatalf("whole value overlap = %q", got) + } +} + +func TestMatcherLongestAndProperSuffix(t *testing.T) { + tests := []struct { + values []string + input string + }{ + {[]string{"a", "aba", testOverlapReverse, "abcabc"}, "abababcabcab"}, + {[]string{"a\x00\xffb", "\x00\xff", "\xffb"}, "a\x00\xffba\x00\xff"}, + {[]string{"line\nsecret", "line"}, "safe line\nsec"}, + } + for _, test := range tests { + r := New(Options{Values: test.values}) + assertMatcherLengths(t, r, test.input, test.values) + assertMatcherBoundaries(t, r, test.input, test.values) + } +} + +func assertMatcherLengths(t *testing.T, r *Redactor, input string, values []string) { + t.Helper() + for cut := 0; cut <= len(input); cut++ { + lengths := r.matchLengths(input, cut) + for i := range cut { + got := 0 + if i < len(lengths) { + got = lengths[i] + } + if want := referenceLongestMatch(input[i:], values); got != want { + t.Fatalf("input %q cut %d index %d: got %d, want %d", input, cut, i, got, want) + } + } + } +} + +func assertMatcherBoundaries(t *testing.T, r *Redactor, value string, values []string) { + t.Helper() + for end := 0; end <= len(value); end++ { + input := value[:end] + want := len(input) + for i := range len(input) { + for _, secret := range values { + if len(input)-i < len(secret) && strings.HasPrefix(secret, input[i:]) { + want = min(want, i) + } + } + } + if got := r.secretPrefixBoundary(input); got != want { + t.Fatalf("input %q boundary = %d, want %d", input, got, want) + } + } +} + +func TestMatcherLongSharedPrefixAndNoMatchAllocations(t *testing.T) { + secret := strings.Repeat("a", 2048) + "b" + r := New(Options{Values: []string{secret}}) + prefix := strings.Repeat("a", 32768) + "c " + input := prefix + secret + " end" + if got, want := r.Redact(input), prefix+"*** end"; got != want { + t.Fatal("long shared-prefix redaction differed") + } + noMatch := strings.Repeat("a", 32768) + "c" + if got := r.matchLengths(noMatch, len(noMatch)); got != nil { + t.Fatal("no-match scan allocated a result") + } + if allocs := testing.AllocsPerRun(10, func() { + _ = r.matchLengths(noMatch, len(noMatch)) + }); allocs != 0 { + t.Fatalf("no-match matcher allocations = %v, want zero", allocs) + } + stream := NewStream(r) + split := len(input) - len(secret) + 1024 + got := stream.RedactChunk(input[:split]) + stream.RedactChunk(input[split:]) + stream.Flush() + if got != prefix+"*** end" { + t.Fatal("long shared-prefix stream differed") + } +} + +func BenchmarkRedactSharedPrefix(b *testing.B) { + for _, size := range []struct { + name string + n int + }{ + {name: "4KiB", n: 4096}, + {name: "128KiB", n: 131072}, + } { + b.Run(size.name, func(b *testing.B) { + prefix := strings.Repeat("a", size.n) + r := New(Options{Values: []string{prefix + "b"}}) + input := prefix + "c" + b.SetBytes(int64(len(input))) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + _ = r.Redact(input) + } + }) + } +} diff --git a/logging/redact/stream.go b/logging/redact/stream.go new file mode 100644 index 000000000..55d70cd6f --- /dev/null +++ b/logging/redact/stream.go @@ -0,0 +1,327 @@ +package redact + +import ( + "regexp" + "strings" + "sync" +) + +const formatCarryLimit = 512 + +var incompleteURL = regexp.MustCompile(`(?i)(https?://)[^\s/@?#]*\z`) + +type Stream struct { + mu sync.Mutex + base *Redactor + pending []byte + search *matchNode + lastMatch int + anchors int + covered int + markerEnd int + urlOpen bool + auth authorizationState +} + +func NewStream(redactor *Redactor) *Stream { + return &Stream{base: redactor} +} + +func (s *Stream) RedactChunk(chunk string) string { + if s == nil { + return chunk + } + s.mu.Lock() + defer s.mu.Unlock() + if s.base == nil { + return chunk + } + if chunk == "" { + return "" + } + return s.processChunk(chunk) +} + +func (s *Stream) Flush() string { + if s == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + if s.base == nil { + return "" + } + result := s.process(string(s.pending), true) + s.pending = nil + s.search = nil + s.lastMatch = 0 + s.anchors = 0 + s.covered = 0 + s.markerEnd = 0 + s.urlOpen = false + s.auth = authorizationState{} + return result +} + +func (s *Stream) processChunk(chunk string) string { + s.trackChunk(chunk) + if s.holdPrefix(chunk) { + return "" + } + if len(s.pending) == 0 { + return s.process(chunk, false) + } + if s.canEmitKnownSafe() { + if result, ok := s.emitKnownSafe(chunk); ok { + return result + } + } + if s.deferUnresolved(chunk) { + return "" + } + return s.process(string(s.pending)+chunk, false) +} + +func (s *Stream) process(value string, final bool) string { + frame := findURLFrame(value, s.urlOpen) + urls := frame.matches + headers := scanAuthorization(value, len(value), s.auth) + if frame.overflow { + urls = append(urls, []int{frame.start, len(value) + 1, frame.start, frame.authority}) + } + cut := s.streamCut(value, frame, urls, final) + s.advanceURL(frame) + authAtCut, headers := s.advanceAuthorization(value, cut, headers) + result, covered, markerEnd := s.base.known(value, matchState{ + cut: cut, covered: s.covered, markerEnd: s.markerEnd, urls: urls, headers: headers, + }) + s.retain(value[cut:]) + s.covered = covered + s.markerEnd = markerEnd + s.urlOpen = frame.activeAfter(cut, len(value), s.urlOpen) + s.auth = authAtCut + return result +} + +func (s *Stream) streamCut(value string, frame urlFrame, urls [][]int, final bool) int { + if final { + return len(value) + } + cut := s.base.secretPrefixBoundary(value) + if start := partialSchemeStart(value, frame.continuation); start >= 0 { + cut = min(cut, start) + } + if frame.start >= 0 && !frame.overflow { + cut = min(cut, frame.start) + } + return formatPrefixBoundary(cut, urls) +} + +func (s *Stream) advanceURL(frame urlFrame) { + if s.urlOpen { + s.covered = max(s.covered, frame.continuation) + s.markerEnd = max(s.markerEnd, frame.continuation) + } +} + +func (s *Stream) advanceAuthorization( + value string, + cut int, + headers [][]int, +) (authorizationState, [][]int) { + authAtCut := s.auth + for i := range cut { + authAtCut.step(value[i]) + if s.base.matcher != nil && (value[i] == ':' || value[i] == '=') { + s.anchors-- + } + } + s.lastMatch = max(0, s.lastMatch-cut) + if s.auth.phase == authToken && len(headers) != 0 && headers[0][0] == 0 { + continuation := headers[0][1] + s.covered = max(s.covered, continuation) + s.markerEnd = max(s.markerEnd, continuation) + headers = headers[1:] + } + return authAtCut, headers +} + +func (s *Stream) trackChunk(chunk string) { + if s.base.matcher == nil { + return + } + root := s.base.matcher.forward + if s.search == nil { + s.search = root + } + offset := len(s.pending) + for i := range len(chunk) { + if chunk[i] == ':' || chunk[i] == '=' { + s.anchors++ + } + s.search = transition(root, s.search, chunk[i]) + if s.search.longest != 0 { + s.lastMatch = offset + i + 1 + } + } +} + +func (s *Stream) canEmitKnownSafe() bool { + return s.noKnownMatchPending() && s.noFormatRisk() +} + +func (s *Stream) noKnownMatchPending() bool { + return s.search != nil && s.lastMatch == 0 && s.covered == 0 && s.markerEnd == 0 +} + +func (s *Stream) noFormatRisk() bool { + return s.anchors == 0 && !s.urlOpen && s.auth.phase <= authAfterName +} + +func (s *Stream) emitKnownSafe(chunk string) (string, bool) { + size := len(s.pending) + len(chunk) + if s.search.proper > size { + return "", false + } + s.pending = append(s.pending, chunk...) + cut := len(s.pending) - s.search.proper + tailStart := max(0, len(s.pending)-len("http\u017f://")+1) + if start := partialSchemeStart(string(s.pending[tailStart:]), 0); start >= 0 { + cut = min(cut, tailStart+start) + } + for _, value := range s.pending[:cut] { + s.auth.step(value) + } + result := string(s.pending[:cut]) + s.pending = s.pending[cut:] + if len(s.pending) == 0 { + s.pending = nil + } + if cap(s.pending) > 1024 && len(s.pending) < cap(s.pending)/4 { + s.pending = append([]byte(nil), s.pending...) + } + return result, true +} + +func (s *Stream) deferUnresolved(chunk string) bool { + if s.search == nil || s.search.proper < 4*formatCarryLimit { + return false + } + tail := s.search.proper + size := len(s.pending) + len(chunk) + if tail > size || size-tail >= tail/2 || len(chunk) >= tail/2 { + return false + } + s.pending = append(s.pending, chunk...) + return true +} + +func (s *Stream) holdPrefix(chunk string) bool { + if s.search == nil || s.search.proper != len(s.pending)+len(chunk) { + return false + } + s.pending = append(s.pending, chunk...) + return true +} + +func (s *Stream) retain(value string) { + if value == "" { + s.pending = nil + return + } + if cap(s.pending) > 1024 && len(value) < cap(s.pending)/4 { + s.pending = nil + } + s.pending = append(s.pending[:0], value...) +} + +func formatMatches(value string, offset int) ([][]int, [][]int) { + urls := credentialURL.FindAllStringSubmatchIndex(value[offset:], -1) + headers := authorization.FindAllStringSubmatchIndex(value[offset:], -1) + for _, matches := range [][][]int{urls, headers} { + for _, match := range matches { + for i := range match { + match[i] += offset + } + } + } + return urls, headers +} + +func formatPrefixBoundary(cut int, groups ...[][]int) int { + for { + previous := cut + for _, matches := range groups { + for _, match := range matches { + if match[0] < cut && cut <= match[3] { + cut = match[0] + } + } + } + if cut == previous { + return cut + } + } +} + +func authorizationEnd(value byte) bool { + return value == ',' || formatSpace(value) +} + +func formatSpace(value byte) bool { + return value == ' ' || value == '\t' || value == '\n' || value == '\r' || value == '\f' +} + +type urlFrame struct { + matches [][]int + continuation int + start int + authority int + overflow bool +} + +func findURLFrame(value string, active bool) urlFrame { + frame := urlFrame{start: -1} + if active { + for frame.continuation < len(value) && !urlAuthorityEnd(value[frame.continuation]) { + frame.continuation++ + } + } + offset := frame.continuation + frame.matches = credentialURL.FindAllStringSubmatchIndex(value[offset:], -1) + for _, match := range frame.matches { + for i := range match { + match[i] += offset + } + } + if match := incompleteURL.FindStringSubmatchIndex(value[offset:]); match != nil { + frame.start = offset + match[0] + frame.authority = offset + match[3] + frame.overflow = len(value)-frame.start > formatCarryLimit + } + return frame +} + +func (f urlFrame) activeAfter(cut, length int, active bool) bool { + if f.overflow && cut > f.authority { + return true + } + return active && (cut < f.continuation || (cut == f.continuation && f.continuation == length)) +} + +func urlAuthorityEnd(value byte) bool { + return value == '@' || value == '/' || value == '?' || value == '#' || formatSpace(value) +} + +func partialSchemeStart(value string, skip int) int { + start := max(skip, len(value)-len("http\u017f://")+1) + for i := start; i < len(value); i++ { + tail := value[i:] + for _, scheme := range []string{"http://", "https://", "http\u017f://"} { + if len(tail) < len(scheme) && strings.EqualFold(tail, scheme[:len(tail)]) { + return i + } + } + } + return -1 +} diff --git a/logging/redact/stream_test.go b/logging/redact/stream_test.go new file mode 100644 index 000000000..c2c25427e --- /dev/null +++ b/logging/redact/stream_test.go @@ -0,0 +1,751 @@ +package redact + +import ( + "strings" + "testing" +) + +const ( + testURLScheme = "https://" + testURLUser = "user:" + testURLSuffix = "@example.test/x" + testFormatPart = "http:" + testDenseToken = "xx" +) + +const ( + benchComplete = "complete" + benchMismatch = "mismatch" + benchFlush = "flush" + benchSliding = "sliding" + benchDense = "dense" + benchFormat = "format" + benchURL = "url" + benchAuthURL = "auth_url" +) + +func TestStreamKnownSecretEverySplit(t *testing.T) { + tests := []struct { + values []string + input string + }{ + {[]string{testRepeatedSecret}, "ordinary abcabc abcabc done"}, + {[]string{"aaaaa"}, "ordinary aaaaa done"}, + {[]string{"abababa"}, "ordinary abababa done"}, + {[]string{"abcabcX", testRepeatedSecret}, "ordinary abcabcX abcabc done"}, + {[]string{testSelfOverlap, testOverlapReverse}, "ordinary abab done"}, + { + []string{testShortSecret, testCrossingSecret, testCrossingTail}, + "safe abc-secret-part-tail visible", + }, + {[]string{"abc\x00\xffabc"}, "ordinary abc\x00\xffabc done"}, + {[]string{"secret"}, "ordinary no credentials here"}, + } + for _, test := range tests { + r := New(Options{Values: test.values}) + assertAllStreamCuts(t, r, test.input, r.Redact(test.input)) + } +} + +func TestStreamEmitsCompleteDiagnosticAndRetainsAmbiguousSuffix(t *testing.T) { + r := New(Options{Values: []string{"token", "token12345", "top\nsecret"}}) + stream := NewStream(r) + diagnostic := "{\"level\":\"info\",\"msg\":\"safe\"}\n" + if got := stream.RedactChunk(diagnostic); got != diagnostic { + t.Fatalf("complete diagnostic = %q, want %q", got, diagnostic) + } + if got := stream.RedactChunk("prefix token"); got != "prefix " { + t.Fatalf("ambiguous prefix output = %q", got) + } + if got := stream.RedactChunk("12345 end\n"); got != "*** end\n" { + t.Fatalf("resolved credential output = %q", got) + } + if got := stream.Flush(); got != "" { + t.Fatalf("unexpected final output = %q", got) + } +} + +func TestStreamPromptDiagnosticWithLongKnownSecret(t *testing.T) { + secret := strings.Repeat("x", 65536) + "Z" + stream := NewStream(New(Options{Values: []string{secret}})) + diagnostic := "{\"level\":\"info\",\"msg\":\"safe\"}\n" + if got := stream.RedactChunk(diagnostic); got != diagnostic { + t.Fatalf("complete diagnostic = %q, want %q", got, diagnostic) + } + if got := stream.Flush(); got != "" { + t.Fatalf("unexpected final output = %q", got) + } +} + +func TestStreamRetainsMultilineSecretAcrossNewline(t *testing.T) { + stream := NewStream(New(Options{Values: []string{"top\nsecret"}})) + if got := stream.RedactChunk("top\n"); got != "" { + t.Fatalf("multiline prefix leaked = %q", got) + } + if got := stream.RedactChunk("secret end\n"); got != "*** end\n" { + t.Fatalf("multiline output = %q", got) + } + if got := stream.Flush(); got != "" { + t.Fatalf("unexpected final output = %q", got) + } +} + +func TestStreamExhaustiveIntersectingMatches(t *testing.T) { + values := []string{testSelfOverlap, testOverlapReverse, "bb"} + r := New(Options{Values: values}) + for length := range 9 { + for bits := range 1 << length { + var input strings.Builder + for i := range length { + if bits&(1<= markerEnd { + markers[i] = true + markerEnd = i + longest + } + } + var output strings.Builder + for i := range len(input) { + if markers[i] { + output.WriteString(defaultMask) + } + if !covered[i] { + output.WriteByte(input[i]) + } + } + return output.String() +} + +func referenceLongestMatch(input string, secrets []string) int { + longest := 0 + for _, secret := range secrets { + if strings.HasPrefix(input, secret) && len(secret) > longest { + longest = len(secret) + } + } + return longest +} + +func TestStreamFormatCredentialsEverySplit(t *testing.T) { + r := New(Options{}) + for _, test := range []struct{ input, want string }{ + {"clone https://user:pass@example.com/repo done", "clone https://***@example.com/repo done"}, + {"Authorization: Bearer api-token, safe", "Authorization: Bearer ***, safe"}, + {"before authorization = Basic\tcredential after", "before authorization = Basic\t*** after"}, + {"Authorization: Ba\u017fic opaque-token, safe", "Authorization: Ba\u017fic ***, safe"}, + {"clone http\u017f://user:pass@example.com/repo done", "clone http\u017f://***@example.com/repo done"}, + {"Authorization: Bearer a", "Authorization: Bearer ***"}, + {"https://example.com/path and ordinary output", "https://example.com/path and ordinary output"}, + { + "first Authorization: Bearer token next https://user:pass@example.com/end", + "first Authorization: Bearer *** next https://***@example.com/end", + }, + } { + if got := r.Redact(test.input); got != test.want { + t.Fatalf("complete %q: got %q, want %q", test.input, got, test.want) + } + assertAllStreamCuts(t, r, test.input, test.want) + } +} + +func TestStreamAuthorizationRecoversOverlappingNames(t *testing.T) { + r := New(Options{}) + for _, test := range []struct{ input, want string }{ + { + "authorizAuthorization: Bearer canary-token, safe", + "authorizAuthorization: Bearer ***, safe", + }, + { + "Authorization: BAuthorization: Basic nested-token, safe", + "Authorization: BAuthorization: Basic ***, safe", + }, + { + "Authorization: BaAuthorization: Bearer nested-token, safe", + "Authorization: BaAuthorization: Bearer ***, safe", + }, + { + "Authorization Authorization: Bearer another-token, safe", + "Authorization Authorization: Bearer ***, safe", + }, + { + "Authorization: Bearer abcAuthorization:def, safe", + "Authorization: Bearer ***, safe", + }, + { + "Authorization: Bearer abcAuthorization:def, Authorization: Basic other-token", + "Authorization: Bearer ***, Authorization: Basic ***", + }, + } { + if got := r.Redact(test.input); got != test.want { + t.Fatalf("complete %q = %q, want %q", test.input, got, test.want) + } + assertAllStreamCuts(t, r, test.input, test.want) + } +} + +func TestStreamCustomMaskAndIntersectingSecrets(t *testing.T) { + r := New(Options{ + Values: []string{testShortSecret, testCrossingSecret, testCrossingTail}, + Mask: testProviderMask, + }) + input := "safe abc-secret-part-tail Authorization: Bearer api-token, visible" + want := "safe [REDACTED] Authorization: Bearer [REDACTED], visible" + if got := r.Redact(input); got != want { + t.Fatalf("complete redaction = %q, want %q", got, want) + } + assertAllStreamCuts(t, r, input, want) +} + +func TestCredentialRecognitionUsesOriginalInput(t *testing.T) { + tests := []struct { + name, secret, input, want string + }{ + { + "authorization scheme", + "Bearer", + "before Authorization: Bearer unknown-token after", + "before Authorization: *** *** after", + }, + { + "authorization spanning", + "Bearer unknown-token", + "before Authorization: Bearer unknown-token after", + "before Authorization: *** after", + }, + { + "url scheme", + testURLScheme, + "before https://user:unknown-password@example.com after", + "before ******@example.com after", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := New(Options{Values: []string{test.secret}}) + if got := r.Redact(test.input); got != test.want { + t.Fatalf("complete = %q, want %q", got, test.want) + } + assertAllStreamCuts(t, r, test.input, test.want) + }) + } +} + +func assertAllStreamCuts(t *testing.T, r *Redactor, input, want string) { + t.Helper() + for split := 0; split <= len(input); split++ { + stream := NewStream(r) + got := stream.RedactChunk( + input[:split], + ) + stream.RedactChunk( + input[split:], + ) + stream.Flush() + if got != want { + t.Fatalf("input %q split %d: got %q, want %q", input, split, got, want) + } + } + stream := NewStream(r) + var output strings.Builder + for i := range len(input) { + output.WriteString(stream.RedactChunk(input[i : i+1])) + } + output.WriteString(stream.Flush()) + if got := output.String(); got != want { + t.Fatalf("input %q byte chunks: got %q, want %q", input, got, want) + } +} + +func TestStreamBoundedRepeatedKnownSecret(t *testing.T) { + const count = 1 << 14 + input := strings.Repeat(testRepeatedSecret, count) + r := New(Options{Values: []string{testRepeatedSecret}}) + stream := NewStream(r) + emitted := stream.RedactChunk(input) + if len(emitted) < 3*(count-1) { + t.Fatalf("repeated output retained: emitted %d bytes", len(emitted)) + } + if len(stream.pending) > max(r.maxLength-1, formatCarryLimit) { + t.Fatalf("unbounded carry: %d", len(stream.pending)) + } + if got, want := emitted+stream.Flush(), r.Redact(input); got != want { + t.Fatal("stream and complete redaction differ") + } +} + +func TestStreamLongAuthorizationAndOrdinaryURL(t *testing.T) { + r := New(Options{}) + stream := NewStream(r) + var output strings.Builder + output.WriteString(stream.RedactChunk("before Authorization: Bearer ")) + for range 1024 { + output.WriteString(stream.RedactChunk("secret-token")) + if len(stream.pending) > formatCarryLimit { + t.Fatal("authorization carry grew") + } + } + output.WriteString(stream.RedactChunk(", after")) + output.WriteString(stream.Flush()) + if got, want := output.String(), "before Authorization: Bearer ***, after"; got != want { + t.Fatalf("authorization = %q, want %q", got, want) + } + ordinary := testURLScheme + strings.Repeat("a", 1024) + "/path" + stream = NewStream(r) + got := stream.RedactChunk(ordinary[:600]) + stream.RedactChunk(ordinary[600:]) + stream.Flush() + if got != "https://***/path" { + t.Fatalf("overflowed ordinary URL = %q", got) + } + stream = NewStream(r) + if got := stream.RedactChunk(ordinary) + stream.Flush(); got != ordinary { + t.Fatal("fully resolved ordinary URL changed") + } +} + +func TestStreamURLOverflowMasksUntilDelimiter(t *testing.T) { + scheme := testURLScheme + authority := testURLUser + strings.Repeat("x", 600) + tests := []struct { + name, tail, want string + }{ + {"userinfo", testURLSuffix, scheme + "***@example.test/x"}, + {"path", "/path", scheme + "***/path"}, + {"query", "?u=@example.test/x", scheme + "***?u=@example.test/x"}, + {"fragment", "#u=@example.test/x", scheme + "***#u=@example.test/x"}, + {"space", " done", scheme + "*** done"}, + {"eof", "", scheme + defaultMask}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + stream := NewStream(New(Options{Values: []string{testURLUser}})) + first := stream.RedactChunk(scheme + authority) + got := first + stream.RedactChunk(test.tail) + stream.Flush() + if got != test.want { + t.Fatalf("got %q, want %q", got, test.want) + } + if strings.Contains(got, authority) || strings.Count(got, defaultMask) != 1 { + t.Fatalf("unmasked authority or duplicate marker: %q", got) + } + }) + } +} + +func TestStreamURLOverflowChunkBoundaries(t *testing.T) { + input := testURLScheme + testURLUser + strings.Repeat("x", 1100) + testURLSuffix + want := "https://***@example.test/x" + for _, chunkSize := range []int{1, 7, 128, 511, 512, 513, len(input)} { + stream := NewStream(New(Options{})) + var output strings.Builder + for i := 0; i < len(input); i += chunkSize { + output.WriteString(stream.RedactChunk(input[i:min(i+chunkSize, len(input))])) + if len(stream.pending) > formatCarryLimit { + t.Fatalf("chunk %d retained %d bytes", chunkSize, len(stream.pending)) + } + } + output.WriteString(stream.Flush()) + if got := output.String(); got != want { + t.Fatalf("chunk %d: got %q, want %q", chunkSize, got, want) + } + } +} + +func TestStreamURLQueryAndFragmentBoundaries(t *testing.T) { + r := New(Options{}) + for _, suffix := range []string{"?next=@example.test/x", "#next=@example.test/x"} { + short := testURLScheme + "example.test" + suffix + if got := r.Redact(short); got != short { + t.Fatalf("complete ordinary URL = %q, want %q", got, short) + } + assertAllStreamCuts(t, r, short, short) + long := testURLScheme + strings.Repeat("a", 600) + suffix + "\n" + stream := NewStream(r) + if got := stream.RedactChunk(long); got != long { + t.Fatalf( + "resolved long ordinary URL changed: got %d bytes, want %d", + len(got), + len(long), + ) + } + if got := stream.Flush(); got != "" { + t.Fatalf("unexpected trailing output = %q", got) + } + } +} + +func TestStreamEncodedURLUserinfoAcrossSplits(t *testing.T) { + r := New(Options{}) + input := testURLScheme + "user%3Fpart%23token@example.test/x" + want := testURLScheme + "***@example.test/x" + if got := r.Redact(input); got != want { + t.Fatalf("complete encoded userinfo = %q, want %q", got, want) + } + assertAllStreamCuts(t, r, input, want) + stream := NewStream(r) + first := testURLScheme + "user%3F" + strings.Repeat("x", 600) + got := stream.RedactChunk( + first, + ) + stream.RedactChunk( + "%23suffix@example.test/x", + ) + stream.Flush() + if got != want { + t.Fatalf("overflow encoded userinfo = %q, want %q", got, want) + } +} + +func TestStreamURLLookaheadThreshold(t *testing.T) { + scheme := testURLScheme + short := scheme + strings.Repeat("x", formatCarryLimit-len(scheme)) + stream := NewStream(New(Options{})) + if got := stream.RedactChunk(short); got != "" { + t.Fatalf("boundary emitted prematurely: %q", got) + } + if got := stream.RedactChunk( + testURLSuffix, + ) + stream.Flush(); got != scheme+"***@example.test/x" { + t.Fatalf("boundary credential = %q", got) + } + stream = NewStream(New(Options{})) + if got := stream.RedactChunk(short + "x"); got != scheme+defaultMask { + t.Fatalf("overflow did not mask promptly: %q", got) + } + if got := stream.RedactChunk(testURLSuffix) + stream.Flush(); got != testURLSuffix { + t.Fatalf("overflow continuation = %q", got) + } + stream = NewStream(New(Options{})) + ordinary := scheme + "example.test" + if got := stream.RedactChunk( + ordinary, + ) + stream.RedactChunk( + "/x", + ) + stream.Flush(); got != ordinary+"/x" { + t.Fatalf("short ordinary URL = %q", got) + } +} + +func TestStreamShortUnresolvedURLAtEOF(t *testing.T) { + stream := NewStream(New(Options{})) + value := testURLScheme + "example.test" + if got := stream.RedactChunk(value); got != "" { + t.Fatalf("unresolved URL emitted early: %q", got) + } + if got := stream.Flush(); got != value { + t.Fatalf("unresolved URL at EOF = %q, want %q", got, value) + } + if got := stream.Flush(); got != "" { + t.Fatalf("second flush = %q", got) + } +} + +func TestStreamURLAuthorityInsideAuthorizationToken(t *testing.T) { + stream := NewStream(New(Options{})) + first := "Authorization: Bearer https://user," + second := strings.Repeat("x", 600) + "@example.test/x done" + got := stream.RedactChunk(first) + stream.RedactChunk(second) + stream.Flush() + want := "Authorization: Bearer ***@example.test/x done" + if got != want { + t.Fatalf("nested URL = %q, want %q", got, want) + } + stream = NewStream(New(Options{})) + got = stream.RedactChunk(first) + stream.RedactChunk(strings.Repeat("x", 600)) + + stream.RedactChunk("@example.test/x done") + stream.Flush() + if got != want { + t.Fatalf("overflowed nested URL = %q, want %q", got, want) + } +} + +func TestStreamURLOverflowWithLongExplicitPrefix(t *testing.T) { + scheme := testURLScheme + secret := scheme + testURLUser + strings.Repeat("x", 700) + "last" + stream := NewStream(New(Options{Values: []string{secret}})) + first := scheme + testURLUser + strings.Repeat("x", 600) + if got := stream.RedactChunk(first); got != "" { + t.Fatalf("explicit prefix leaked = %q", got) + } + got := stream.RedactChunk(strings.Repeat("x", 100)+"last@example.test/x") + stream.Flush() + if got != "***@example.test/x" { + t.Fatalf("explicit credential = %q", got) + } +} + +func TestStreamAuthorizationLongWhitespace(t *testing.T) { + spaces := strings.Repeat(" ", 1100) + input := "before Authorization" + spaces + ":" + spaces + "Bearer" + spaces + "unknown-token, after" + r := New(Options{Values: []string{"Bearer"}}) + want := "before Authorization" + spaces + ":" + spaces + "***" + spaces + "***, after" + if got := r.Redact(input); got != want { + t.Fatal("complete authorization masking differs") + } + for _, chunkSize := range []int{1, 17, 511, 1024} { + stream := NewStream(r) + var output strings.Builder + for i := 0; i < len(input); i += chunkSize { + end := min(i+chunkSize, len(input)) + output.WriteString(stream.RedactChunk(input[i:end])) + if len(stream.pending) > max(r.maxLength-1, formatCarryLimit) { + t.Fatalf("chunk %d retained %d bytes", chunkSize, len(stream.pending)) + } + } + output.WriteString(stream.Flush()) + if got := output.String(); got != want { + t.Fatalf("chunk %d: authorization output differs", chunkSize) + } + } +} + +func TestNilStreamAndBase(t *testing.T) { + var stream *Stream + if got := stream.RedactChunk("unchanged"); got != "unchanged" || stream.Flush() != "" { + t.Fatal("nil stream changed data") + } + stream = NewStream(nil) + if got := stream.RedactChunk(testCredentialURL()); got != testCredentialURL() || + stream.Flush() != "" { + t.Fatal("nil base changed data") + } +} + +func TestStreamLongKnownPrefixChunks(t *testing.T) { + prefix := strings.Repeat("x", 4096) + r := New(Options{Values: []string{prefix + "Z"}}) + tests := []struct { + name, input, want string + }{ + {benchComplete, prefix + "Z", defaultMask}, + {benchMismatch, prefix + "Q", prefix + "Q"}, + {benchFlush, prefix, prefix}, + {benchSliding, prefix + prefix, prefix + prefix}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, chunks := range [][]int{{1}, {1, 7, 3, 13}} { + stream := NewStream(r) + got, peak := streamChunks(stream, test.input, chunks) + if got != test.want { + t.Fatalf("chunks %v: output differs", chunks) + } + if peak > streamPendingBound(r) { + t.Fatalf("chunks %v: retained %d bytes", chunks, peak) + } + } + }) + } +} + +func TestStreamDenseAndFormatSliding(t *testing.T) { + prefix := strings.Repeat("x", 4096) + formatPrefix := strings.Repeat(testFormatPart, 820) + tests := []struct { + name string + values []string + input string + }{ + {benchDense, []string{prefix + "Z", testDenseToken}, prefix + prefix}, + {benchFormat, []string{formatPrefix + "Z"}, formatPrefix + formatPrefix}, + { + benchURL, + []string{formatPrefix + "Z"}, + testURLScheme + formatPrefix + formatPrefix + testURLSuffix, + }, + { + benchAuthURL, + []string{formatPrefix + "Z"}, + "Authorization: Bearer " + testURLScheme + testURLUser + "," + formatPrefix + formatPrefix + testURLSuffix, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := New(Options{Values: test.values}) + want := r.Redact(test.input) + for _, chunks := range [][]int{{1}, {1, 7, 3, 13}} { + stream := NewStream(r) + got, peak := streamChunks(stream, test.input, chunks) + if got != want { + t.Fatalf("chunks %v: output differs", chunks) + } + if peak > streamPendingBound(r) { + t.Fatalf("chunks %v: retained %d bytes", chunks, peak) + } + } + }) + } +} + +func TestStreamDefersDenseAndFormatChurn(t *testing.T) { + prefix := strings.Repeat(testFormatPart, 820) + r := New(Options{Values: []string{prefix + "Z", testDenseToken}}) + stream := NewStream(r) + if got := stream.RedactChunk(prefix); got != "" { + t.Fatalf("proper prefix emitted: %q", got) + } + if got := stream.RedactChunk(prefix[:100]); got != "" { + t.Fatalf("small churn emitted: %q", got) + } + if len(stream.pending) <= r.maxLength { + t.Fatalf("small churn was not retained: %d", len(stream.pending)) + } + if got := stream.RedactChunk(""); got != "" { + t.Fatalf("empty chunk emitted: %q", got) + } + input := prefix + prefix[:100] + if got, want := stream.Flush(), r.Redact(input); got != want { + t.Fatalf("flushed output differs: got %d bytes, want %d", len(got), len(want)) + } + if got := stream.Flush(); got != "" { + t.Fatalf("repeated flush emitted: %q", got) + } +} + +func streamPendingBound(r *Redactor) int { + known := max(0, r.maxLength-1) + return max(known+known/2, max(known, formatCarryLimit)+len("http\u017f://")) +} + +func TestStreamSlidingPreservesLaterCredentials(t *testing.T) { + prefix := strings.Repeat("x", 64) + r := New(Options{Values: []string{prefix + "Z"}}) + for _, suffix := range []string{ + "xxxxxxZ end", + "Authorization: Bearer unknown-token, safe", + "authorization = Basic\tunknown-token done", + testCredentialURL() + "/path", + } { + input := prefix + suffix + want := r.Redact(input) + for _, chunks := range [][]int{{1}, {1, 7, 3, 13}} { + got, _ := streamChunks(NewStream(r), input, chunks) + if got != want { + t.Fatalf("suffix %q chunks %v: got %q, want %q", suffix, chunks, got, want) + } + } + } +} + +func TestStreamKnownPrefixOverlapsURLAndAuthorization(t *testing.T) { + authority := testURLUser + strings.Repeat("x", 600) + secret := testURLScheme + authority + "Z" + r := New(Options{Values: []string{secret}}) + for _, input := range []string{ + secret + testURLSuffix, + testURLScheme + authority + testURLSuffix, + "Authorization: Bearer " + secret + " after", + } { + for _, chunks := range [][]int{{1}, {1, 7, 3, 13}} { + stream := NewStream(r) + got, _ := streamChunks(stream, input, chunks) + if want := r.Redact(input); got != want { + t.Fatalf("chunks %v: got %q, want %q", chunks, got, want) + } + if strings.Contains(got, authority) { + t.Fatal("authority leaked") + } + } + } +} + +func streamChunks(stream *Stream, input string, chunks []int) (string, int) { + var output strings.Builder + peak := 0 + for i, part := 0, 0; i < len(input); part++ { + end := min(i+chunks[part%len(chunks)], len(input)) + output.WriteString(stream.RedactChunk(input[i:end])) + peak = max(peak, len(stream.pending)) + i = end + } + output.WriteString(stream.Flush()) + return output.String(), peak +} + +func BenchmarkStreamKnownPrefix(b *testing.B) { + for _, size := range []struct { + name string + n int + }{ + {"4KiB", 4096}, + {"64KiB", 65536}, + } { + b.Run(size.name, func(b *testing.B) { + for _, chunks := range []struct { + name string + step []int + }{ + {"tiny", []int{1}}, + {"irregular", []int{1, 7, 3, 13}}, + } { + b.Run(chunks.name, func(b *testing.B) { + for _, mode := range []string{ + benchComplete, benchMismatch, benchFlush, benchSliding, benchDense, benchFormat, benchURL, benchAuthURL, + } { + b.Run(mode, func(b *testing.B) { + benchmarkStreamKnownPrefix(b, size.n, chunks.step, mode) + }) + } + }) + } + }) + } +} + +func benchmarkStreamKnownPrefix(b *testing.B, size int, chunks []int, mode string) { + prefix := strings.Repeat("x", size) + if mode == benchFormat || mode == benchURL || mode == benchAuthURL { + prefix = strings.Repeat(testFormatPart, size/len(testFormatPart)) + } + values := []string{prefix + "Z"} + if mode == benchDense { + values = append(values, testDenseToken) + } + r := New(Options{Values: values}) + input, want := benchmarkInput(prefix, mode, r) + b.SetBytes(int64(len(input))) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + got, _ := streamChunks(NewStream(r), input, chunks) + if got != want { + b.Fatal("stream output differs") + } + } +} + +func benchmarkInput(prefix, mode string, r *Redactor) (string, string) { + switch mode { + case benchComplete: + return prefix + "Z", defaultMask + case benchMismatch: + return prefix + "Q", prefix + "Q" + case benchSliding, benchDense, benchFormat: + input := prefix + prefix + return input, r.Redact(input) + case benchURL: + input := testURLScheme + prefix + prefix + testURLSuffix + return input, r.Redact(input) + case benchAuthURL: + input := "Authorization: Bearer " + testURLScheme + testURLUser + "," + prefix + prefix + testURLSuffix + return input, r.Redact(input) + default: + return prefix, prefix + } +}