diff --git a/.markdownlint.json b/.markdownlint.json deleted file mode 100644 index af3d391aa..000000000 --- a/.markdownlint.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "default": true, - "MD013": { - "line_length": 120, - "heading_line_length": 120, - "code_block_line_length": 150, - "tables": false - }, - "MD024": { - "siblings_only": true - }, - "MD033": { - "allowed_elements": ["details", "summary", "br", "sup", "sub", "kbd", "img"] - }, - "MD041": false, - "MD046": { - "style": "fenced" - } -} diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc new file mode 100644 index 000000000..acd50bc43 --- /dev/null +++ b/.markdownlint.jsonc @@ -0,0 +1,24 @@ +// markdownlint config for the pre-commit hook (scripts/pre-commit-hooks/markdownlint-staged.sh), +// `make lint-markdown`, and `npm run lint:md`. +{ + "default": true, + // TODO: the rules set to false below fire on pre-existing prose and are off so a commit touching an + // older doc does not fail on lines it did not write. Re-enable each one (delete its entry, which + // restores the default) once the repo-wide markdown cleanup lands. MD013 and MD040 were the loudest. + "MD001": false, // heading increment (pre-existing hit in CONTRIBUTING.md) + "MD013": false, // line length (was: line_length 120, heading_line_length 120, code_block_line_length 150, tables false) + "MD025": false, // multiple H1s (front-matter title + "# Title" in docs-site pages) + "MD036": false, // emphasis used as a heading + "MD040": false, // fenced code language + "MD060": false, // table column style + "MD024": { + "siblings_only": true + }, + "MD033": { + "allowed_elements": ["details", "summary", "br", "sup", "sub", "kbd", "img"] + }, + "MD041": false, + "MD046": { + "style": "fenced" + } +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 655f6f42e..334f7b3ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,8 @@ The CI-pinned version (currently v2.14.0, see `.github/workflows/quality-checks. **Note:** Lefthook pre-commit-phase hooks will **BLOCK commits** if golangci-lint finds issues. This is intentional - fix the issues before committing. +Staged `.md` files are also checked by markdownlint on commit (config in `.markdownlint.jsonc`); run `make lint-markdown` to check every tracked markdown file. + ### CI/CD Go Version Management GitHub Actions workflows automatically use go 1.26.0 via `GOTOOLCHAIN: auto`, which allows the `setup-go` action to download and use the correct Go version even if the CI environment has an older version installed. This ensures consistent builds across all workflows. diff --git a/Makefile b/Makefile index ceb1ab7bf..5466a7d6e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help plugin-powerdns plugin-powerdns-smoke install test build run clean docker-build docker-run build-offline release go-check gopls-logs lint-backend lint-agent lint-fast lint-staticcheck-only security-local +.PHONY: help plugin-powerdns plugin-powerdns-smoke install test build run clean docker-build docker-run build-offline release go-check gopls-logs lint-backend lint-agent lint-fast lint-staticcheck-only lint-markdown security-local # Default target help: @@ -213,6 +213,10 @@ lint-staticcheck-only: cd backend && golangci-lint run --config ../.golangci-fast.yml --enable-only staticcheck ./... cd agent && golangci-lint run --config ../.golangci-fast.yml --enable-only staticcheck ./... +lint-markdown: + @echo "Running markdownlint on all tracked markdown files..." + scripts/pre-commit-hooks/markdownlint-staged.sh + lint-docker: @echo "Running Hadolint..." docker run --rm -i hadolint/hadolint < Dockerfile diff --git a/backend/internal/api/handlers/crowdsec_handler.go b/backend/internal/api/handlers/crowdsec_handler.go index f73a45afd..36b521c92 100644 --- a/backend/internal/api/handlers/crowdsec_handler.go +++ b/backend/internal/api/handlers/crowdsec_handler.go @@ -82,6 +82,10 @@ type CrowdsecHandler struct { // signalProcess delivers a signal to a PID; overridable in tests. signalProcess func(pid int, sig syscall.Signal) error + // mergeNotifications and scanPlaceholders are the import's notification steps; overridable in tests. + mergeNotifications func(snapshotDir, dataDir string) error + scanPlaceholders func(dataDir string) ([]string, error) + // dataMu serializes every operation that mutates DataDir (preset apply, config import, file write) // and the export build; an export holds it only while writing its temporary file, never while sending. // Lock order is always dataMu first, then HubService.mu. @@ -352,6 +356,9 @@ const ( // nested installation reads. const msgImportFlatArchive = "the archive keeps its configuration at the top level, but this CrowdSec installation reads it from the config folder; export a configuration from this page, or put the files in a config/ folder inside the archive" +// msgKeepCurrentFiles is the fixed message for an import that could not keep the current files. +const msgKeepCurrentFiles = "failed to keep the current CrowdSec files" + // errConfigFileNotFound is the fixed message for an import that contains no config.yaml in a // location the engine reads (top level or config/). var errConfigFileNotFound = errors.New("config.yaml was not found at the top level or in config/") @@ -440,17 +447,19 @@ func NewCrowdsecHandler(db *gorm.DB, executor CrowdsecExecutor, binPath, dataDir consoleSvc = crowdsec.NewConsoleEnrollmentService(db, &crowdsec.SecureCommandExecutor{}, dataDir, consoleSecret) } h := &CrowdsecHandler{ - DB: db, - Executor: executor, - CmdExec: &RealCommandExecutor{}, - BinPath: binPath, - DataDir: dataDir, - Hub: hubSvc, - Console: consoleSvc, - Security: securitySvc, - dashCache: newDashboardCache(), - validateLAPIURL: validateCrowdsecLAPIBaseURLDefault, - signalProcess: signalOSProcess, + DB: db, + Executor: executor, + CmdExec: &RealCommandExecutor{}, + BinPath: binPath, + DataDir: dataDir, + Hub: hubSvc, + Console: consoleSvc, + Security: securitySvc, + dashCache: newDashboardCache(), + validateLAPIURL: validateCrowdsecLAPIBaseURLDefault, + signalProcess: signalOSProcess, + mergeNotifications: crowdsec.MergeRedactedNotifications, + scanPlaceholders: crowdsec.ScanPlaceholders, } hubSvc.Reload = h.reloadCrowdSec if db != nil { @@ -817,13 +826,22 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { } } + // Fill the notification values the exported file blanked from the files being replaced, wherever the + // notification still points at the same target. + if mergeErr := h.mergeNotifications(backupDir, h.DataDir); mergeErr != nil { + logger.Log().WithField("error", sanitizeForLog(mergeErr.Error())).Warn("crowdsec import: keeping the current notification values failed") + rollback() + c.JSON(http.StatusInternalServerError, gin.H{"error": msgKeepCurrentFiles}) + return + } + // The import cleared every file the archive did not replace: put back the ones CrowdSec cannot start // without, and make sure a kept profile still resolves its notifications. keptFiles, restoreErr := crowdsec.RestoreMissingCompanions(backupDir, h.DataDir, liveConfigPrefix(liveLayout)) if restoreErr != nil { logger.Log().WithField("error", sanitizeForLog(restoreErr.Error())).Warn("crowdsec import: restoring the current files failed") rollback() - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to keep the current CrowdSec files"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": msgKeepCurrentFiles}) return } if verifyErr := crowdsec.VerifyKeptProfile(h.DataDir, keptFiles); verifyErr != nil { @@ -848,9 +866,23 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { if len(keptFiles) > 0 { resp["kept_files"] = keptFiles } + if needs := h.placeholderReport(); len(needs) > 0 { + resp["needs_values"] = needs + } c.JSON(http.StatusOK, resp) } +// placeholderReport lists the notification files of the imported tree that still hold blanked values. +// The report is informational: when it cannot be made the import still stands and the list is omitted. +func (h *CrowdsecHandler) placeholderReport() []string { + names, err := h.scanPlaceholders(h.DataDir) + if err != nil { + logger.Log().WithField("reported_files", len(names)).Warn("crowdsec import: could not list the notification files that still need values") + return nil + } + return names +} + // liveConfigPrefix is the folder below DataDir that holds the configuration CrowdSec reads. func liveConfigPrefix(l crowdsec.Layout) string { if l == crowdsec.LayoutNested { diff --git a/backend/internal/api/handlers/crowdsec_import_needs_values_test.go b/backend/internal/api/handlers/crowdsec_import_needs_values_test.go new file mode 100644 index 000000000..9f3a8ff13 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_import_needs_values_test.go @@ -0,0 +1,269 @@ +package handlers + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/Wikid82/charon/backend/internal/crowdsec" + "github.com/Wikid82/charon/backend/internal/logger" +) + +const ( + nvToken = "SECRETTOKEN-live" + nvPlain = "type: http\nname: plain\nurl: https://hooks.example.invalid/" + nvToken + "\nmethod: POST\nheaders:\n Authorization: Bearer " + nvToken + "\n" + nvEmail = "type: email\nname: mail\nsmtp_host: smtp.example.invalid\nsmtp_username: " + nvToken + "\nsmtp_password: " + nvToken + "\nreceiver_emails:\n - ops@example.invalid\n" + // nvAliased reuses a kept value through an anchor, so the exported file still holds an alias. + nvAliased = "type: slack\nname: shared\nwebhook: https://hooks.example.invalid/" + nvToken + "\nchannel: &c ops\nusername: *c\n" +) + +func nvProfiles(names ...string) string { + out := "name: p\nfilters:\n - Alert.Remediation == true\ndecisions:\n - type: ban\n duration: 4h\nnotifications:\n" + for _, n := range names { + out += " - " + n + "\n" + } + return out +} + +func archiveBodies(t *testing.T, data []byte) map[string]string { + t.Helper() + gr, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + tr := tar.NewReader(gr) + bodies := map[string]string{} + for { + hdr, nextErr := tr.Next() + if errors.Is(nextErr, io.EOF) { + return bodies + } + require.NoError(t, nextErr) + body, readErr := io.ReadAll(tr) + require.NoError(t, readErr) + bodies[filepath.ToSlash(hdr.Name)] = string(body) + } +} + +func needsValuesOf(t *testing.T, w *httptest.ResponseRecorder) (names []string, present bool) { + t.Helper() + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var body map[string]json.RawMessage + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + raw, present := body["needs_values"] + if present { + require.NoError(t, json.Unmarshal(raw, &names)) + } + return names, present +} + +func parsedYAML(t *testing.T, text string) []any { + t.Helper() + dec := yaml.NewDecoder(strings.NewReader(text)) + var docs []any + for { + var v any + if err := dec.Decode(&v); err != nil { + require.ErrorIs(t, err, io.EOF) + return docs + } + docs = append(docs, v) + } +} + +func TestImportConfigRoundTripKeepsCurrentNotificationValues(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + writeTree(t, dir, map[string]string{ + "config/profiles.yaml": nvProfiles("plain", "mail", "shared"), + "config/notifications/plain.yaml": nvPlain, + "config/notifications/mail.yaml": nvEmail, + "config/notifications/shared.yaml": nvAliased, + }) + _, r := newImportRouter(t, dir) + exported := exportBytes(t, r) + entries := archiveBodies(t, exported) + require.NotContains(t, entries["config/notifications/plain.yaml"], nvToken) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importBytesRequest(t, exported)) + + names, present := needsValuesOf(t, w) + require.True(t, present) + require.Equal(t, []string{"notifications/shared.yaml"}, names) + require.Equal(t, parsedYAML(t, nvPlain), parsedYAML(t, readTreeFile(t, filepath.Join(dir, "config", "notifications", "plain.yaml")))) + require.Equal(t, parsedYAML(t, nvEmail), parsedYAML(t, readTreeFile(t, filepath.Join(dir, "config", "notifications", "mail.yaml")))) + require.Equal(t, entries["config/notifications/shared.yaml"], readTreeFile(t, filepath.Join(dir, "config", "notifications", "shared.yaml")), + "a file with shared structure is left exactly as extracted") +} + +func TestImportConfigOmitsNeedsValuesWhenNothingIsBlanked(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + writeTree(t, dir, map[string]string{"config/notifications/plain.yaml": nvPlain}) + _, r := newImportRouter(t, dir) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importBytesRequest(t, exportBytes(t, r))) + + _, present := needsValuesOf(t, w) + require.False(t, present) +} + +func TestImportConfigIntoAHostWithoutTheFileReportsIt(t *testing.T) { + t.Parallel() + source := t.TempDir() + seedRealLayout(t, source) + writeTree(t, source, map[string]string{ + "config/profiles.yaml": nvProfiles("plain"), + "config/notifications/plain.yaml": nvPlain, + }) + _, sourceRouter := newImportRouter(t, source) + exported := exportBytes(t, sourceRouter) + + target := t.TempDir() + seedRealLayout(t, target) + _, targetRouter := newImportRouter(t, target) + w := httptest.NewRecorder() + targetRouter.ServeHTTP(w, importBytesRequest(t, exported)) + + names, _ := needsValuesOf(t, w) + require.Equal(t, []string{"notifications/plain.yaml"}, names) + got := readTreeFile(t, filepath.Join(target, "config", "notifications", "plain.yaml")) + require.Contains(t, got, crowdsec.RedactedPlaceholder) + require.NotContains(t, got, nvToken) +} + +func TestImportConfigReportsRestoredFilesThatAreStillBlanked(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + blanked := "type: http\nname: alpha\nurl: " + crowdsec.RedactedPlaceholder + "\n" + writeTree(t, dir, map[string]string{ + "config/profiles.yaml": nvProfiles("alpha"), + "config/notifications/alpha.yaml": blanked, + }) + _, r := newImportRouter(t, dir) + + w := importStatus(t, r, map[string]string{"config/config.yaml": importableConfig}) + + names, _ := needsValuesOf(t, w) + require.Equal(t, []string{"notifications/alpha.yaml"}, names) + require.Contains(t, keptFilesOf(t, w.Body.Bytes()), "notifications/alpha.yaml") + require.Equal(t, blanked, readTreeFile(t, filepath.Join(dir, "config", "notifications", "alpha.yaml"))) +} + +func TestImportConfigKeepsCurrentValuesWhenAProfileNamesTheBlankedNotification(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + writeTree(t, dir, map[string]string{ + "config/profiles.yaml": nvProfiles("alpha"), + "config/notifications/alpha.yaml": strings.Replace(importNotification, "%s", "alpha", 1), + }) + _, r := newImportRouter(t, dir) + blanked := "type: http\nname: alpha\nurl: https://example.invalid/\nmethod: POST\nheaders:\n X-Key: " + crowdsec.RedactedPlaceholder + "\n" + + w := importStatus(t, r, map[string]string{ + "config/config.yaml": importableConfig, + "config/profiles.yaml": nvProfiles("alpha"), + "config/notifications/alpha.yaml": blanked, + }) + + _, present := needsValuesOf(t, w) + require.True(t, present, "the live file has no X-Key header to fill from") + require.Equal(t, blanked, readTreeFile(t, filepath.Join(dir, "config", "notifications", "alpha.yaml"))) +} + +func TestImportConfigRollsBackWhenKeepingCurrentValuesFails(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + writeTree(t, dir, map[string]string{"config/notifications/plain.yaml": nvPlain}) + h, r := newImportRouter(t, dir) + h.mergeNotifications = func(string, string) error { return errors.New("disk problem with " + nvToken) } + before := treeState(t, dir) + + w := importStatus(t, r, map[string]string{ + "config/config.yaml": importableConfig, + "config/notifications/plain.yaml": "type: http\nname: plain\nurl: " + crowdsec.RedactedPlaceholder + "\n", + }) + + require.Equal(t, http.StatusInternalServerError, w.Code) + require.Equal(t, msgKeepCurrentFiles, importNotificationBody(t, w)) + requireSameTree(t, before, dir) +} + +// Not parallel: it adds a hook to the shared logger to see every message. +func TestImportConfigKeepsTheImportWhenTheReportFails(t *testing.T) { + hook := &messageHook{} + logger.Log().Logger.AddHook(hook) + + dir := t.TempDir() + seedRealLayout(t, dir) + writeTree(t, dir, map[string]string{"config/notifications/plain.yaml": nvPlain}) + h, r := newImportRouter(t, dir) + h.scanPlaceholders = func(string) ([]string, error) { + return []string{"notifications/" + nvToken + ".yaml"}, errors.New("walk failed at " + nvToken) + } + archive := "type: http\nname: plain\nurl: " + crowdsec.RedactedPlaceholder + "\n" + + w := importStatus(t, r, map[string]string{ + "config/config.yaml": importableConfig, + "config/notifications/plain.yaml": archive, + }) + + _, present := needsValuesOf(t, w) + require.False(t, present) + require.Contains(t, readTreeFile(t, filepath.Join(dir, "config", "notifications", "plain.yaml")), nvToken, "the import itself stands") + hook.mu.Lock() + defer hook.mu.Unlock() + var sawReport bool + for _, m := range hook.messages { + require.NotContains(t, m, nvToken) + sawReport = sawReport || strings.Contains(m, "still need values") + } + require.True(t, sawReport) +} + +type messageHook struct { + mu sync.Mutex + messages []string +} + +func (h *messageHook) Levels() []logrus.Level { return logrus.AllLevels } + +func (h *messageHook) Fire(e *logrus.Entry) error { + h.mu.Lock() + defer h.mu.Unlock() + line := e.Message + for k, v := range e.Data { + line += " " + k + "=" + toString(v) + } + h.messages = append(h.messages, line) + return nil +} + +func toString(v any) string { + switch x := v.(type) { + case string: + return x + case error: + return x.Error() + } + b, _ := json.Marshal(v) + return string(b) +} diff --git a/backend/internal/crowdsec/export.go b/backend/internal/crowdsec/export.go index 57a74ee9d..826eb97d7 100644 --- a/backend/internal/crowdsec/export.go +++ b/backend/internal/crowdsec/export.go @@ -78,7 +78,9 @@ func checkExportImportable(compressed, uncompressed, maxCompressed int64, maxRat // skipped with fs.SkipDir only when excluded, and non-regular entries (symlinks, pipes, sockets, // devices) are skipped so that nothing is followed or blocks on open. A file that vanishes is skipped; // an unreadable directory is logged and skipped; the installed hub items are listed in a generated -// ManifestEntryName entry (only when there are any); an unreadable regular file, or one whose size changes +// ManifestEntryName entry (only when there are any); notification settings files have their sensitive +// values blanked (RedactNotificationFile) or, when that is not possible, are exported empty, and stray +// non-YAML files in a notification folder are left out; an unreadable regular file, or one whose size changes // while it is copied, fails the export with ErrExportFileUnreadable. func WriteExportArchive(ctx context.Context, dataDir string, w io.Writer) (int64, error) { return writeExportArchive(ctx, dataDir, w, MaxImportUncompressedBytes, exportHooks{}) @@ -92,6 +94,7 @@ func writeExportArchive(ctx context.Context, dataDir string, w io.Writer, maxByt gw := gzip.NewWriter(&limitWriter{w: w, limit: maxCompressed}) tw := tar.NewWriter(gw) var total int64 + dirs := treeNotificationDirs(dataDir) walkErr := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error { if ctxErr := ctx.Err(); ctxErr != nil { @@ -122,7 +125,7 @@ func writeExportArchive(ctx context.Context, dataDir string, w io.Writer, maxByt logger.Log().Warnf("skipping non-regular entry %s during export", util.SanitizeForLog(rel)) return nil } - n, copyErr := addExportFile(tw, dataDir, rel, total, maxBytes, hooks) + n, copyErr := addExportFile(tw, dataDir, rel, total, maxBytes, dirs, hooks) total += n return copyErr }) @@ -158,20 +161,49 @@ func handleExportWalkError(path string, d fs.DirEntry, err error) error { } } -// addExportFile copies one regular file into tw and returns the bytes it contributed. -func addExportFile(tw *tar.Writer, dataDir, rel string, total, maxBytes int64, hooks exportHooks) (int64, error) { +// addExportFile copies one regular file into tw and returns the bytes it contributed. Files that are +// notification settings (IsNotificationFile) are rewritten by addRedactedExportFile, or left out. +func addExportFile(tw *tar.Writer, dataDir, rel string, total, maxBytes int64, dirs []string, hooks exportHooks) (int64, error) { + slash := filepath.ToSlash(rel) + kind := IsNotificationFile(slash, nil, dirs) + if kind == InDirOther { + // CrowdSec never reads these, and they cannot be rewritten structurally. + logger.Log().Debugf("leaving %s out of the export: not a notification file CrowdSec reads", util.SanitizeForLog(rel)) + return 0, nil + } + f, info, err := openExportFile(dataDir, rel, hooks) + if f == nil || err != nil { + return 0, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + logger.Log().WithError(closeErr).Warnf("failed to close %s while archiving", util.SanitizeForLog(rel)) + } + }() + + // A YAML file outside the notification folders may still be one (look-alike), so it is read and + // probed too, as long as it is small enough to be parsed. + if kind == InDirYAML || (hasYAMLExt(slash) && info.Size() <= maxInspectEntryBytes) { + return addRedactedExportFile(tw, f, info, rel, kind, dirs, total, maxBytes) + } + if total+info.Size() > maxBytes { + return 0, ErrExportTooLarge + } + return copyExportFile(tw, f, info, rel) +} + +// openExportFile opens the walked entry rel for reading and verifies that it is still the regular file +// the walk listed. A nil file with a nil error means the entry is skipped (vanished or swapped for a link). +func openExportFile(dataDir, rel string, hooks exportHooks) (*os.File, os.FileInfo, error) { path := filepath.Join(dataDir, rel) // Lstat first: the entry came from the directory listing, so a swap to a link or pipe afterwards // must not be opened. info, err := os.Lstat(path) if errors.Is(err, fs.ErrNotExist) { - return 0, nil + return nil, nil, nil } if err != nil || !info.Mode().IsRegular() { - return 0, unreadableFile(rel, err) - } - if total+info.Size() > maxBytes { - return 0, ErrExportTooLarge + return nil, nil, unreadableFile(rel, err) } if hooks.beforeOpen != nil { hooks.beforeOpen(path) @@ -181,60 +213,143 @@ func addExportFile(tw *tar.Writer, dataDir, rel string, total, maxBytes int64, h // swapped-in FIFO from blocking while the data lock is held. The opened file is verified below. f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0) //nolint:gosec // G304: path is dataDir (Charon-owned) joined with a walked entry; O_NOFOLLOW|O_NONBLOCK plus the fstat identity check below guard against a swap after Lstat if errors.Is(err, fs.ErrNotExist) { - return 0, nil + return nil, nil, nil } if errors.Is(err, syscall.ELOOP) { logger.Log().Warnf("skipping %s during export: replaced by a symlink", util.SanitizeForLog(rel)) - return 0, nil + return nil, nil, nil } if err != nil { - return 0, unreadableFile(rel, err) + return nil, nil, unreadableFile(rel, err) } - defer func() { - if closeErr := f.Close(); closeErr != nil { - logger.Log().WithError(closeErr).Warnf("failed to close %s while archiving", util.SanitizeForLog(rel)) - } - }() opened, err := f.Stat() if err != nil { - return 0, unreadableFile(rel, err) + _ = f.Close() + return nil, nil, unreadableFile(rel, err) } if !opened.Mode().IsRegular() { + _ = f.Close() logger.Log().Warnf("skipping %s during export: no longer a regular file", util.SanitizeForLog(rel)) - return 0, nil + return nil, nil, nil } if !os.SameFile(info, opened) { - return 0, unreadableFile(rel, errors.New("file was replaced while exporting")) + _ = f.Close() + return nil, nil, unreadableFile(rel, errors.New("file was replaced while exporting")) } + return f, info, nil +} - hdr := &tar.Header{ - Name: filepath.ToSlash(rel), - Size: info.Size(), - Mode: int64(info.Mode().Perm()), - ModTime: info.ModTime(), - } - if err = tw.WriteHeader(hdr); err != nil { +// copyExportFile streams the opened file into tw byte for byte. +func copyExportFile(tw *tar.Writer, f *os.File, info os.FileInfo, rel string) (int64, error) { + if err := tw.WriteHeader(exportHeader(rel, info, info.Size())); err != nil { return 0, fmt.Errorf("write tar header: %w", err) } - if _, err = io.CopyN(tw, f, info.Size()); err != nil { + if _, err := io.CopyN(tw, f, info.Size()); err != nil { if errors.Is(err, io.EOF) { return 0, unreadableFile(rel, fmt.Errorf("file shrank while copying: %w", err)) } return 0, fmt.Errorf("copy file into archive: %w", err) } + if err := checkUnchanged(f, info, rel); err != nil { + return 0, err + } + return info.Size(), nil +} + +// checkUnchanged fails when the file no longer has the size it had when it was opened. +func checkUnchanged(f *os.File, info os.FileInfo, rel string) error { after, err := f.Stat() if err != nil { - return 0, unreadableFile(rel, err) + return unreadableFile(rel, err) } if after.Size() != info.Size() { - return 0, unreadableFile(rel, fmt.Errorf("file size changed while copying: %d -> %d", info.Size(), after.Size())) + return unreadableFile(rel, fmt.Errorf("file size changed while copying: %d -> %d", info.Size(), after.Size())) } - return info.Size(), nil + return nil +} + +func exportHeader(rel string, info os.FileInfo, size int64) *tar.Header { + return &tar.Header{ + Name: filepath.ToSlash(rel), + Size: size, + Mode: int64(info.Mode().Perm()), + ModTime: info.ModTime(), + } +} + +// writeExportEntry writes data as the content of one archive entry and returns its length. +func writeExportEntry(tw *tar.Writer, rel string, info os.FileInfo, data []byte) (int64, error) { + if err := tw.WriteHeader(exportHeader(rel, info, int64(len(data)))); err != nil { + return 0, fmt.Errorf("write tar header: %w", err) + } + if _, err := tw.Write(data); err != nil { + return 0, fmt.Errorf("write file into archive: %w", err) + } + return int64(len(data)), nil +} + +// addRedactedExportFile writes a notification file with its sensitive values blanked. A file that +// cannot be rewritten safely (too large, unparsable, anything RedactNotificationFile refuses) is +// exported as an empty stub instead; the original bytes never reach the archive. The entry size and the +// running total use the length of what is written. +func addRedactedExportFile(tw *tar.Writer, f *os.File, info os.FileInfo, rel string, kind NotificationFileKind, dirs []string, total, maxBytes int64) (int64, error) { + var out []byte + if info.Size() > maxInspectEntryBytes { + out = []byte(RedactStubContent) + logStubbed(rel) + } else { + data, err := io.ReadAll(io.LimitReader(f, maxInspectEntryBytes+1)) + if err != nil { + return 0, unreadableFile(rel, err) + } + if int64(len(data)) != info.Size() { + return 0, unreadableFile(rel, fmt.Errorf("file size changed while copying: %d -> %d", info.Size(), len(data))) + } + if err := checkUnchanged(f, info, rel); err != nil { + return 0, err + } + if kind == NotAFile && IsNotificationFile(filepath.ToSlash(rel), LookAlikeDocuments(data), dirs) == NotAFile { + out = data + } else if res, redactErr := RedactNotificationFile(data); redactErr != nil { + out = []byte(RedactStubContent) + logStubbed(rel) + } else { + out = res.Data + } + } + if total+int64(len(out)) > maxBytes { + return 0, ErrExportTooLarge + } + return writeExportEntry(tw, rel, info, out) +} + +// logStubbed notes, without any file content, that a notification file was exported empty. +func logStubbed(rel string) { + logger.Log().Warnf("exporting %s as an empty file: it could not be read safely", util.SanitizeForLog(rel)) +} + +// treeNotificationDirs returns the notification folders of the tree below dataDir (the live tree, an +// extracted archive or a snapshot), honouring a redirected folder named by the tree's own config.yaml. +// An unreadable config means the default folders. Export, merge and scan all resolve through it. +func treeNotificationDirs(dataDir string) []string { + for _, rel := range []string{"config/config.yaml", "config.yaml"} { + f, info, err := openExportFile(dataDir, filepath.FromSlash(rel), exportHooks{}) + if err != nil || f == nil { + continue + } + data, readErr := io.ReadAll(io.LimitReader(f, maxInspectEntryBytes+1)) + _ = f.Close() + if readErr != nil || info.Size() > maxInspectEntryBytes || int64(len(data)) > maxInspectEntryBytes { + continue + } + return NotificationDirs(dataDir, data) + } + return NotificationDirs(dataDir, nil) } // unreadableFile logs the cause with the sanitized relative path and returns the fixed sentinel. func unreadableFile(rel string, cause error) error { - logger.Log().WithError(cause).Warnf("cannot export %s", util.SanitizeForLog(rel)) + logger.Log().WithError(cause).Warnf("cannot read %s", util.SanitizeForLog(rel)) return ErrExportFileUnreadable } diff --git a/backend/internal/crowdsec/export_redact_test.go b/backend/internal/crowdsec/export_redact_test.go new file mode 100644 index 000000000..3ebdb2034 --- /dev/null +++ b/backend/internal/crowdsec/export_redact_test.go @@ -0,0 +1,381 @@ +package crowdsec + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + "github.com/Wikid82/charon/backend/internal/logger" +) + +const planted = "SECRETTOKEN-" + +// exportEntries exports dir and returns the archive bytes, the entries by name and the reported total. +func exportEntries(t *testing.T, dir string) (archive []byte, entries map[string]string, total int64) { + t.Helper() + var buf bytes.Buffer + total, err := WriteExportArchive(context.Background(), dir, &buf) + require.NoError(t, err) + return buf.Bytes(), readEntries(t, buf.Bytes()), total +} + +func readEntries(t *testing.T, archive []byte) map[string]string { + t.Helper() + gr, err := gzip.NewReader(bytes.NewReader(archive)) + require.NoError(t, err) + tr := tar.NewReader(gr) + entries := map[string]string{} + for { + hdr, nextErr := tr.Next() + if errors.Is(nextErr, io.EOF) { + break + } + require.NoError(t, nextErr) + body, readErr := io.ReadAll(tr) + require.NoError(t, readErr) + require.EqualValues(t, len(body), hdr.Size) + entries[hdr.Name] = string(body) + } + return entries +} + +// requireNoToken scans the whole decompressed archive (headers, names and bodies) for the planted prefix. +func requireNoToken(t *testing.T, archive []byte) { + t.Helper() + gr, err := gzip.NewReader(bytes.NewReader(archive)) + require.NoError(t, err) + raw, err := io.ReadAll(gr) + require.NoError(t, err) + require.NotContains(t, string(raw), planted) +} + +const plantedHTTP = `# comment ` + planted + `head +type: http # ` + planted + `line +name: http_default +url: https://user:` + planted + `userinfo@hooks.example.invalid/` + planted + `path +headers: &` + planted + `anchor + Authorization: Bearer ` + planted + `bearer + X-Api-Token: ` + planted + `apitoken +format: '{{.|toJson}}' +# ` + planted + `foot +` + +var plantedPlugins = map[string]string{ + "slack.yaml": "type: slack\nname: s\nwebhook: https://hooks.example.invalid/" + planted + "hook\nchannel: ops\n", + "splunk.yaml": "type: splunk\nname: sp\nurl: https://splunk.example.invalid\ntoken: " + planted + "splunk\n", + "email.yaml": "type: email\nname: e\nsmtp_host: smtp.example.invalid\nsmtp_username: " + planted + "user\nsmtp_password: " + planted + "pass\nreceiver_emails: [a@example.invalid]\n", + "sentinel.yaml": "type: sentinel\nname: se\ncustomer_id: " + planted + "cust\nshared_key: " + planted + "key\nlog_type: Charon\n", + "unknown.yaml": "type: mystery\nname: m\nsecret: " + planted + "unknown\nnested:\n " + planted + "key: " + planted + "value\n", + "yes.yaml": "type: splunk\nname: y\ntoken: yes\nurl: on\n", + "http.yaml": plantedHTTP, +} + +func TestExportArchiveHoldsNoPlantedValueAnywhere(t *testing.T) { + t.Parallel() + dir := t.TempDir() + files := map[string]string{ + "config/config.yaml": "config_paths:\n notification_dir: " + filepath.Join(dir, "config", "custom_notify") + "\n", + "config/notifications.bak/copy.yaml": plantedHTTP, + "config/notifications/sub/deeper/http.yaml": plantedHTTP, + "config/Notifications/UPPER.YML": plantedHTTP, + "config/custom_notify/http.yaml": plantedHTTP, + "config/notifications/http.yaml.bak": plantedHTTP, + "config/notifications/notes.txt": planted + "notes", + "config/notifications/sub/old.bak": planted + "old", + "config/notifications/binary.yaml": "\x00\x01" + planted + "\xff", + "config/notifications/duplicate_keys.yaml": "type: http\ntype: http\nurl: " + planted + "dup\n", + "config/notifications/oversize.yaml": "type: http\nurl: " + planted + strings.Repeat("a", int(maxInspectEntryBytes)) + "\n", + "config/profiles.yaml": "name: p\nnotifications:\n - http_default\n", + "config/parsers/s01-parse/custom.yaml": "name: custom\n", + "config/notifications/not_a_notification.md": planted + "readme", + } + for name, content := range plantedPlugins { + files["config/notifications/"+name] = content + } + writeExportTree(t, dir, files) + outside := filepath.Join(t.TempDir(), "outside.yaml") + require.NoError(t, os.WriteFile(outside, []byte("type: http\nurl: "+planted+"outside\n"), 0o600)) + require.NoError(t, os.Symlink(outside, filepath.Join(dir, "config", "notifications", "link.yaml"))) + linkedDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(linkedDir, "x.yaml"), []byte("type: http\nurl: "+planted+"dirlink\n"), 0o600)) + require.NoError(t, os.Symlink(linkedDir, filepath.Join(dir, "config", "notifications", "linked_dir"))) + + archive, entries, _ := exportEntries(t, dir) + requireNoToken(t, archive) + + for name := range entries { + require.NotContains(t, name, planted) + } + require.NotContains(t, entries, "config/notifications/http.yaml.bak") + require.NotContains(t, entries, "config/notifications/notes.txt") + require.NotContains(t, entries, "config/notifications/sub/old.bak") + require.NotContains(t, entries, "config/notifications/not_a_notification.md") + require.NotContains(t, entries, "config/notifications/link.yaml") + require.NotContains(t, entries, "config/notifications/linked_dir/x.yaml") + + for _, name := range []string{ + "config/notifications/http.yaml", + "config/notifications/sub/deeper/http.yaml", + "config/Notifications/UPPER.YML", + "config/custom_notify/http.yaml", + "config/notifications.bak/copy.yaml", + } { + require.Contains(t, entries, name) + require.Contains(t, entries[name], "name: http_default") + require.Contains(t, entries[name], RedactedPlaceholder) + } + for _, name := range []string{"binary.yaml", "duplicate_keys.yaml", "oversize.yaml"} { + require.Equal(t, RedactStubContent, entries["config/notifications/"+name], name) + } + require.Equal(t, "name: p\nnotifications:\n - http_default\n", entries["config/profiles.yaml"]) + require.Equal(t, "name: custom\n", entries["config/parsers/s01-parse/custom.yaml"]) +} + +func TestExportArchiveBlanksFlatLayoutNotifications(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{ + "config.yaml": "common:\n log_level: info\n", + "notifications/http.yaml": plantedHTTP, + "notifications/http.bak": plantedHTTP, + }) + archive, entries, _ := exportEntries(t, dir) + requireNoToken(t, archive) + require.Contains(t, entries, "notifications/http.yaml") + require.NotContains(t, entries, "notifications/http.bak") +} + +func TestExportArchiveLeavesOtherYAMLUnchanged(t *testing.T) { + t.Parallel() + dir := t.TempDir() + other := map[string]string{ + "config/config.yaml": "db_config:\n type: sqlite\n", + "config/typeless.yaml": "name: x\nurl: https://example.invalid\n", + "config/unparsable.yaml": "a: [unclosed\n", + "config/type_unknown.yaml": "type: parser\nname: x\n", + "config/type_not_a_str.yaml": "type: !!binary aHR0cA==\n", + "config/parsers/dup_keys.yaml": "a: 1\na: 2\n", + "config/plain.txt": "type: http\nurl: x\n", + } + writeExportTree(t, dir, other) + _, entries, _ := exportEntries(t, dir) + for name, want := range other { + require.Equal(t, want, entries[name], name) + } +} + +func TestExportArchiveLargeLookAlikeIsStreamedUnchanged(t *testing.T) { + t.Parallel() + dir := t.TempDir() + big := "type: http\nurl: x\n# " + strings.Repeat("a", int(maxInspectEntryBytes)) + "\n" + writeExportTree(t, dir, map[string]string{"config/big.yaml": big}) + _, entries, _ := exportEntries(t, dir) + require.Equal(t, big, entries["config/big.yaml"]) +} + +func TestExportArchiveSizeAccountingUsesWrittenBytes(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{ + "config/config.yaml": "a: 1\n", + "config/notifications/http.yaml": plantedHTTP, + "config/notifications/stub.yaml": "a: [unclosed\n", + "config/notifications/ignored.back": "left out", + }) + _, entries, total := exportEntries(t, dir) + var sum int64 + for _, body := range entries { + sum += int64(len(body)) + } + require.Equal(t, sum, total) + require.Equal(t, RedactStubContent, entries["config/notifications/stub.yaml"]) + require.NotEqual(t, int64(len(plantedHTTP)), int64(len(entries["config/notifications/http.yaml"]))) +} + +func TestExportArchiveSizeCapCountsRedactedLength(t *testing.T) { + t.Parallel() + dir := t.TempDir() + original := "type: splunk\nname: n\ntoken: " + strings.Repeat("x", 600) + "\n" + writeExportTree(t, dir, map[string]string{"notifications/a.yaml": original}) + var buf bytes.Buffer + redacted, err := RedactNotificationFile([]byte(original)) + require.NoError(t, err) + require.Less(t, len(redacted.Data), len(original)) + + total, err := writeExportArchive(context.Background(), dir, &buf, int64(len(redacted.Data)), exportHooks{}) + require.NoError(t, err, "the cap applies to the redacted length") + require.EqualValues(t, len(redacted.Data), total) + + _, err = writeExportArchive(context.Background(), dir, io.Discard, int64(len(redacted.Data))-1, exportHooks{}) + require.ErrorIs(t, err, ErrExportTooLarge) +} + +func TestExportArchiveNotificationFileChangedWhileReading(t *testing.T) { + t.Parallel() + cases := map[string]func(path string){ + "grows": func(path string) { appendTo(path, "# more\n") }, + "shrinks": func(path string) { _ = os.Truncate(path, 4) }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"notifications/http.yaml": plantedHTTP}) + _, err := writeExportArchive(context.Background(), dir, io.Discard, MaxImportUncompressedBytes, + exportHooks{beforeOpen: mutate}) + require.ErrorIs(t, err, ErrExportFileUnreadable) + }) + } +} + +func appendTo(path, text string) { + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) //nolint:gosec // test fixture + if err == nil { + _, _ = f.WriteString(text) + _ = f.Close() + } +} + +func TestExportArchiveNotificationFileSwappedForSymlinkIsNotFollowed(t *testing.T) { + t.Parallel() + secret := filepath.Join(t.TempDir(), "secret.yaml") + require.NoError(t, os.WriteFile(secret, []byte("type: http\nurl: "+planted+"swap\n"), 0o600)) + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"notifications/http.yaml": plantedHTTP, "notifications/z.yaml": "type: dummy\nname: z\n"}) + var buf bytes.Buffer + _, err := writeExportArchive(context.Background(), dir, &buf, MaxImportUncompressedBytes, exportHooks{ + beforeOpen: func(path string) { + if filepath.Base(path) == "http.yaml" { + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Symlink(secret, path)) + } + }, + }) + require.NoError(t, err) + requireNoToken(t, buf.Bytes()) + require.Equal(t, []string{"notifications/z.yaml"}, exportNames(t, buf.Bytes())) +} + +func TestExportArchiveNotificationFileSwappedForAnotherFileFails(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"notifications/http.yaml": plantedHTTP}) + _, err := writeExportArchive(context.Background(), dir, io.Discard, MaxImportUncompressedBytes, exportHooks{ + beforeOpen: func(path string) { + replacement := path + ".new" + require.NoError(t, os.WriteFile(replacement, []byte(plantedHTTP), 0o600)) + require.NoError(t, os.Rename(replacement, path)) + }, + }) + require.ErrorIs(t, err, ErrExportFileUnreadable) +} + +func TestLiveNotificationDirsFallsBackToDefaults(t *testing.T) { + t.Parallel() + defaults := NotificationDirs(t.TempDir(), nil) + + missing := t.TempDir() + require.Equal(t, defaults, treeNotificationDirs(missing)) + + oversize := t.TempDir() + writeExportTree(t, oversize, map[string]string{"config/config.yaml": strings.Repeat("#", int(maxInspectEntryBytes)+1)}) + require.Equal(t, defaults, treeNotificationDirs(oversize)) + + flat := t.TempDir() + writeExportTree(t, flat, map[string]string{"config.yaml": "config_paths:\n notification_dir: " + filepath.Join(flat, "alerts") + "\n"}) + require.Equal(t, append(append([]string(nil), defaults...), "alerts"), treeNotificationDirs(flat)) +} + +// recordingHook keeps the messages of every log entry while enabled. +type recordingHook struct { + mu sync.Mutex + messages []string +} + +func (h *recordingHook) Levels() []logrus.Level { return logrus.AllLevels } + +func (h *recordingHook) Fire(e *logrus.Entry) error { + h.mu.Lock() + defer h.mu.Unlock() + line := e.Message + for _, v := range e.Data { + if s, ok := v.(error); ok { + line += " " + s.Error() + } + } + h.messages = append(h.messages, line) + return nil +} + +// Not parallel: it switches the shared logger to debug level to see every message. +func TestExportArchiveLogsHoldNoNotificationContent(t *testing.T) { + hook := &recordingHook{} + logger.Log().Logger.AddHook(hook) + previous := logger.Log().Logger.GetLevel() + logger.Log().Logger.SetLevel(logrus.DebugLevel) + t.Cleanup(func() { logger.Log().Logger.SetLevel(previous) }) + + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{ + "notifications/http.yaml": plantedHTTP, + "notifications/dup.yaml": "type: http\ntype: http\nurl: " + planted + "dup\n", + "notifications/stray.bak": planted + "stray", + }) + _, _, _ = exportEntries(t, dir) + + hook.mu.Lock() + defer hook.mu.Unlock() + var sawStub bool + for _, m := range hook.messages { + if strings.Contains(m, "dup.yaml") { + sawStub = true + } + require.NotContains(t, m, planted) + } + require.True(t, sawStub) +} + +func TestExportArchiveBlanksLookAlikesWithQuotedOrLaterDocumentTypes(t *testing.T) { + t.Parallel() + dir := t.TempDir() + files := map[string]string{ + "config/other/double.yml": "type: \"http\"\nname: n\nurl: https://h.example.invalid/" + planted + "a\n", + "config/other/single.yaml": "type: 'splunk'\nname: n\ntoken: " + planted + "b\n", + "config/other/second.yml": "name: first\n---\ntype: slack\nname: n\nwebhook: https://h.example.invalid/" + planted + "c\n", + "config/other/flow.yaml": "{type: http, name: n, url: \"https://h.example.invalid/" + planted + "d\"}\n", + } + writeExportTree(t, dir, files) + archive, entries, _ := exportEntries(t, dir) + requireNoToken(t, archive) + for name := range files { + require.Contains(t, entries, name) + require.Contains(t, entries[name], RedactedPlaceholder, name) + } +} + +func TestExportArchiveKeepsNearMissesOutsideTheFoldersUnchanged(t *testing.T) { + t.Parallel() + dir := t.TempDir() + other := map[string]string{ + "config/other/key_case.yml": "Type: http\nurl: x\n", + "config/other/unknown_q.yml": "type: \"parser\"\nname: x\n", + "config/other/later_broken.yml": "type: \"http\"\n---\na: [unclosed\n", + "config/other/no_plugin_docs.yml": "type: webhook\n---\ntype: parser\n", + } + writeExportTree(t, dir, other) + _, entries, _ := exportEntries(t, dir) + for name, want := range other { + require.Equal(t, want, entries[name], name) + } +} diff --git a/backend/internal/crowdsec/export_test.go b/backend/internal/crowdsec/export_test.go index e78f7e74e..6400acf78 100644 --- a/backend/internal/crowdsec/export_test.go +++ b/backend/internal/crowdsec/export_test.go @@ -291,12 +291,12 @@ func TestAddExportFileRefusesWhatLstatRejects(t *testing.T) { "lstat fails (ENOTDIR)": filepath.Join("real.yaml", "child"), } for name, rel := range cases { - n, err := addExportFile(tw, dir, rel, 0, MaxImportUncompressedBytes, exportHooks{}) + n, err := addExportFile(tw, dir, rel, 0, MaxImportUncompressedBytes, nil, exportHooks{}) require.ErrorIs(t, err, ErrExportFileUnreadable, name) require.Zero(t, n, name) } - n, err := addExportFile(tw, dir, "missing.yaml", 0, MaxImportUncompressedBytes, exportHooks{}) + n, err := addExportFile(tw, dir, "missing.yaml", 0, MaxImportUncompressedBytes, nil, exportHooks{}) require.NoError(t, err, "a vanished entry is skipped") require.Zero(t, n) } diff --git a/backend/internal/crowdsec/notifications_files.go b/backend/internal/crowdsec/notifications_files.go new file mode 100644 index 000000000..6471cc4a2 --- /dev/null +++ b/backend/internal/crowdsec/notifications_files.go @@ -0,0 +1,260 @@ +package crowdsec + +import ( + "bytes" + "path" + "path/filepath" + "regexp" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/Wikid82/charon/backend/internal/logger" +) + +// NotificationFileKind is the class IsNotificationFile assigns to a path of a configuration tree. +type NotificationFileKind int + +const ( + // NotAFile is anything that is not treated as a notification file. + NotAFile NotificationFileKind = iota + // InDirYAML is a YAML file inside a notification directory. + InDirYAML + // InDirOther is a non-YAML file inside a notification directory; CrowdSec never reads it. + InDirOther + // LookAlikeYAML is a YAML file outside the notification directories whose root names a known + // notification plugin type. + LookAlikeYAML +) + +// knownPluginTypes are the notification plugin types CrowdSec ships. +var knownPluginTypes = map[string]struct{}{ + "dummy": {}, "email": {}, "file": {}, "http": {}, "sentinel": {}, "slack": {}, "splunk": {}, +} + +// defaultNotificationDirs are the notification directories below the data folder for the flat and the +// nested layout. +var defaultNotificationDirs = []string{"notifications", "config/notifications"} + +// contentDirs are the CrowdSec content directories of the config root. A redirected notification +// folder at or below one of them is ignored, so custom parsers and scenarios are never rewritten. +var contentDirs = map[string]struct{}{ + "parsers": {}, "scenarios": {}, "collections": {}, "postoverflows": {}, "contexts": {}, + "appsec-configs": {}, "appsec-rules": {}, "appsec-data": {}, "patterns": {}, "hub": {}, "data": {}, +} + +func hasYAMLExt(name string) bool { + switch strings.ToLower(path.Ext(name)) { + case ".yaml", ".yml": + return true + } + return false +} + +// IsNotificationFile is the single place that decides what a notification file is. rel is a slash path +// below the root being examined and dirs is NotificationDirs for that same root. docs are the parsed +// documents of the file (see LookAlikeDocuments), or nil when the caller has not parsed it; they are only +// consulted for YAML files outside the notification directories, where any one document naming a known +// plugin makes the file a look-alike. The function never reads files itself. +func IsNotificationFile(rel string, docs []*yaml.Node, dirs []string) NotificationFileKind { + rel = path.Clean(filepath.ToSlash(rel)) + if rel == "." || rel == "" || rel == ".." || strings.HasPrefix(rel, "../") || strings.HasPrefix(rel, "/") { + return NotAFile + } + if inNotificationDir(rel, dirs) { + if hasYAMLExt(rel) { + return InDirYAML + } + return InDirOther + } + if hasYAMLExt(rel) && anyNamesKnownPlugin(docs) { + return LookAlikeYAML + } + return NotAFile +} + +// inNotificationDir reports whether rel lies below a directory component named "notifications" (any +// case, any position) or below one of the exact configured directories in dirs. +func inNotificationDir(rel string, dirs []string) bool { + parts := strings.Split(rel, "/") + for _, p := range parts[:len(parts)-1] { + if strings.EqualFold(p, "notifications") { + return true + } + } + for _, d := range dirs { + if d != "" && strings.HasPrefix(rel, d+"/") { + return true + } + } + return false +} + +// LookAlikeDocuments parses data for the look-alike probe shared by export, merge and scan: every +// document, within the redactor's node bound. It returns nil when the text cannot contain a "type" key, +// when it does not parse, or when it is too large to inspect, so such a file is never a look-alike. +func LookAlikeDocuments(data []byte) []*yaml.Node { + if int64(len(data)) > maxInspectEntryBytes || !bytes.Contains(data, []byte("type")) { + return nil + } + docs, err := ParseYAMLDocuments(bytes.NewReader(data), false) + if err != nil { + return nil + } + nodes := 0 + for _, d := range docs { + nodes += countNodes(d) + if nodes > maxRedactNodes { + return nil + } + } + return docs +} + +func anyNamesKnownPlugin(docs []*yaml.Node) bool { + for _, d := range docs { + if namesKnownPlugin(d) { + return true + } + } + return false +} + +// namesKnownPlugin reports whether root is a mapping with a direct scalar string "type" (plain or +// quoted, any scalar style) naming a known plugin type. +func namesKnownPlugin(root *yaml.Node) bool { + if root == nil { + return false + } + if root.Kind == yaml.DocumentNode { + if len(root.Content) != 1 { + return false + } + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return false + } + for i := 0; i+1 < len(root.Content); i += 2 { + k, v := root.Content[i], root.Content[i+1] + if k.Kind == yaml.ScalarNode && k.Value == "type" { + if v.Kind != yaml.ScalarNode || v.Tag != "!!str" { + return false + } + _, ok := knownPluginTypes[strings.ToLower(v.Value)] + return ok + } + } + return false +} + +// NotificationDirs returns the notification directories of the tree below dataDir as slash paths +// relative to it: the two defaults plus the folder config_paths.notification_dir of configYAML names +// when it resolves safely inside the tree. A value that cannot be resolved statically or fails the +// safety rules is ignored and the defaults alone are returned. configYAML may be nil. +func NotificationDirs(dataDir string, configYAML []byte) []string { + dirs := append([]string(nil), defaultNotificationDirs...) + value, ok := configuredNotificationDir(configYAML) + if !ok { + return dirs + } + rel, ok := resolveNotificationDir(dataDir, value) + if !ok { + logger.Log().Debug("crowdsec notification folder setting ignored") + return dirs + } + for _, d := range dirs { + if d == rel { + return dirs + } + } + return append(dirs, rel) +} + +// configuredNotificationDir reads config_paths.notification_dir from the first document of configYAML. +func configuredNotificationDir(configYAML []byte) (string, bool) { + if len(configYAML) == 0 { + return "", false + } + docs, err := ParseYAMLDocuments(bytes.NewReader(configYAML), true) + if err != nil || len(docs) == 0 || len(docs[0].Content) != 1 { + if err != nil { + logger.Log().Debug("crowdsec config unreadable; using the default notification folders") + } + return "", false + } + node := mappingValue(docs[0].Content[0], "config_paths") + node = mappingValue(node, "notification_dir") + if node == nil || node.Kind != yaml.ScalarNode || node.Tag != "!!str" { + return "", false + } + return node.Value, true +} + +// resolveNotificationDir maps a configured folder to a slash path below dataDir. Nothing is expanded +// (a "$" anywhere means it cannot be resolved statically) and no link is followed. +func resolveNotificationDir(dataDir, value string) (string, bool) { + if value == "" || strings.ContainsRune(value, 0) || strings.Contains(value, "$") { + return "", false + } + value = filepath.ToSlash(value) + for _, part := range strings.Split(value, "/") { + if part == ".." { + return "", false + } + } + if !strings.HasPrefix(value, "/") { + return "", false + } + value = path.Clean(value) + root := path.Clean(filepath.ToSlash(filepath.Clean(dataDir))) + var rel string + switch { + case value == root: + return "", false + case strings.HasPrefix(value, root+"/"): + rel = strings.TrimPrefix(value, root+"/") + case (value == "/etc/crowdsec" || strings.HasPrefix(value, "/etc/crowdsec/")) && DetectLayout(dataDir) == LayoutNested: + rel = "config" + strings.TrimPrefix(value, "/etc/crowdsec") + default: + return "", false + } + if rel == "config" || rel == "" || isContentDirPath(rel) { + return "", false + } + return rel, true +} + +// isContentDirPath reports whether rel is at or below a CrowdSec content directory, directly below the +// data folder or below its config folder. +func isContentDirPath(rel string) bool { + for _, p := range []string{rel, strings.TrimPrefix(rel, "config/")} { + first, _, _ := strings.Cut(p, "/") + if _, ok := contentDirs[strings.ToLower(first)]; ok { + return true + } + } + return false +} + +// maxEnvRefBytes bounds the values the environment reference test looks at. +const maxEnvRefBytes = 4096 + +var envRefPattern = regexp.MustCompile(`^\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)$`) + +// envRefSchemes are the header schemes that may precede a reference. +var envRefSchemes = []string{"Bearer ", "Basic ", "token ", "Token "} + +// IsEnvReference reports whether v is nothing but one environment reference ("${NAME}" or "$NAME"), +// optionally behind one header scheme. A value with any other text, or longer than 4 KiB, is not one. +func IsEnvReference(v string) bool { + if len(v) > maxEnvRefBytes { + return false + } + for _, s := range envRefSchemes { + if rest, ok := strings.CutPrefix(v, s); ok { + return envRefPattern.MatchString(rest) + } + } + return envRefPattern.MatchString(v) +} diff --git a/backend/internal/crowdsec/notifications_files_test.go b/backend/internal/crowdsec/notifications_files_test.go new file mode 100644 index 000000000..91bb7e1be --- /dev/null +++ b/backend/internal/crowdsec/notifications_files_test.go @@ -0,0 +1,199 @@ +package crowdsec + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func parseDocs(t *testing.T, src string) []*yaml.Node { + t.Helper() + docs, err := ParseYAMLDocuments(strings.NewReader(src), false) + require.NoError(t, err) + return docs +} + +func TestIsNotificationFileClasses(t *testing.T) { + t.Parallel() + dirs := []string{"notifications", "config/notifications", "custom_notify"} + httpRoot := "type: http\nname: n\n" + cases := []struct { + name string + rel string + root string + want NotificationFileKind + }{ + {"default dir yaml", "notifications/http.yaml", "", InDirYAML}, + {"nested layout dir", "config/notifications/http.yml", "", InDirYAML}, + {"any depth", "config/notifications/sub/deeper/http.yaml", "", InDirYAML}, + {"case variants", "Notifications/HTTP.YAML", "", InDirYAML}, + {"notifications at any position", "alerts/Notifications/x.yaml", "", InDirYAML}, + {"hidden yaml name", "notifications/.yaml", "", InDirYAML}, + {"other file in dir", "notifications/sub/x.txt", "", InDirOther}, + {"backup file", "notifications/http.yaml.bak", "", InDirOther}, + {"no extension", "config/notifications/a/b/x", "", InDirOther}, + {"redirected dir", "custom_notify/http.yaml", "", InDirYAML}, + {"redirected dir other", "custom_notify/x.txt", "", InDirOther}, + {"redirected dir is case sensitive", "Custom_Notify/http.yaml", "", NotAFile}, + {"name merely starts with notifications", "notifications.bak/x.yaml", "", NotAFile}, + {"dash suffix", "notifications-old/x.yaml", "", NotAFile}, + {"look alike by type", "notifications.bak/x.yaml", httpRoot, LookAlikeYAML}, + {"look alike type case", "other/x.YML", "type: Slack\n", LookAlikeYAML}, + {"look alike via document node", "x.yaml", httpRoot, LookAlikeYAML}, + {"unknown type", "x.yaml", "type: webhook\n", NotAFile}, + {"typeless", "config/config.yaml", "api:\n server: {}\n", NotAFile}, + {"double quoted type", "x.yaml", "type: \"http\"\n", LookAlikeYAML}, + {"single quoted type", "x.yaml", "type: 'Slack'\n", LookAlikeYAML}, + {"quoted type key", "x.yaml", "\"type\": http\n", LookAlikeYAML}, + {"flow style mapping", "x.yaml", "{type: http, name: n}\n", LookAlikeYAML}, + {"second document names a plugin", "x.yaml", "name: a\n---\ntype: splunk\n", LookAlikeYAML}, + {"later document after a non mapping", "x.yaml", "- a\n---\ntype: \"email\"\n", LookAlikeYAML}, + {"no document names a plugin", "x.yaml", "type: webhook\n---\ntype: parser\n", NotAFile}, + {"key case is exact", "x.yaml", "Type: http\n", NotAFile}, + {"quoted unknown type", "x.yaml", "type: \"webhook\"\n", NotAFile}, + {"explicit non string tag", "x.yaml", "type: !!binary aHR0cA==\n", NotAFile}, + {"unparsable later document", "x.yaml", "type: http\n---\na: [\n", NotAFile}, + {"aliased type", "x.yaml", "t: &t http\ntype: *t\n", NotAFile}, + {"non scalar type", "x.yaml", "type: [http]\n", NotAFile}, + {"non mapping root", "x.yaml", "- type: http\n", NotAFile}, + {"yaml look alike needs a yaml extension", "x.txt", httpRoot, NotAFile}, + {"unparsed", "x.yaml", "", NotAFile}, + {"dot", ".", "", NotAFile}, + {"escaping path", "../notifications/x.yaml", "", NotAFile}, + {"absolute path", "/notifications/x.yaml", "", NotAFile}, + {"unclean path", "./notifications//x.yaml", "", InDirYAML}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var docs []*yaml.Node + if tc.root != "" && !strings.Contains(tc.name, "unparsable") { + docs = parseDocs(t, tc.root) + } else if tc.root != "" { + require.Nil(t, LookAlikeDocuments([]byte(tc.root))) + } + require.Equal(t, tc.want, IsNotificationFile(tc.rel, docs, dirs)) + }) + } +} + +func TestIsNotificationFileEmptyDocument(t *testing.T) { + t.Parallel() + require.Equal(t, NotAFile, IsNotificationFile("x.yaml", []*yaml.Node{{Kind: yaml.DocumentNode}}, nil)) +} + +func writeTree(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for rel, body := range files { + p := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(t, os.WriteFile(p, []byte(body), 0o600)) + } + return dir +} + +func cfgWithDir(v string) []byte { + return []byte("config_paths:\n notification_dir: " + v + "\n") +} + +func TestNotificationDirsResolver(t *testing.T) { + t.Parallel() + defaults := []string{"notifications", "config/notifications"} + nested := map[string]string{"config/config.yaml": "x: 1\n"} + cases := []struct { + name string + files map[string]string + cfg func(root string) []byte + want []string + }{ + {"no config", nested, func(string) []byte { return nil }, defaults}, + {"unparsable config", nested, func(string) []byte { return []byte("a: [\n") }, defaults}, + {"empty stream", nested, func(string) []byte { return []byte("# nothing\n") }, defaults}, + {"no setting", nested, func(string) []byte { return []byte("api: {}\n") }, defaults}, + {"non string setting", nested, func(string) []byte { return []byte("config_paths:\n notification_dir: [a]\n") }, defaults}, + {"default value", nested, func(string) []byte { return cfgWithDir("/etc/crowdsec/notifications/") }, defaults}, + {"nested mapping of etc path", nested, func(string) []byte { return cfgWithDir("/etc/crowdsec/alerts/") }, append(append([]string{}, defaults...), "config/alerts")}, + {"flat layout does not map etc path", nil, func(string) []byte { return cfgWithDir("/etc/crowdsec/alerts") }, defaults}, + {"path inside the data folder", nested, func(r string) []byte { return cfgWithDir(r + "/custom_notify") }, append(append([]string{}, defaults...), "custom_notify")}, + {"path inside the data folder below config", nested, func(r string) []byte { return cfgWithDir(r + "/config/x/y") }, append(append([]string{}, defaults...), "config/x/y")}, + {"outside the tree", nested, func(string) []byte { return cfgWithDir("/var/lib/other") }, defaults}, + {"relative", nested, func(string) []byte { return cfgWithDir("alerts") }, defaults}, + {"relative with dotdot", nested, func(string) []byte { return cfgWithDir("../alerts") }, defaults}, + {"dotdot inside absolute", nested, func(r string) []byte { return cfgWithDir(r + "/a/../../etc") }, defaults}, + {"empty value", nested, func(string) []byte { return cfgWithDir(`""`) }, defaults}, + {"nul byte", nested, func(r string) []byte { return cfgWithDir(`"` + r + `/a\0b"`) }, defaults}, + {"env value braces", nested, func(string) []byte { return cfgWithDir(`"${DIR}/x"`) }, defaults}, + {"env value plain", nested, func(r string) []byte { return cfgWithDir(`"` + r + `/$DIR"`) }, defaults}, + {"equals data folder", nested, cfgWithDir, defaults}, + {"ancestor of the data folder", nested, func(r string) []byte { return cfgWithDir(filepath.Dir(r)) }, defaults}, + {"equals config root", nested, func(r string) []byte { return cfgWithDir(r + "/config") }, defaults}, + {"equals etc root", nested, func(string) []byte { return cfgWithDir("/etc/crowdsec") }, defaults}, + {"flat config root is the data folder", nil, func(r string) []byte { return cfgWithDir(r + "/") }, defaults}, + } + for _, d := range []string{"parsers", "scenarios", "collections", "postoverflows", "contexts", "appsec-configs", "appsec-rules", "appsec-data", "patterns", "hub", "data", "Parsers"} { + cases = append(cases, + struct { + name string + files map[string]string + cfg func(root string) []byte + want []string + }{"content dir " + d, nested, func(string) []byte { return cfgWithDir("/etc/crowdsec/" + d + "/sub") }, defaults}, + struct { + name string + files map[string]string + cfg func(root string) []byte + want []string + }{"content dir at top " + d, nested, func(r string) []byte { return cfgWithDir(r + "/" + d) }, defaults}, + ) + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + root := writeTree(t, tc.files) + if tc.files == nil { + require.NoError(t, os.MkdirAll(root, 0o750)) + } + require.Equal(t, tc.want, NotificationDirs(root, tc.cfg(root))) + }) + } +} + +func TestNotificationDirsDoesNotMutateDefaults(t *testing.T) { + t.Parallel() + root := writeTree(t, map[string]string{"config/config.yaml": "x: 1\n"}) + _ = NotificationDirs(root, cfgWithDir(root+"/extra")) + require.Equal(t, []string{"notifications", "config/notifications"}, NotificationDirs(root, nil)) +} + +func TestIsEnvReference(t *testing.T) { + t.Parallel() + yes := []string{"${TOKEN}", "$TOKEN", "$_x1", "Bearer ${TOKEN}", "Basic $CREDS", "token ${T}", "Token $T"} + no := []string{ + "", "$", "${}", "${1A}", "$1A", "plain", "https://hooks.example/${X}/abc", "${A}${B}", "${A} ", " ${A}", + "bearer ${A}", "Bearer ${A}", "Bearer Bearer ${A}", "Bearer ", "$A\n", "${A", "$A-b", + "Bearer " + strings.Repeat("a", 5000), "${" + strings.Repeat("A", 5000) + "}", + } + for _, v := range yes { + require.True(t, IsEnvReference(v), v) + } + for _, v := range no { + require.False(t, IsEnvReference(v), v) + } + require.False(t, IsEnvReference("$"+strings.Repeat("A", maxEnvRefBytes))) + require.True(t, IsEnvReference("$"+strings.Repeat("A", maxEnvRefBytes-1))) +} + +func TestLookAlikeDocumentsBounds(t *testing.T) { + t.Parallel() + require.Nil(t, LookAlikeDocuments([]byte("name: x\nurl: y\n")), "no type text: not parsed") + require.Nil(t, LookAlikeDocuments(nil)) + require.Nil(t, LookAlikeDocuments([]byte("type: [\n"))) + require.Nil(t, LookAlikeDocuments([]byte("type: http\n"+strings.Repeat("#", int(maxInspectEntryBytes))+"\n"))) + require.Nil(t, LookAlikeDocuments([]byte("type: x\nlist: ["+strings.Repeat("1,", maxRedactNodes)+"1]\n"))) + require.Len(t, LookAlikeDocuments([]byte("a: 1\n---\ntype: http\n")), 2) +} diff --git a/backend/internal/crowdsec/redact.go b/backend/internal/crowdsec/redact.go new file mode 100644 index 000000000..9c384cb7b --- /dev/null +++ b/backend/internal/crowdsec/redact.go @@ -0,0 +1,431 @@ +package crowdsec + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// RedactedPlaceholder replaces every sensitive value of a notification file. It is a plain YAML +// scalar without "$", so CrowdSec's environment expansion leaves it alone. +const RedactedPlaceholder = "" + +// redactHeaderComment is the only comment a rewritten file carries, and only while it holds at least +// one placeholder. +const redactHeaderComment = "Sensitive values were replaced by Charon when this configuration was exported." + +// RedactStubContent is what an exported notification file that cannot be rewritten safely holds: no +// YAML document at all. +const RedactStubContent = "# This file could not be read safely and was left blank by Charon.\n" + +const ( + // maxRedactNodes bounds the decoded node count of one file. + maxRedactNodes = 50_000 + // minCanaryLen is the shortest replaced value the residual scan looks for. + minCanaryLen = 8 + // maxCanaryWork bounds values x output bytes scanned by the residual check. + maxCanaryWork = 256 << 20 +) + +// ErrRedactUnsafe is returned when a notification file cannot be rewritten safely. It carries no +// file content. +var ErrRedactUnsafe = errors.New("notification file cannot be redacted safely") + +// RedactResult is the rewritten file. +type RedactResult struct { + // Data is the redacted file; it never holds an input value classified as sensitive. + Data []byte + // Redacted is the number of placeholder scalars in Data (replaced now or already present). Zero + // means the output holds no placeholder. + Redacted int +} + +// IsPlaceholder reports whether n is the placeholder scalar. +func IsPlaceholder(n *yaml.Node) bool { + return n != nil && n.Kind == yaml.ScalarNode && n.Tag == "!!str" && n.Value == RedactedPlaceholder +} + +func unsafeRedact(reason string) error { + return fmt.Errorf("%w: %s", ErrRedactUnsafe, reason) +} + +// shape is how a kept key's value may look. +type shape int + +const ( + shapeScalar shape = iota + shapeScalarList + shapeHeaders + shapeRotate +) + +// keepSet is the allowlist of one mapping. Every key outside it has its whole value replaced. +type keepSet struct { + keys map[string]shape + // headers keeps every key (header names) and redacts every value. + headers bool +} + +var ( + headersKeep = &keepSet{headers: true} + rotateKeep = &keepSet{keys: scalarKeys("enabled", "max_size", "max_files", "max_age", "compress")} + + commonKeepKeys = []string{"type", "name", "log_level", "group_wait", "group_threshold", "max_retry", "timeout", "format"} + + // keepSets maps a plugin type to its allowlist. Keys that hold credentials are deliberately + // absent; the table is documentation of what is safe, not of what is secret. + keepSets = map[string]*keepSet{ + "dummy": pluginKeep(nil, "output_file"), + "email": pluginKeep(map[string]shape{"receiver_emails": shapeScalarList}, + "smtp_host", "smtp_port", "sender_email", "sender_name", "email_subject", "encryption_type", + "auth_type", "helo_host", "connect_timeout", "send_timeout"), + "file": pluginKeep(map[string]shape{"rotate": shapeRotate}, "log_path"), + "http": pluginKeep(map[string]shape{"headers": shapeHeaders}, + "unix_socket", "skip_tls_verification", "method", "cert_path", "key_path", "ca_cert_path"), + "sentinel": pluginKeep(nil, "log_type"), + "slack": pluginKeep(nil, "channel", "username", "icon_emoji"), + "splunk": pluginKeep(nil), + } + commonOnlyKeep = pluginKeep(nil) +) + +func scalarKeys(names ...string) map[string]shape { + m := make(map[string]shape, len(names)) + for _, n := range names { + m[n] = shapeScalar + } + return m +} + +func pluginKeep(special map[string]shape, scalars ...string) *keepSet { + m := scalarKeys(commonKeepKeys...) + for _, n := range scalars { + m[n] = shapeScalar + } + for k, s := range special { + m[k] = s + } + return &keepSet{keys: m} +} + +type visitKey struct { + n *yaml.Node + ks *keepSet +} + +// redactor rewrites document trees in place. Replacement is monotonic: a node once replaced stays +// replaced, so a node reachable through several paths ends up redacted if any path calls for it. +type redactor struct { + visited map[visitKey]struct{} + replaced []string +} + +// RedactNotificationFile rewrites a CrowdSec notification file so it keeps its structure but not its +// sensitive values. Everything not known to be harmless is replaced (fail closed). Comments are +// dropped, tags are cleared and anchors are renamed. On any doubt it returns ErrRedactUnsafe and the +// caller must not fall back to src. +func RedactNotificationFile(src []byte) (RedactResult, error) { + if int64(len(src)) > maxInspectEntryBytes { + return RedactResult{}, unsafeRedact("file too large") + } + docs, parseErr := ParseYAMLDocuments(bytes.NewReader(src), false) + if parseErr != nil { + return RedactResult{}, unsafeRedact("file cannot be parsed") + } + if len(docs) == 0 { + return RedactResult{Data: []byte{}}, nil + } + nodes := 0 + for _, d := range docs { + nodes += countNodes(d) + if nodes > maxRedactNodes { + return RedactResult{}, unsafeRedact("too many nodes") + } + } + + r := &redactor{visited: map[visitKey]struct{}{}} + for _, d := range docs { + r.document(d) + } + for _, d := range docs { + if err := scrub(d); err != nil { + return RedactResult{}, err + } + } + count := 0 + for _, d := range docs { + count += countPlaceholders(d) + } + if count > 0 && len(docs) > 0 { + docs[0].HeadComment = "# " + redactHeaderComment + } + + out, encodeErr := encodeDocuments(docs) + if encodeErr != nil { + return RedactResult{}, encodeErr + } + if err := checkResidue(docs, r.replaced, out); err != nil { + return RedactResult{}, err + } + if _, err := ParseYAMLDocuments(bytes.NewReader(out), false); err != nil { + return RedactResult{}, unsafeRedact("output does not parse") + } + return RedactResult{Data: out, Redacted: count}, nil +} + +func countNodes(n *yaml.Node) int { + total := 1 + for _, c := range n.Content { + total += countNodes(c) + } + return total +} + +func countPlaceholders(n *yaml.Node) int { + total := 0 + if n.Kind == yaml.ScalarNode && n.Value == RedactedPlaceholder { + total++ + } + for _, c := range n.Content { + total += countPlaceholders(c) + } + return total +} + +// document redacts one document. A root that is not a mapping is replaced as a whole. +func (r *redactor) document(doc *yaml.Node) { + if len(doc.Content) != 1 { + return + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + r.value(root, false) + return + } + r.mapping(root, keepSetFor(root)) +} + +// keepSetFor selects the allowlist from a direct, plain, non-aliased string "type" of the root. +func keepSetFor(root *yaml.Node) *keepSet { + for i := 0; i+1 < len(root.Content); i += 2 { + k, v := root.Content[i], root.Content[i+1] + if k.Kind != yaml.ScalarNode || k.Value != "type" || k.Tag != "!!str" { + continue + } + if v.Kind == yaml.ScalarNode && v.Tag == "!!str" && v.Style == 0 { + if ks, ok := keepSets[v.Value]; ok { + return ks + } + } + break + } + return commonOnlyKeep +} + +func (r *redactor) mapping(m *yaml.Node, ks *keepSet) { + vk := visitKey{m, ks} + if _, seen := r.visited[vk]; seen { + return + } + r.visited[vk] = struct{}{} + for i := 0; i+1 < len(m.Content); i += 2 { + k, v := m.Content[i], m.Content[i+1] + if k.Tag == "!!merge" { + r.merge(v, ks) + continue + } + if ks.headers { + r.value(v, true) + continue + } + sh, ok := ks.keys[k.Value] + if k.Kind != yaml.ScalarNode || !ok { + r.value(v, true) + continue + } + r.keep(v, sh) + } +} + +// merge walks the mapping(s) a merge key pulls in within the merging mapping's context. +func (r *redactor) merge(v *yaml.Node, ks *keepSet) { + switch target := resolveAlias(v); target.Kind { + case yaml.MappingNode: + r.mapping(target, ks) + case yaml.SequenceNode: + for _, item := range target.Content { + if m := resolveAlias(item); m.Kind == yaml.MappingNode { + r.mapping(m, ks) + } else { + r.value(item, true) + } + } + default: + r.value(v, true) + } +} + +// keep applies the expected shape of a kept key; any other shape is replaced as a whole. +func (r *redactor) keep(v *yaml.Node, sh shape) { + switch sh { + case shapeScalar: + if resolveAlias(v).Kind == yaml.ScalarNode { + return + } + case shapeScalarList: + if v.Kind == yaml.SequenceNode && allScalars(v.Content) { + return + } + case shapeHeaders: + if v.Kind == yaml.MappingNode { + r.mapping(v, headersKeep) + return + } + case shapeRotate: + if v.Kind == yaml.MappingNode { + r.mapping(v, rotateKeep) + return + } + } + r.value(v, true) +} + +func allScalars(nodes []*yaml.Node) bool { + for _, n := range nodes { + if n.Kind != yaml.ScalarNode { + return false + } + } + return true +} + +// value replaces a node in a redacted position. A mapping or sequence becomes one placeholder scalar, +// so no nested key or value survives. Kept as they are: a truly empty null, a placeholder already +// present and (with allowEnv) a pure environment reference. +func (r *redactor) value(n *yaml.Node, allowEnv bool) { + switch n.Kind { + case yaml.ScalarNode: + if n.Value == "" && n.Tag == "!!null" { + return + } + if n.Value == RedactedPlaceholder || (allowEnv && IsEnvReference(n.Value)) { + return + } + r.note(n.Value) + case yaml.MappingNode, yaml.SequenceNode: + r.noteSubtree(n) + } + setPlaceholder(n) +} + +func setPlaceholder(n *yaml.Node) { + n.Kind = yaml.ScalarNode + n.Content = nil + n.Alias = nil + n.Tag = "!!str" + n.Value = RedactedPlaceholder + n.Style = 0 +} + +// note remembers a replaced value for the residual scan. +func (r *redactor) note(v string) { + if len(v) >= minCanaryLen && v != RedactedPlaceholder { + r.replaced = append(r.replaced, v) + } +} + +func (r *redactor) noteSubtree(n *yaml.Node) { + if n.Kind == yaml.ScalarNode { + r.note(n.Value) + } + for _, c := range n.Content { + r.noteSubtree(c) + } +} + +// scrub clears comments and tags on every node and renames anchors to a1, a2, ... in document order, +// rewriting alias names to match. An alias whose anchor is no longer part of the document (its +// definition was replaced) cannot be written, so the file is unsafe. +func scrub(doc *yaml.Node) error { + names := map[*yaml.Node]string{} + var walk func(n *yaml.Node) error + walk = func(n *yaml.Node) error { + n.HeadComment, n.LineComment, n.FootComment = "", "", "" + n.Tag = "" + if n.Anchor != "" { + n.Anchor = fmt.Sprintf("a%d", len(names)+1) + names[n] = n.Anchor + } + if n.Kind == yaml.AliasNode { + name, ok := names[n.Alias] + if !ok { + return unsafeRedact("alias without definition") + } + n.Value = name + } + for _, c := range n.Content { + if err := walk(c); err != nil { + return err + } + } + return nil + } + return walk(doc) +} + +func encodeDocuments(docs []*yaml.Node) ([]byte, error) { + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + for _, d := range docs { + if err := enc.Encode(d); err != nil { + return nil, unsafeRedact("output cannot be encoded") + } + } + if err := enc.Close(); err != nil { + return nil, unsafeRedact("output cannot be encoded") + } + return buf.Bytes(), nil +} + +// checkResidue is the last line of defence: no replaced value of at least minCanaryLen characters may +// appear in the encoded output. Values that equal a scalar kept in the output (for example a sender +// address that is also the login name) and the text Charon writes itself are not leaks. +func checkResidue(docs []*yaml.Node, replaced []string, out []byte) error { + if len(replaced) == 0 { + return nil + } + kept := map[string]struct{}{ + RedactedPlaceholder: {}, redactHeaderComment: {}, strings.TrimSpace(RedactStubContent): {}, + } + for _, d := range docs { + collectScalars(d, kept) + } + unique := map[string]struct{}{} + for _, v := range replaced { + if _, ok := kept[v]; !ok { + unique[v] = struct{}{} + } + } + if len(unique)*len(out) > maxCanaryWork { + return unsafeRedact("output too large to verify") + } + for v := range unique { + if bytes.Contains(out, []byte(v)) { + return unsafeRedact("residual value in output") + } + } + return nil +} + +func collectScalars(n *yaml.Node, into map[string]struct{}) { + if n.Kind == yaml.ScalarNode { + into[n.Value] = struct{}{} + } + for _, c := range n.Content { + collectScalars(c, into) + } +} diff --git a/backend/internal/crowdsec/redact_merge.go b/backend/internal/crowdsec/redact_merge.go new file mode 100644 index 000000000..13e4f6411 --- /dev/null +++ b/backend/internal/crowdsec/redact_merge.go @@ -0,0 +1,452 @@ +package crowdsec + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/Wikid82/charon/backend/internal/logger" + "github.com/Wikid82/charon/backend/internal/util" +) + +// maxNeedsValues caps the number of file names ScanPlaceholders reports. +const maxNeedsValues = 50 + +// targetKeys lists, per plugin type, the keys that say where a notification goes. Types missing here are +// treated as unknown: every key outside commonKeepKeys counts. +var targetKeys = map[string][]string{ + "http": {"url", "unix_socket", "skip_tls_verification", "ca_cert_path", "cert_path", "key_path"}, + "slack": {"webhook"}, + "splunk": {"url"}, + "email": {"smtp_host", "smtp_port", "encryption_type", "auth_type", "helo_host"}, + "sentinel": {"customer_id"}, +} + +// notificationFile is a notification file of a tree: its parsed documents and the permission bits it +// had, capped at 0600. +type notificationFile struct { + rel string // slash path below the tree root + docs []*yaml.Node + mode os.FileMode +} + +// MergeRedactedNotifications fills placeholders in the freshly extracted notification files of dataDir +// from the running system's files in the snapshot taken before the import. It only merges (and rewrites +// the files it changed); it does not report, see ScanPlaceholders. +// +// A document is merged only when its type and name match exactly one document of the snapshot and its +// target is unchanged (see sameTarget). Documents that use aliases or merge keys are never merged. Files +// that need no merge stay byte-identical. +func MergeRedactedNotifications(snapshotDir, dataDir string) error { + archive, err := collectNotificationFiles(dataDir, treeNotificationDirs(dataDir)) + if err != nil { + return fmt.Errorf("read the imported notification files: %w", err) + } + var pending []notificationFile + for _, f := range archive { + if filePlaceholders(f) > 0 { + pending = append(pending, f) + } + } + if len(pending) == 0 { + return nil + } + live, err := collectNotificationFiles(snapshotDir, treeNotificationDirs(snapshotDir)) + if err != nil { + return fmt.Errorf("read the current notification files: %w", err) + } + index := indexDocuments(live) + + merged, remaining := 0, 0 + for _, f := range pending { + mode, changed := os.FileMode(0o600), false + for _, doc := range f.docs { + if src, ok := matchLiveDocument(doc, index); ok && fillDocument(doc, src.root) { + changed = true + mode &= src.mode + } + } + if !changed { + remaining++ + continue + } + if filePlaceholders(f) == 0 { + dropHeaderComment(f.docs) + } else { + remaining++ + } + if err := rewriteNotificationFile(dataDir, f, mode); err != nil { + return err + } + merged++ + } + logger.Log().Infof("crowdsec import: %d notification files merged, %d still need values", merged, remaining) + return nil +} + +// ScanPlaceholders returns the notification files of the final tree below dataDir that still hold +// placeholders, named by their path below dataDir (a leading "config/" is removed in the nested layout, +// as for kept files): sorted, de-duplicated and capped at 50. It selects files exactly like the merge. +func ScanPlaceholders(dataDir string) ([]string, error) { + files, err := collectNotificationFiles(dataDir, treeNotificationDirs(dataDir)) + if err != nil { + return nil, fmt.Errorf("scan notification files: %w", err) + } + nested := DetectLayout(dataDir) == LayoutNested + seen := map[string]struct{}{} + for _, f := range files { + if filePlaceholders(f) == 0 { + continue + } + name := f.rel + if nested { + name = strings.TrimPrefix(name, "config/") + } + seen[name] = struct{}{} + } + names := make([]string, 0, len(seen)) + for n := range seen { + names = append(names, n) + } + sort.Strings(names) + if len(names) > maxNeedsValues { + names = names[:maxNeedsValues] + } + return names, nil +} + +// collectNotificationFiles walks root and returns every notification file the shared selector picks +// (IsNotificationFile), parsed. Hub and preserved state are skipped, links are never followed, and a file +// that is too large or does not parse is skipped. +func collectNotificationFiles(root string, dirs []string) ([]notificationFile, error) { + var files []notificationFile + walkErr := fs.WalkDir(os.DirFS(root), ".", func(rel string, d fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if rel == "." { + return nil + } + if IsExportExcluded(rel) { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + kind := IsNotificationFile(rel, nil, dirs) + if kind == InDirOther || (kind == NotAFile && !hasYAMLExt(rel)) { + return nil + } + if len(files) >= maxInspectEntries { + logger.Log().Warn("crowdsec import: too many notification files; the rest are left as they are") + return fs.SkipAll + } + file, ok, readErr := readNotificationFile(root, rel, kind, dirs) + if readErr != nil { + return readErr + } + if ok { + files = append(files, file) + } + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("walk notification files: %w", withoutPath(walkErr)) + } + return files, nil +} + +// readNotificationFile reads and parses one candidate file. It reports false for a file that is gone, +// too large, unparsable, or (outside the notification folders) not shaped like a notification. +func readNotificationFile(root, rel string, kind NotificationFileKind, dirs []string) (notificationFile, bool, error) { + f, info, err := openExportFile(root, filepath.FromSlash(rel), exportHooks{}) + if f == nil { // err is nil when the file vanished or was swapped for a link + return notificationFile{}, false, err + } + defer func() { _ = f.Close() }() + data, err := io.ReadAll(io.LimitReader(f, maxInspectEntryBytes+1)) + if err != nil { + return notificationFile{}, false, unreadableFile(rel, err) + } + if int64(len(data)) > maxInspectEntryBytes { + logger.Log().Debugf("crowdsec import: %s is too large to examine", util.SanitizeForLog(rel)) + return notificationFile{}, false, nil + } + if kind == NotAFile && IsNotificationFile(rel, LookAlikeDocuments(data), dirs) != LookAlikeYAML { + return notificationFile{}, false, nil + } + docs, parseErr := ParseYAMLDocuments(bytes.NewReader(data), false) + if parseErr != nil { + logger.Log().Debugf("crowdsec import: %s does not parse; leaving it alone", util.SanitizeForLog(rel)) + return notificationFile{}, false, nil + } + nodes := 0 + for _, d := range docs { + nodes += countNodes(d) + } + if nodes > maxRedactNodes { + return notificationFile{}, false, nil + } + return notificationFile{rel: rel, docs: docs, mode: info.Mode().Perm() & 0o600}, true, nil +} + +// liveDocument is a root mapping of the snapshot with the mode of the file it came from. +type liveDocument struct { + root *yaml.Node + mode os.FileMode +} + +// indexDocuments groups the snapshot's documents by (type, name). A name defined more than once is kept +// as a list so the caller can tell it is ambiguous. +func indexDocuments(files []notificationFile) map[string][]liveDocument { + index := map[string][]liveDocument{} + for _, f := range files { + for _, doc := range f.docs { + root := documentRoot(doc) + if key, ok := identityOf(root); ok { + index[key] = append(index[key], liveDocument{root: root, mode: f.mode}) + } + } + } + return index +} + +// matchLiveDocument returns the single snapshot document an archive document may be filled from. Several +// definitions of the name, a merge key in the snapshot document, or a different shape give no match. +func matchLiveDocument(doc *yaml.Node, index map[string][]liveDocument) (liveDocument, bool) { + root := documentRoot(doc) + key, ok := identityOf(root) + if !ok || hasAliasOrMerge(doc) { + return liveDocument{}, false + } + candidates := index[key] + if len(candidates) != 1 || hasMergeKey(candidates[0].root) { + return liveDocument{}, false + } + return candidates[0], true +} + +// fillDocument fills the placeholders of an archive document from live when the target is unchanged. It +// reports whether anything was filled. +func fillDocument(doc, live *yaml.Node) bool { + root := documentRoot(doc) + if !sameTarget(root, live) { + return false + } + return fillPlaceholders(root, live) +} + +// documentRoot returns the root mapping of a document node, or nil. +func documentRoot(doc *yaml.Node) *yaml.Node { + if doc == nil || doc.Kind != yaml.DocumentNode || len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode { + return nil + } + return doc.Content[0] +} + +// identityOf returns "type\x00name" of a root mapping when both are plain strings. +func identityOf(root *yaml.Node) (string, bool) { + typ, ok1 := plainString(rawValue(root, "type")) + name, ok2 := plainString(rawValue(root, "name")) + if !ok1 || !ok2 { + return "", false + } + return typ + "\x00" + name, true +} + +func plainString(n *yaml.Node) (string, bool) { + if n == nil || n.Kind != yaml.ScalarNode || n.Tag != "!!str" { + return "", false + } + return n.Value, true +} + +// rawValue returns the value node stored under key in m without resolving aliases (nil when absent). +func rawValue(m *yaml.Node, key string) *yaml.Node { + if m == nil || m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if k := m.Content[i]; k.Kind == yaml.ScalarNode && k.Value == key { + return m.Content[i+1] + } + } + return nil +} + +// hasMergeKey reports whether the root mapping uses a merge key. +func hasMergeKey(root *yaml.Node) bool { + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Tag == "!!merge" { + return true + } + } + return false +} + +// hasAliasOrMerge reports whether the tree holds an alias node or a merge key anywhere. Such documents +// are never merged: a value written into a shared node would change every place that refers to it. +func hasAliasOrMerge(n *yaml.Node) bool { + if n.Kind == yaml.AliasNode || n.Tag == "!!merge" { + return true + } + return slices.ContainsFunc(n.Content, hasAliasOrMerge) +} + +// sameTarget reports whether the archive document still points where the current one does: only merge +// when the target is unchanged. For every key that says where the notification goes, both documents must +// lack it, or both must have a plain value that is either the placeholder (archive side) or equal to the +// current one. The comparison is deliberately exact (tag and text): no trimming, case folding, URL +// cleanup or number/string equivalence, because any such rule would make two different targets compare +// equal. It runs on the archive values as extracted, before anything is filled in. +func sameTarget(archive, live *yaml.Node) bool { + for _, key := range targetKeysOf(archive, live) { + a, l := rawValue(archive, key), rawValue(live, key) + switch { + case a == nil && l == nil: + continue + case a == nil || l == nil: + return false + case a.Kind != yaml.ScalarNode || l.Kind != yaml.ScalarNode: + return false + case IsPlaceholder(a): + continue + case a.Tag != l.Tag || a.Value != l.Value: + return false + } + } + return true +} + +// targetKeysOf returns the keys sameTarget compares: the fixed list of a known type, otherwise every key +// of either document outside the common ones. +func targetKeysOf(archive, live *yaml.Node) []string { + typ, _ := plainString(rawValue(archive, "type")) + if keys, ok := targetKeys[typ]; ok { + return keys + } + var keys []string + for _, m := range []*yaml.Node{archive, live} { + for i := 0; i+1 < len(m.Content); i += 2 { + k := m.Content[i].Value + if !slices.Contains(commonKeepKeys, k) && !slices.Contains(keys, k) { + keys = append(keys, k) + } + } + } + return keys +} + +// fillPlaceholders copies the current plain value into every placeholder of a that has a plain, +// non-placeholder counterpart at the same position in l. Values are copied into nodes, never spliced into +// text. Archive values that are not placeholders always win, and keys only l has are not added. +func fillPlaceholders(a, l *yaml.Node) bool { + if a == nil || l == nil { + return false + } + changed := false + switch a.Kind { + case yaml.ScalarNode: + if IsPlaceholder(a) && l.Kind == yaml.ScalarNode && !IsPlaceholder(l) { + a.Value, a.Tag, a.Style = l.Value, l.Tag, l.Style + changed = true + } + case yaml.MappingNode: + for i := 0; i+1 < len(a.Content); i += 2 { + if fillPlaceholders(a.Content[i+1], rawValue(l, a.Content[i].Value)) { + changed = true + } + } + case yaml.SequenceNode: + if l.Kind != yaml.SequenceNode { + return false + } + for i := 0; i < len(a.Content) && i < len(l.Content); i++ { + if fillPlaceholders(a.Content[i], l.Content[i]) { + changed = true + } + } + } + return changed +} + +func filePlaceholders(f notificationFile) int { + total := 0 + for _, d := range f.docs { + total += countPlaceholders(d) + } + return total +} + +// dropHeaderComment removes the comment Charon adds to files that hold placeholders. +func dropHeaderComment(docs []*yaml.Node) { + if len(docs) == 0 { + return + } + header := "# " + redactHeaderComment + // A comment above the first line is attached to the document, the root mapping or its first key. + candidates := []*yaml.Node{docs[0], documentRoot(docs[0])} + if root := candidates[1]; root != nil && len(root.Content) > 0 { + candidates = append(candidates, root.Content[0]) + } + for _, n := range candidates { + if n != nil && strings.TrimSpace(n.HeadComment) == header { + n.HeadComment = "" + } + } +} + +// rewriteNotificationFile replaces the extracted file with the merged documents. It writes a new file +// next to the old one (created exclusively, never through a link) and renames it over the old one, so a +// link in the path is never followed and the permission bits are the ones given, whatever the archive +// entry had. +func rewriteNotificationFile(root string, f notificationFile, mode os.FileMode) error { + out, err := encodeDocuments(f.docs) + if err != nil { + return fmt.Errorf("encode merged notification file: %w", err) + } + dest := filepath.Join(filepath.Clean(root), filepath.FromSlash(f.rel)) + if _, err = checkNoSymlinkBelow(root, dest); err != nil { + return fmt.Errorf("write merged notification file: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(dest), ".charon-merge-*") + if err != nil { + return fmt.Errorf("create merged notification file: %w", withoutPath(err)) + } + if err = writeAndClose(tmp, out, mode); err == nil { + err = os.Rename(tmp.Name(), dest) + } + if err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("write merged notification file: %w", withoutPath(err)) + } + return nil +} + +// writeAndClose sets mode on the open file, writes data and closes it. +func writeAndClose(f *os.File, data []byte, mode os.FileMode) error { + if err := f.Chmod(mode); err != nil { + _ = f.Close() + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/backend/internal/crowdsec/redact_merge_test.go b/backend/internal/crowdsec/redact_merge_test.go new file mode 100644 index 000000000..48cb80277 --- /dev/null +++ b/backend/internal/crowdsec/redact_merge_test.go @@ -0,0 +1,785 @@ +package crowdsec + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/Wikid82/charon/backend/internal/logger" +) + +const ( + liveMarker = "LIVE-SECRET-1234" + headerLine = "# " + redactHeaderComment + mergeConfig = "api:\n server:\n listen_uri: 0.0.0.0:8080\n" +) + +// secretKeys is the non-target key each plugin type uses to carry the value that gets merged. +var secretKeys = map[string]string{ + "http": "headers:\n Authorization", "slack": "icon_url", "splunk": "token", "email": "smtp_password", + "sentinel": "shared_key", "mystery": "secret", +} + +// mergeDoc builds one notification document of the given type with extra lines and a secret line. +func mergeDoc(typ, secret string, lines ...string) string { + body := "type: " + typ + "\nname: hook\n" + for _, l := range lines { + body += l + "\n" + } + return body + secretKeys[typ] + ": " + secret + "\n" +} + +func blankedFile(typ string, lines ...string) string { + return headerLine + "\n" + mergeDoc(typ, RedactedPlaceholder, lines...) +} + +// mergeTrees returns a snapshot and a data folder holding the given files. +func mergeTrees(t *testing.T, live, archive map[string]string) (snap, data string) { + t.Helper() + return writeTree(t, live), writeTree(t, archive) +} + +func relPath(root, rel string) string { return filepath.Join(root, filepath.FromSlash(rel)) } + +func TestMergeKeepsCurrentValuesForBlankedFile(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker, "url: https://hooks.example.invalid/a")}, + map[string]string{"notifications/hook.yaml": blankedFile("http", "url: https://hooks.example.invalid/a")}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + got := readFile(t, relPath(data, "notifications/hook.yaml")) + require.Contains(t, got, liveMarker) + require.NotContains(t, got, RedactedPlaceholder) + require.NotContains(t, got, redactHeaderComment) +} + +func TestMergeDecidesByTargetOfTheNotification(t *testing.T) { + t.Parallel() + const a = "url: https://hooks.example.invalid/a" + rows := []struct { + name string + typ string + live []string + archive []string + wantFilledIn bool + }{ + {"http same address", "http", []string{a}, []string{a}, true}, + {"http blanked address", "http", []string{a}, []string{"url: " + RedactedPlaceholder}, true}, + {"http no target keys at all", "http", nil, nil, true}, + {"http different address", "http", []string{a}, []string{"url: https://hooks.example.invalid/b"}, false}, + {"http trailing slash differs", "http", []string{a}, []string{a + "/"}, false}, + {"http letter case differs", "http", []string{a}, []string{"url: https://HOOKS.example.invalid/a"}, false}, + {"http address only in archive", "http", nil, []string{a}, false}, + {"http address only in current", "http", []string{a}, nil, false}, + {"http tls verification differs", "http", []string{a, "skip_tls_verification: false"}, []string{a, "skip_tls_verification: true"}, false}, + {"http tls verification equal", "http", []string{a, "skip_tls_verification: true"}, []string{a, "skip_tls_verification: true"}, true}, + {"http ca path only in current", "http", []string{a, "ca_cert_path: /ca.pem"}, []string{a}, false}, + {"http ca path only in archive", "http", []string{a}, []string{a, "ca_cert_path: /ca.pem"}, false}, + {"http cert path differs", "http", []string{a, "cert_path: /a.pem"}, []string{a, "cert_path: /b.pem"}, false}, + {"http key path differs", "http", []string{a, "key_path: /a.key"}, []string{a, "key_path: /b.key"}, false}, + {"http unix socket differs", "http", []string{a, "unix_socket: /a.sock"}, []string{a, "unix_socket: /b.sock"}, false}, + {"http method is not part of the target", "http", []string{a, "method: POST"}, []string{a, "method: PUT"}, true}, + {"http address as list in archive", "http", []string{a}, []string{"url: [x]"}, false}, + {"http address as mapping in current", "http", []string{"url: {x: y}"}, []string{"url: " + RedactedPlaceholder}, false}, + {"http address as alias in current", "http", []string{"other: &u https://hooks.example.invalid/a", "url: *u"}, []string{a}, false}, + {"email same port", "email", []string{"smtp_port: 25"}, []string{"smtp_port: 25"}, true}, + {"email port written differently", "email", []string{"smtp_port: 25"}, []string{`smtp_port: "25"`}, false}, + {"email quoted port equal", "email", []string{`smtp_port: "25"`}, []string{`smtp_port: "25"`}, true}, + {"email host differs", "email", []string{"smtp_host: a.example.invalid"}, []string{"smtp_host: b.example.invalid"}, false}, + {"email encryption differs", "email", []string{"encryption_type: ssltls"}, []string{"encryption_type: none"}, false}, + {"email auth differs", "email", []string{"auth_type: plain"}, []string{"auth_type: login"}, false}, + {"email helo host differs", "email", []string{"helo_host: a.example.invalid"}, []string{"helo_host: b.example.invalid"}, false}, + {"email sender is not part of the target", "email", []string{"sender_email: a@example.invalid"}, []string{"sender_email: b@example.invalid"}, true}, + {"slack blanked webhook", "slack", []string{"webhook: https://hooks.example.invalid/a"}, []string{"webhook: " + RedactedPlaceholder}, true}, + {"slack different webhook", "slack", []string{"webhook: https://hooks.example.invalid/a"}, []string{"webhook: https://hooks.example.invalid/b"}, false}, + {"splunk different address", "splunk", []string{a}, []string{"url: https://hooks.example.invalid/b"}, false}, + {"splunk same address", "splunk", []string{a}, []string{a}, true}, + {"sentinel different workspace", "sentinel", []string{"customer_id: one"}, []string{"customer_id: two"}, false}, + {"sentinel blanked workspace", "sentinel", []string{"customer_id: one"}, []string{"customer_id: " + RedactedPlaceholder}, true}, + {"unknown type same other keys", "mystery", []string{"extra: a"}, []string{"extra: a"}, true}, + {"unknown type different other keys", "mystery", []string{"extra: a"}, []string{"extra: b"}, false}, + {"unknown type key only in current", "mystery", []string{"extra: a"}, nil, false}, + {"unknown type key only in archive", "mystery", nil, []string{"extra: a"}, false}, + } + for _, row := range rows { + t.Run(row.name, func(t *testing.T) { + t.Parallel() + archive := blankedFile(row.typ, row.archive...) + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc(row.typ, liveMarker, row.live...)}, + map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + got := readFile(t, relPath(data, "notifications/hook.yaml")) + if row.wantFilledIn { + require.Contains(t, got, liveMarker) + require.NotContains(t, got, redactHeaderComment) + return + } + require.Equal(t, archive, got) + names, err := ScanPlaceholders(data) + require.NoError(t, err) + require.Equal(t, []string{"notifications/hook.yaml"}, names) + }) + } +} + +func TestMergeLeavesFreshHostPlaceholdersAndReportsTheFile(t *testing.T) { + t.Parallel() + archive := blankedFile("http", "url: https://hooks.example.invalid/a") + snap, data := mergeTrees(t, map[string]string{"config.yaml": mergeConfig}, map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Equal(t, archive, readFile(t, relPath(data, "notifications/hook.yaml"))) + names, err := ScanPlaceholders(data) + require.NoError(t, err) + require.Equal(t, []string{"notifications/hook.yaml"}, names) +} + +func TestMergeNeedsTheSameTypeAndASingleCurrentDefinition(t *testing.T) { + t.Parallel() + archive := blankedFile("http", "url: https://hooks.example.invalid/a") + cases := map[string]map[string]string{ + "other type under the same name": {"notifications/hook.yaml": mergeDoc("splunk", liveMarker, "url: https://hooks.example.invalid/a")}, + "name defined twice in one file": {"notifications/hook.yaml": mergeDoc("http", liveMarker) + "---\n" + mergeDoc("http", liveMarker)}, + "name defined in two files": { + "notifications/a.yaml": mergeDoc("http", liveMarker), + "notifications/b.yaml": mergeDoc("http", liveMarker), + }, + "current definition uses a merge key": {"notifications/hook.yaml": "base: &b {url: 'https://hooks.example.invalid/a'}\ntype: http\nname: hook\n<<: *b\nheaders:\n Authorization: " + liveMarker + "\n"}, + "other name": {"notifications/hook.yaml": strings.Replace(mergeDoc("http", liveMarker), "name: hook", "name: other", 1)}, + } + for name, live := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, live, map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Equal(t, archive, readFile(t, relPath(data, "notifications/hook.yaml"))) + }) + } +} + +func TestMergeFindsTheCurrentDefinitionInAnyFile(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/renamed.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": blankedFile("http")}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Contains(t, readFile(t, relPath(data, "notifications/hook.yaml")), liveMarker) +} + +func TestMergeNeverFillsDocumentsWithAliasesOrMergeKeys(t *testing.T) { + t.Parallel() + live := map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker, "url: https://hooks.example.invalid/a")} + cases := map[string]string{ + "alias": "type: http\nname: hook\nurl: https://hooks.example.invalid/a\nheaders: &h\n Authorization: " + RedactedPlaceholder + "\nmore: *h\n", + "merge key": "base: &b {method: POST}\ntype: http\nname: hook\nurl: https://hooks.example.invalid/a\n<<: *b\nheaders:\n Authorization: " + + RedactedPlaceholder + "\n", + } + for name, archive := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, live, map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Equal(t, archive, readFile(t, relPath(data, "notifications/hook.yaml"))) + names, err := ScanPlaceholders(data) + require.NoError(t, err) + require.Equal(t, []string{"notifications/hook.yaml"}, names) + }) + } +} + +func TestMergeKeepsEditedValuesAndAddsNoKeys(t *testing.T) { + t.Parallel() + live := mergeDoc("http", liveMarker, "url: https://hooks.example.invalid/a", "method: PUT", "log_level: debug") + archive := headerLine + "\ntype: http\nname: hook\nurl: https://hooks.example.invalid/a\nmethod: POST\nheaders:\n Authorization: " + + RedactedPlaceholder + "\n X-Edited: edited-by-hand\n" + snap, data := mergeTrees(t, map[string]string{"notifications/hook.yaml": live}, map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + got := readFile(t, relPath(data, "notifications/hook.yaml")) + require.Contains(t, got, liveMarker) + require.Contains(t, got, "method: POST") + require.Contains(t, got, "X-Edited: edited-by-hand") + require.NotContains(t, got, "log_level") + require.NotContains(t, got, "PUT") +} + +func TestMergeKeepsHeaderWhilePlaceholdersRemain(t *testing.T) { + t.Parallel() + live := mergeDoc("http", liveMarker, "url: https://hooks.example.invalid/a") + archive := headerLine + "\ntype: http\nname: hook\nurl: https://hooks.example.invalid/a\nheaders:\n Authorization: " + + RedactedPlaceholder + "\n X-Extra: " + RedactedPlaceholder + "\n" + snap, data := mergeTrees(t, map[string]string{"notifications/hook.yaml": live}, map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + got := readFile(t, relPath(data, "notifications/hook.yaml")) + require.Contains(t, got, liveMarker) + require.Equal(t, 1, strings.Count(got, redactHeaderComment)) + require.Equal(t, 1, strings.Count(got, RedactedPlaceholder)) +} + +func TestMergeFillsEachDocumentOfAFileOnItsOwn(t *testing.T) { + t.Parallel() + second := strings.Replace(blankedFile("splunk", "url: https://s.example.invalid"), headerLine+"\n", "", 1) + second = strings.Replace(second, "name: hook", "name: second", 1) + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": blankedFile("http") + "---\n" + second}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + got := readFile(t, relPath(data, "notifications/hook.yaml")) + require.Contains(t, got, liveMarker) + require.Equal(t, 1, strings.Count(got, RedactedPlaceholder)) + require.Equal(t, 1, strings.Count(got, redactHeaderComment)) +} + +func TestMergeFillsABlankedValueThatCarriesAnAnchor(t *testing.T) { + t.Parallel() + archive := "type: http\nname: hook\nheaders:\n Authorization: &a " + RedactedPlaceholder + "\n" + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Contains(t, readFile(t, relPath(data, "notifications/hook.yaml")), liveMarker) +} + +func TestMergeKeepsFilesWithoutPlaceholdersByteIdentical(t *testing.T) { + t.Parallel() + odd := "type: http # untouched\nname: hook\nurl: https://hooks.example.invalid/a\n" + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": odd, "notifications/notes.txt": RedactedPlaceholder}) + before, err := os.Stat(relPath(data, "notifications/hook.yaml")) + require.NoError(t, err) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Equal(t, odd, readFile(t, relPath(data, "notifications/hook.yaml"))) + after, err := os.Stat(relPath(data, "notifications/hook.yaml")) + require.NoError(t, err) + require.True(t, os.SameFile(before, after), "an untouched file must not be rewritten") + require.Equal(t, RedactedPlaceholder, readFile(t, relPath(data, "notifications/notes.txt"))) +} + +func TestMergeDoesNotFillFromCurrentPlaceholders(t *testing.T) { + t.Parallel() + archive := blankedFile("http", "url: https://hooks.example.invalid/a") + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", RedactedPlaceholder, "url: https://hooks.example.invalid/a")}, + map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Equal(t, archive, readFile(t, relPath(data, "notifications/hook.yaml"))) +} + +func TestMergeSkipsCurrentFilesThatDoNotParse(t *testing.T) { + t.Parallel() + archive := blankedFile("http") + snap, data := mergeTrees(t, + map[string]string{ + "notifications/broken.yaml": "type: http\nname: hook\n bad: [\n", + "notifications/hook.yaml": mergeDoc("http", liveMarker), + }, + map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Contains(t, readFile(t, relPath(data, "notifications/hook.yaml")), liveMarker) +} + +func TestMergeWritesTheCurrentFilesModeCappedToOwnerOnly(t *testing.T) { + t.Parallel() + cases := map[string]struct{ live, archive, want os.FileMode }{ + "archive entry is world readable": {0o600, 0o644, 0o600}, + "current file is group readable": {0o640, 0o644, 0o600}, + "current file is read only": {0o400, 0o666, 0o400}, + "current file is world accessible": {0o666, 0o644, 0o600}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": blankedFile("http")}) + require.NoError(t, os.Chmod(relPath(snap, "notifications/hook.yaml"), c.live)) + require.NoError(t, os.Chmod(relPath(data, "notifications/hook.yaml"), c.archive)) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + info, err := os.Stat(relPath(data, "notifications/hook.yaml")) + require.NoError(t, err) + require.Equal(t, c.want, info.Mode().Perm()) + }) + } +} + +func TestMergeSelectsFilesLikeTheExport(t *testing.T) { + t.Parallel() + redirect := "config_paths:\n notification_dir: %s\n" + live := map[string]string{ + "config/config.yaml": mergeConfig, + "config/notifications/sub/hook.yaml": mergeDoc("http", liveMarker), + "config/custom_notify/hook2.yaml": strings.Replace(mergeDoc("http", liveMarker), "name: hook", "name: hook2", 1), + "config/notifications.bak/hook3.yaml": strings.Replace(mergeDoc("http", liveMarker), "name: hook", "name: hook3", 1), + "config/other/hook4.yml": strings.Replace(strings.Replace(mergeDoc("http", liveMarker), "type: http", "type: \"http\"", 1), "name: hook", "name: hook4", 1), + "config/other/hook5.yml": "name: first\n---\n" + strings.Replace(mergeDoc("http", liveMarker), "name: hook", "name: hook5", 1), + } + data := writeTree(t, map[string]string{"config/config.yaml": "x: y\n"}) + archive := map[string]string{ + "config/config.yaml": fmt.Sprintf(redirect, filepath.Join(data, "config", "custom_notify")), + "config/notifications/sub/hook.yaml": blankedFile("http"), + "config/custom_notify/hook2.yaml": strings.Replace(blankedFile("http"), "name: hook", "name: hook2", 1), + "config/notifications.bak/hook3.yaml": strings.Replace(blankedFile("http"), "name: hook", "name: hook3", 1), + "config/other/hook4.yml": strings.Replace(strings.Replace(blankedFile("http"), "type: http", "type: \"http\"", 1), "name: hook", "name: hook4", 1), + "config/other/hook5.yml": "name: first\n---\n" + strings.Replace(blankedFile("http"), "name: hook", "name: hook5", 1), + "config/parsers/other.yaml": "type: mystery\nname: hook\nsecret: " + RedactedPlaceholder + "\n", + } + for rel, body := range archive { + p := relPath(data, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(t, os.WriteFile(p, []byte(body), 0o600)) + } + snap := writeTree(t, live) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + for _, rel := range []string{"config/notifications/sub/hook.yaml", "config/custom_notify/hook2.yaml", "config/notifications.bak/hook3.yaml", "config/other/hook4.yml", "config/other/hook5.yml"} { + require.Contains(t, readFile(t, relPath(data, rel)), liveMarker, rel) + } + require.Contains(t, readFile(t, relPath(data, "config/parsers/other.yaml")), RedactedPlaceholder) +} + +func TestScanAndMergeResolveTheSameFolders(t *testing.T) { + t.Parallel() + redirect := func(v string) string { return "config_paths:\n notification_dir: " + v + "\n" } + rows := []struct { + name string + config func(data string) string + rel string + picked bool + }{ + {"redirected folder inside the tree", func(d string) string { return redirect(filepath.Join(d, "alerts")) }, "alerts/odd.yaml", true}, + {"rejected redirect falls back to the defaults", func(string) string { return redirect("${HOME}/alerts") }, "alerts/odd.yaml", false}, + {"redirect at a content folder is rejected", func(d string) string { return redirect(filepath.Join(d, "parsers")) }, "parsers/odd.yaml", false}, + {"no config file", func(string) string { return "" }, "alerts/odd.yaml", false}, + {"default folder without a config file", func(string) string { return "" }, "notifications/odd.yaml", true}, + } + for _, row := range rows { + t.Run(row.name, func(t *testing.T) { + t.Parallel() + // A type that is not a known plugin, so only the folder rules can select the file. + body := "type: mystery\nname: hook\nsecret: %s\n" + data := t.TempDir() + snap := t.TempDir() + if cfg := row.config(data); cfg != "" { + writeFile(t, filepath.Join(data, "config.yaml"), cfg) + writeFile(t, filepath.Join(snap, "config.yaml"), cfg) + } + writeFile(t, relPath(data, row.rel), fmt.Sprintf(body, RedactedPlaceholder)) + writeFile(t, relPath(snap, row.rel), fmt.Sprintf(body, liveMarker)) + if strings.HasPrefix(row.rel, "alerts/") && row.config(snap) != "" { + writeFile(t, filepath.Join(snap, "config.yaml"), row.config(snap)) + } + + names, err := ScanPlaceholders(data) + require.NoError(t, err) + require.NoError(t, MergeRedactedNotifications(snap, data)) + + merged := strings.Contains(readFile(t, relPath(data, row.rel)), liveMarker) + require.Equal(t, row.picked, merged) + if row.picked { + require.Equal(t, []string{row.rel}, names) + } else { + require.Empty(t, names) + } + }) + } +} + +func TestScanReportsFinalFilesByTheirPathBelowTheDataFolder(t *testing.T) { + t.Parallel() + ph := "type: http\nname: n\nurl: " + RedactedPlaceholder + "\n" + cases := map[string]struct { + files map[string]string + want []string + }{ + "flat layout": { + map[string]string{"notifications/a.yaml": ph, "notifications/b.yaml": strings.ReplaceAll(ph, RedactedPlaceholder, "x")}, + []string{"notifications/a.yaml"}, + }, + "nested layout drops the config folder": { + map[string]string{"config/config.yaml": mergeConfig, "config/notifications/sub/a.yaml": ph, "config/notifications.bak/z.yaml": ph}, + []string{"notifications.bak/z.yaml", "notifications/sub/a.yaml"}, + }, + "nothing to report": {map[string]string{"config/config.yaml": mergeConfig}, []string{}}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + names, err := ScanPlaceholders(writeTree(t, c.files)) + require.NoError(t, err) + require.Equal(t, c.want, append([]string{}, names...)) + }) + } +} + +func TestScanCapsTheReportAtFiftyNames(t *testing.T) { + t.Parallel() + files := map[string]string{} + for i := range 60 { + files[fmt.Sprintf("notifications/n%02d.yaml", i)] = "type: http\nname: n" + fmt.Sprint(i) + "\nurl: " + RedactedPlaceholder + "\n" + } + + names, err := ScanPlaceholders(writeTree(t, files)) + + require.NoError(t, err) + require.Len(t, names, maxNeedsValues) + require.Equal(t, "notifications/n00.yaml", names[0]) +} + +func TestCollectNotificationFilesStopsAtTheFileLimit(t *testing.T) { + t.Parallel() + files := map[string]string{} + for i := range maxInspectEntries + 5 { + files[fmt.Sprintf("notifications/n%03d.yaml", i)] = "type: http\nname: x\n" + } + root := writeTree(t, files) + + found, err := collectNotificationFiles(root, defaultNotificationDirs) + + require.NoError(t, err) + require.Len(t, found, maxInspectEntries) +} + +func TestCollectNotificationFilesSkipsWhatCannotBeExamined(t *testing.T) { + t.Parallel() + many := "type: http\nname: x\nlist:\n" + strings.Repeat("- a\n", maxRedactNodes+10) + root := writeTree(t, map[string]string{ //nolint:gosec // G101: fixture file names, no credentials + "notifications/ok.yaml": "type: http\nname: ok\n", + "notifications/broken.yaml": "type: [\n", + "notifications/big.yaml": "type: http\n# " + strings.Repeat("a", int(maxInspectEntryBytes)) + "\n", + "notifications/many.yaml": many, + "notifications/notes.txt": "type: http\nname: x\n", + "config/hub/x.yaml": "type: http\nname: hub\n", + "config/local_api_credentials.yaml": "type: http\nname: creds\n", + "parsers/typeless.yaml": "name: x\n", + "notifications/dir.yaml/in.txt": "x", + }) + require.NoError(t, os.Symlink(relPath(root, "notifications/ok.yaml"), relPath(root, "notifications/link.yaml"))) + + found, err := collectNotificationFiles(root, defaultNotificationDirs) + + require.NoError(t, err) + var names []string + for _, f := range found { + names = append(names, f.rel) + } + require.Equal(t, []string{"notifications/ok.yaml"}, names) +} + +func TestMergeAndScanNeverWriteOrReadThroughLinkedFolders(t *testing.T) { + t.Parallel() + outside := writeTree(t, map[string]string{"hook.yaml": blankedFile("http")}) + snap := writeTree(t, map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}) + data := t.TempDir() + require.NoError(t, os.Symlink(outside, filepath.Join(data, "notifications"))) + before := readFile(t, filepath.Join(outside, "hook.yaml")) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + names, err := ScanPlaceholders(data) + + require.NoError(t, err) + require.Empty(t, names) + require.Equal(t, before, readFile(t, filepath.Join(outside, "hook.yaml"))) +} + +func TestMergeFailsWhenTheFolderCannotBeWritten(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": blankedFile("http")}) + dir := relPath(data, "notifications") + require.NoError(t, os.Chmod(dir, 0o500)) //nolint:gosec // G302: a folder needs the owner execute bit + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) //nolint:gosec // G302: a folder needs the owner execute bit + + err := MergeRedactedNotifications(snap, data) + + require.Error(t, err) + require.NotContains(t, err.Error(), liveMarker) + require.NotContains(t, err.Error(), data) + require.Contains(t, readFile(t, relPath(data, "notifications/hook.yaml")), RedactedPlaceholder) +} + +func TestMergeAndScanReportUnreadableFolders(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker), "locked/x.yaml": "a: b\n"}, + map[string]string{"notifications/hook.yaml": blankedFile("http"), "locked/x.yaml": "a: b\n"}) + for _, root := range []string{snap, data} { + dir := relPath(root, "locked") + require.NoError(t, os.Chmod(dir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) //nolint:gosec // G302: a folder needs the owner execute bit + } + + require.Error(t, MergeRedactedNotifications(snap, data)) + _, err := ScanPlaceholders(data) + require.Error(t, err) + + require.NoError(t, os.Chmod(relPath(data, "locked"), 0o700)) //nolint:gosec // G302: a folder needs the owner execute bit + err = MergeRedactedNotifications(snap, data) + require.Error(t, err, "the current files must be readable too") +} + +func TestRewriteNotificationFileRefusesLinksAndKeepsTheOldFileOnFailure(t *testing.T) { + t.Parallel() + docs, err := ParseYAMLDocuments(strings.NewReader("a: b\n"), false) + require.NoError(t, err) + + t.Run("link in the path", func(t *testing.T) { + t.Parallel() + outside := t.TempDir() + root := t.TempDir() + require.NoError(t, os.Symlink(outside, filepath.Join(root, "link"))) + + err := rewriteNotificationFile(root, notificationFile{rel: "link/x.yaml", docs: docs}, 0o600) + + require.ErrorIs(t, err, ErrSymlinkInPath) + require.NoFileExists(t, filepath.Join(outside, "x.yaml")) + }) + t.Run("destination is a folder", func(t *testing.T) { + t.Parallel() + root := writeTree(t, map[string]string{"x.yaml/inner.txt": "keep"}) + + err := rewriteNotificationFile(root, notificationFile{rel: "x.yaml", docs: docs}, 0o600) + + require.Error(t, err) + entries, readErr := os.ReadDir(root) + require.NoError(t, readErr) + require.Len(t, entries, 1, "no temporary file may be left behind") + }) + t.Run("document cannot be encoded", func(t *testing.T) { + t.Parallel() + bad := &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Kind: yaml.AliasNode}}} + + err := rewriteNotificationFile(t.TempDir(), notificationFile{rel: "x.yaml", docs: []*yaml.Node{bad}}, 0o600) + + require.ErrorIs(t, err, ErrRedactUnsafe) + }) +} + +func TestWriteAndCloseReportsEachFailure(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "f") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + closed, err := os.Open(path) //nolint:gosec // G304: test fixture + require.NoError(t, err) + require.NoError(t, closed.Close()) + require.Error(t, writeAndClose(closed, []byte("x"), 0o600), "chmod on a closed file") + + readOnly, err := os.Open(path) //nolint:gosec // G304: test fixture + require.NoError(t, err) + require.Error(t, writeAndClose(readOnly, []byte("x"), 0o600), "write on a read-only handle") +} + +func TestDropHeaderCommentOnlyRemovesCharonsLine(t *testing.T) { + t.Parallel() + docs, err := ParseYAMLDocuments(strings.NewReader("# my own note\na: b\n"), false) + require.NoError(t, err) + dropHeaderComment(docs) + out, err := encodeDocuments(docs) + require.NoError(t, err) + require.Contains(t, string(out), "my own note") + dropHeaderComment(nil) +} + +func TestMergeKeepsTheDefinedNamesAndTheProfileCheckPasses(t *testing.T) { + t.Parallel() + profiles := "name: p\nfilters:\n - Alert.Remediation == true\ndecisions:\n - type: ban\n duration: 4h\nnotifications:\n - hook\n" + snap, data := mergeTrees(t, + map[string]string{ + "config.yaml": mergeConfig, "profiles.yaml": profiles, + "notifications/hook.yaml": mergeDoc("http", liveMarker, "url: https://hooks.example.invalid/a"), + }, + map[string]string{"config.yaml": mergeConfig}) + before, err := InspectTree(snap) + require.NoError(t, err) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + kept, err := RestoreMissingCompanions(snap, data, "") + require.NoError(t, err) + + require.Contains(t, kept, "notifications/hook.yaml") + require.NoError(t, VerifyKeptProfile(data, kept)) + after, err := InspectTree(data) + require.NoError(t, err) + namesBefore, _ := definedNotifications(before.Files, "notifications") + namesAfter, _ := definedNotifications(after.Files, "notifications") + require.Equal(t, namesBefore, namesAfter) +} + +func TestScanReportsRestoredFilesThatStillHoldPlaceholders(t *testing.T) { + t.Parallel() + profiles := "name: p\nfilters:\n - Alert.Remediation == true\ndecisions:\n - type: ban\n duration: 4h\nnotifications:\n - hook\n" + snap, data := mergeTrees(t, + map[string]string{ + "config.yaml": mergeConfig, "profiles.yaml": profiles, + "notifications/hook.yaml": blankedFile("http", "url: https://hooks.example.invalid/a"), + }, + map[string]string{"config.yaml": mergeConfig}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + kept, err := RestoreMissingCompanions(snap, data, "") + require.NoError(t, err) + names, err := ScanPlaceholders(data) + + require.NoError(t, err) + require.Contains(t, kept, "notifications/hook.yaml") + require.Equal(t, []string{"notifications/hook.yaml"}, names) +} + +func TestMergeAfterRedactionRestoresCurrentValuesExceptWhereStructureIsShared(t *testing.T) { + t.Parallel() + live := map[string]string{} + archive := map[string]string{} + for name, body := range plantedPlugins { + live["notifications/"+name] = body + res, err := RedactNotificationFile([]byte(body)) + require.NoError(t, err) + archive["notifications/"+name] = string(res.Data) + } + snap, data := mergeTrees(t, live, archive) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + names, err := ScanPlaceholders(data) + require.NoError(t, err) + require.Equal(t, []string{"notifications/unknown.yaml"}, names, "the nested value of an unknown plugin has no plain counterpart") + for name, body := range live { + if strings.HasSuffix(name, "unknown.yaml") { + continue + } + require.Equal(t, decodeAll(t, body), decodeAll(t, readFile(t, relPath(data, name))), name) + } +} + +func decodeAll(t *testing.T, text string) []any { + t.Helper() + dec := yaml.NewDecoder(strings.NewReader(text)) + var docs []any + for { + var v any + err := dec.Decode(&v) + if err != nil { + require.ErrorContains(t, err, "EOF") + return docs + } + docs = append(docs, v) + } +} + +// Not parallel: it switches the shared logger to debug level to see every message. +func TestMergeLogsHoldNoNotificationContent(t *testing.T) { + hook := &recordingHook{} + logger.Log().Logger.AddHook(hook) + previous := logger.Log().Logger.GetLevel() + logger.Log().Logger.SetLevel(logrus.DebugLevel) + t.Cleanup(func() { logger.Log().Logger.SetLevel(previous) }) + + snap, data := mergeTrees(t, + map[string]string{ + "notifications/hook.yaml": mergeDoc("http", planted+"live", "url: https://hooks.example.invalid/"+planted+"a"), + "notifications/broken.yaml": "type: http\nurl: " + planted + "broken\n bad: [\n", + }, + map[string]string{"notifications/hook.yaml": blankedFile("http", "url: https://hooks.example.invalid/"+planted+"b")}) + require.NoError(t, MergeRedactedNotifications(snap, data)) + _, err := ScanPlaceholders(data) + require.NoError(t, err) + + hook.mu.Lock() + defer hook.mu.Unlock() + require.NotEmpty(t, hook.messages) + for _, m := range hook.messages { + require.NotContains(t, m, planted) + } +} + +func TestMergeToleratesAMissingSnapshot(t *testing.T) { + t.Parallel() + archive := blankedFile("http") + data := writeTree(t, map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(filepath.Join(t.TempDir(), "missing"), data)) + + require.Equal(t, archive, readFile(t, relPath(data, "notifications/hook.yaml"))) +} + +func TestMergeFailsOnAnUnreadableNotificationFile(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": blankedFile("http")}) + require.NoError(t, os.Chmod(relPath(snap, "notifications/hook.yaml"), 0o000)) + t.Cleanup(func() { _ = os.Chmod(relPath(snap, "notifications/hook.yaml"), 0o600) }) + + require.Error(t, MergeRedactedNotifications(snap, data)) +} + +func TestMergeLeavesDocumentsWithoutAMappingOrIdentityAlone(t *testing.T) { + t.Parallel() + archives := map[string]string{ + "scalar document": RedactedPlaceholder + "\n", + "no name": "type: http\nurl: " + RedactedPlaceholder + "\n", + "number as type": "type: 5\nname: hook\nurl: " + RedactedPlaceholder + "\n", + } + for name, archive := range archives { + t.Run(name, func(t *testing.T) { + t.Parallel() + snap, data := mergeTrees(t, + map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker)}, + map[string]string{"notifications/hook.yaml": archive}) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + require.Equal(t, archive, readFile(t, relPath(data, "notifications/hook.yaml"))) + }) + } +} + +func TestMergeFillsListsPositionByPosition(t *testing.T) { + t.Parallel() + live := map[string]string{"notifications/hook.yaml": mergeDoc("http", liveMarker, "extra: [one, two]", "other: {a: b}")} + archive := map[string]string{"notifications/hook.yaml": blankedFile("http", + "extra: ["+RedactedPlaceholder+", "+RedactedPlaceholder+", "+RedactedPlaceholder+"]", "other: ["+RedactedPlaceholder+"]")} + snap, data := mergeTrees(t, live, archive) + + require.NoError(t, MergeRedactedNotifications(snap, data)) + + got := readFile(t, relPath(data, "notifications/hook.yaml")) + require.Contains(t, got, "one") + require.Contains(t, got, "two") + require.Equal(t, 2, strings.Count(got, RedactedPlaceholder), "the third item and the mismatched list stay blanked") +} diff --git a/backend/internal/crowdsec/redact_test.go b/backend/internal/crowdsec/redact_test.go new file mode 100644 index 000000000..7b690c67c --- /dev/null +++ b/backend/internal/crowdsec/redact_test.go @@ -0,0 +1,541 @@ +package crowdsec + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +const redactGoldenDir = "testdata/redact" + +// redactTokenPrefix marks every planted fixture value; fixtures never use real credentials. +const redactTokenPrefix = "SECRETTOKEN-" + +func mustRedact(t *testing.T, in string) RedactResult { + t.Helper() + res, err := RedactNotificationFile([]byte(in)) + require.NoError(t, err) + return res +} + +func goldenInputs(t *testing.T) []string { + t.Helper() + files, err := filepath.Glob(filepath.Join(redactGoldenDir, "*.in.txt")) + require.NoError(t, err) + require.NotEmpty(t, files) + return files +} + +func TestRedactGoldenFiles(t *testing.T) { + for _, in := range goldenInputs(t) { + name := strings.TrimSuffix(filepath.Base(in), ".in.txt") + t.Run(name, func(t *testing.T) { + src, err := os.ReadFile(filepath.Clean(in)) + require.NoError(t, err) + res := mustRedact(t, string(src)) + golden := filepath.Join(redactGoldenDir, name+".golden.txt") + if os.Getenv("CHARON_UPDATE_GOLDEN") != "" { + require.NoError(t, os.WriteFile(golden, res.Data, 0o600)) + } + want, err := os.ReadFile(filepath.Clean(golden)) + require.NoError(t, err) + require.Equal(t, string(want), string(res.Data)) + require.NotContains(t, string(res.Data), redactTokenPrefix) + require.Equal(t, strings.Count(string(res.Data), RedactedPlaceholder), res.Redacted) + + again := mustRedact(t, string(res.Data)) + require.Equal(t, string(res.Data), string(again.Data), "second pass must not change the output") + require.Equal(t, res.Redacted, again.Redacted) + }) + } +} + +func TestRedactHeaderCommentOnlyWhenPlaceholdersExist(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: file\nname: n\nlog_path: /tmp/a\n") + require.Zero(t, res.Redacted) + require.NotContains(t, string(res.Data), "#") + + res = mustRedact(t, "type: splunk\nname: n\ntoken: abc\n") + require.Equal(t, 1, res.Redacted) + require.Equal(t, 1, strings.Count(string(res.Data), redactHeaderComment)) +} + +func TestRedactEmptyAndCommentOnlyStreams(t *testing.T) { + t.Parallel() + for _, in := range []string{"", "# only a comment\n", " \n"} { + res := mustRedact(t, in) + require.Empty(t, res.Data) + require.Zero(t, res.Redacted) + } +} + +func TestRedactAlreadyRedactedFileIsByteIdentical(t *testing.T) { + t.Parallel() + src := "type: http\nname: n\nurl: https://hooks.example.invalid/SECRETTOKEN-aaaaaaaa\nheaders:\n Authorization: Bearer SECRETTOKEN-bbbbbbbb\n" + first := mustRedact(t, src) + second := mustRedact(t, string(first.Data)) + require.Equal(t, string(first.Data), string(second.Data)) + require.Equal(t, 2, second.Redacted) + require.Equal(t, 1, strings.Count(string(second.Data), redactHeaderComment)) +} + +func TestRedactKeepsStructureForEachPlugin(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: email\nname: n\nsmtp_host: smtp.example.invalid\nsmtp_password: pw\nreceiver_emails:\n - a@example.invalid\n") + docs, err := ParseYAMLDocuments(bytes.NewReader(res.Data), false) + require.NoError(t, err) + var out map[string]any + require.NoError(t, docs[0].Decode(&out)) + require.Equal(t, "email", out["type"]) + require.Equal(t, "n", out["name"]) + require.Equal(t, "smtp.example.invalid", out["smtp_host"]) + require.Equal(t, RedactedPlaceholder, out["smtp_password"]) + require.Equal(t, []any{"a@example.invalid"}, out["receiver_emails"]) +} + +func TestRedactNamesAndTypesSurvive(t *testing.T) { + t.Parallel() + for _, in := range goldenInputs(t) { + src, err := os.ReadFile(filepath.Clean(in)) + require.NoError(t, err) + res := mustRedact(t, string(src)) + require.Equal(t, definedPairs(t, src), definedPairs(t, res.Data), in) + } +} + +func definedPairs(t *testing.T, data []byte) []string { + t.Helper() + docs, err := ParseYAMLDocuments(bytes.NewReader(data), false) + require.NoError(t, err) + var pairs []string + for _, d := range docs { + var m map[string]any + if d.Decode(&m) != nil || m == nil { + continue + } + if typ, ok := m["type"].(string); ok { + name, _ := m["name"].(string) + pairs = append(pairs, typ+"/"+name) + } + } + return pairs +} + +func TestRedactUnsafeInputsBecomeErrors(t *testing.T) { + t.Parallel() + expanding := "a: &a [x,x,x,x,x,x,x,x,x]\n" + prev := "a" + for _, n := range []string{"b", "c", "d", "e", "f", "g", "h", "i", "j", "k"} { + expanding += fmt.Sprintf("%s: &%s [*%s,*%s,*%s,*%s,*%s,*%s,*%s,*%s,*%s]\n", n, n, prev, prev, prev, prev, prev, prev, prev, prev, prev) + prev = n + } + cases := map[string]string{ + "duplicate keys": "type: http\nurl: a\nurl: b\n", + "alias expansion": expanding, + "complex key": "? [a]\n: b\n", + "nul byte": "type: http\nurl: a\x00b\n", + "invalid utf8": "type: http\nurl: \xff\xfe\n", + "binary": string([]byte{0x01, 0x02, 0x03, 0xff, 0x00}), + "unterminated": "type: http\nurl: [a\n", + "oversize": "type: file\n# " + strings.Repeat("a", int(maxInspectEntryBytes)) + "\n", + "too many nodes": "a: [" + strings.Repeat("1,", maxRedactNodes) + "1]\n", + "unknown anchor": "type: http\nurl: *nope\n", + "anchor in collapsed": "type: slack\nunk: {inner: &S SECRETTOKEN-detached}\nchannel: *S\n", + } + for name, in := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + res, err := RedactNotificationFile([]byte(in)) + require.ErrorIs(t, err, ErrRedactUnsafe) + require.Empty(t, res.Data) + require.NotContains(t, err.Error(), redactTokenPrefix) + require.NotContains(t, err.Error(), "detached") + }) + } +} + +func TestRedactDetachedAnchorStubCarriesNothing(t *testing.T) { + t.Parallel() + _, err := RedactNotificationFile([]byte("type: slack\nunk: {inner: &S TOKEN}\nchannel: *S\n")) + require.ErrorIs(t, err, ErrRedactUnsafe) + require.NotContains(t, err.Error(), "TOKEN") + require.NotContains(t, err.Error(), "S ") + + res := mustRedact(t, "type: http\nunk: {inner: &S TOKEN}\nchannel: *S\n") + require.NotContains(t, string(res.Data), "TOKEN") + require.NotContains(t, string(res.Data), "&S") + require.NotContains(t, string(res.Data), "*S") + require.Equal(t, 2, res.Redacted) +} + +func TestRedactStubContentHoldsNoDocument(t *testing.T) { + t.Parallel() + docs, err := ParseYAMLDocuments(strings.NewReader(RedactStubContent), false) + require.NoError(t, err) + require.Empty(t, docs) +} + +func TestRedactFailsClosedOnShapes(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: http\nname: n\nheaders: SECRETTOKEN-text\nmethod: [POST]\ntimeout: &t 5s\nlog_level: *t\n") + require.NotContains(t, string(res.Data), redactTokenPrefix) + require.Contains(t, string(res.Data), "headers: "+RedactedPlaceholder) + require.Contains(t, string(res.Data), "method: "+RedactedPlaceholder) + require.Contains(t, string(res.Data), "log_level: *a1") + + res = mustRedact(t, "type: http\nname: n\nmethod: &m\n a: SECRETTOKEN-one\nlog_level: *m\n") + require.NotContains(t, string(res.Data), redactTokenPrefix) + require.Contains(t, string(res.Data), "log_level: *a1") +} + +func TestRedactTypeMustBeDirectPlainString(t *testing.T) { + t.Parallel() + for name, in := range map[string]string{ + "quoted": "type: \"slack\"\nwebhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + "merged": "base: &b {type: slack}\n<<: *b\nwebhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + "aliased": "t: &t slack\ntype: *t\nwebhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + "upper": "type: SLACK\nwebhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + "binary": "type: !!binary c2xhY2s=\nwebhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + "list": "type: [slack]\nwebhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + "missing": "webhook: SECRETTOKEN-x\nchannel: SECRETTOKEN-chan\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + res, err := RedactNotificationFile([]byte(in)) + if err == nil { + require.NotContains(t, string(res.Data), redactTokenPrefix) + } else { + require.ErrorIs(t, err, ErrRedactUnsafe) + } + }) + } +} + +func TestRedactMergeKeyVariants(t *testing.T) { + t.Parallel() + for name, in := range map[string]string{ + "alias to mapping": "type: http\nname: n\nbase: &b\n url: SECRETTOKEN-merge-url\n method: PUT\n<<: *b\n", + "inline mapping": "type: http\nname: n\n<<: {url: SECRETTOKEN-inline-url}\n", + "sequence of aliases": "type: http\nname: n\na: &a {url: SECRETTOKEN-a-url}\nb: &b {headers: {X: SECRETTOKEN-b-hdr}}\n<<: [*a, *b]\n", + "sequence with scalar": "type: http\nname: n\na: &a {url: SECRETTOKEN-a-url}\n<<: [*a]\n", + "alias to sequence": "type: http\nname: n\nl: &l [SECRETTOKEN-list]\n<<: *l\n", + "scalar merge value": "type: http\nname: n\n<<: SECRETTOKEN-scalar\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + res, err := RedactNotificationFile([]byte(in)) + if err == nil { + require.NotContains(t, string(res.Data), redactTokenPrefix) + require.NotContains(t, string(res.Data), "!!merge") + } else { + require.ErrorIs(t, err, ErrRedactUnsafe) + } + }) + } + res := mustRedact(t, "type: http\nname: n\nheaders: &h {url: SECRETTOKEN-merge-url, method: PUT}\n<<: *h\n") + require.NotContains(t, string(res.Data), "SECRETTOKEN") + require.Contains(t, string(res.Data), "url: "+RedactedPlaceholder) +} + +func TestRedactMergeKeyPrintsWithoutTag(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: http\nname: n\nheaders: &h\n A: ${A}\n<<: *h\n") + require.Contains(t, string(res.Data), "<<: *a1") + require.NotContains(t, string(res.Data), "!!") +} + +func TestRedactKeyTextLimitIsPinned(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: file\nname: n\nlog_path: /a\nUnknownRoot: SECRETTOKEN-v1\nrotate:\n UnknownChild: SECRETTOKEN-v2\n---\ntype: http\nname: n\nheaders:\n Header-Name: SECRETTOKEN-v3\n") + out := string(res.Data) + require.Contains(t, out, "UnknownRoot: "+RedactedPlaceholder) + require.Contains(t, out, "UnknownChild: "+RedactedPlaceholder) + require.Contains(t, out, "Header-Name: "+RedactedPlaceholder) + require.NotContains(t, out, "SECRETTOKEN") +} + +func TestRedactEnvReferencesStayLiteralTextElsewhereDoesNot(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: http\nname: n\nurl: ${URL}\nheaders:\n A: Bearer ${T}\n B: https://h.example.invalid/${X}/abc\n") + out := string(res.Data) + require.Contains(t, out, "url: ${URL}") + require.Contains(t, out, "A: Bearer ${T}") + require.Contains(t, out, "B: "+RedactedPlaceholder) +} + +func TestRedactBoolLikeScalarsAreReplaced(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: sentinel\nname: n\nshared_key: yes\ncustomer_id: off\nextra: ~\nmore: null\nlog_type: on\nempty:\n") + out := string(res.Data) + for _, key := range []string{"shared_key", "customer_id", "extra", "more"} { + require.Contains(t, out, key+": "+RedactedPlaceholder) + } + require.Contains(t, out, "log_type: on") + require.Contains(t, out, "empty:") + require.NotContains(t, out, "empty: "+RedactedPlaceholder) +} + +func TestRedactScrubsCommentsTagsAndAnchorNames(t *testing.T) { + t.Parallel() + res := mustRedact(t, "# SECRETTOKEN-doc\ntype: http # SECRETTOKEN-line\nname: !SECRETTOKEN-tag n\nheaders: &SECRETTOKEN-anchor\n A: ${A}\n B: &SECRETTOKEN-second ${B}\nrotate: *SECRETTOKEN-anchor\n# SECRETTOKEN-foot\n") + out := string(res.Data) + require.NotContains(t, out, "SECRETTOKEN") + require.Contains(t, out, "headers: &a1") + require.Contains(t, out, "&a2") + require.NotContains(t, out, "!") + require.Equal(t, 1, strings.Count(out, "#")) +} + +func TestRedactAnchorNumberingRestartsPerDocument(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: http\nname: a\nheaders: &x\n A: ${A}\n---\ntype: http\nname: b\nheaders: &y\n A: ${A}\n") + require.Equal(t, 2, strings.Count(string(res.Data), "&a1")) +} + +func TestRedactAcceptsPlaceholderInDifferentStyles(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: splunk\nname: n\nurl: \"\"\ntoken: ''\n") + require.Equal(t, 2, res.Redacted) + again := mustRedact(t, string(res.Data)) + require.Equal(t, string(res.Data), string(again.Data)) +} + +func TestRedactCanary(t *testing.T) { + t.Parallel() + t.Run("kept duplicate of a replaced value is not a leak", func(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: email\nname: n\nsender_email: alerts@example.invalid\nsmtp_username: alerts@example.invalid\nsmtp_password: SECRETTOKEN-pw\n") + require.Contains(t, string(res.Data), "sender_email: alerts@example.invalid") + require.Contains(t, string(res.Data), "smtp_username: "+RedactedPlaceholder) + }) + t.Run("replaced value inside a kept scalar yields an error", func(t *testing.T) { + t.Parallel() + _, err := RedactNotificationFile([]byte("type: email\nname: n\nsmtp_password: hunter2hunter2\nemail_subject: alert hunter2hunter2 now\n")) + require.ErrorIs(t, err, ErrRedactUnsafe) + }) + t.Run("collapsed subtree value inside a kept scalar yields an error", func(t *testing.T) { + t.Parallel() + _, err := RedactNotificationFile([]byte("type: http\nname: n\nfuture:\n k: hunter2hunter2\nformat: \"{{ hunter2hunter2 }}\"\n")) + require.ErrorIs(t, err, ErrRedactUnsafe) + }) + t.Run("short values are only replaced", func(t *testing.T) { + t.Parallel() + res := mustRedact(t, "type: http\nname: n\nurl: abc\nformat: abc\n") + require.Contains(t, string(res.Data), "url: "+RedactedPlaceholder) + }) + t.Run("placeholder and header text never trigger the scan", func(t *testing.T) { + t.Parallel() + src := "# " + redactHeaderComment + "\ntype: http\nname: n\nurl: \nformat: \n" + res := mustRedact(t, src) + require.Equal(t, 1, res.Redacted) + }) + t.Run("work bound", func(t *testing.T) { + t.Parallel() + var b strings.Builder + b.WriteString("type: http\nname: n\nformat: " + strings.Repeat("x", 900_000) + "\nfuture:\n") + for i := 0; i < 400; i++ { + fmt.Fprintf(&b, " k%03d: value-%05d\n", i, i) + } + _, err := RedactNotificationFile([]byte(b.String())) + require.ErrorIs(t, err, ErrRedactUnsafe) + }) +} + +func TestIsPlaceholder(t *testing.T) { + t.Parallel() + require.True(t, IsPlaceholder(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: RedactedPlaceholder})) + require.False(t, IsPlaceholder(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "x"})) + require.False(t, IsPlaceholder(&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!binary", Value: RedactedPlaceholder})) + require.False(t, IsPlaceholder(&yaml.Node{Kind: yaml.MappingNode, Tag: "!!str", Value: RedactedPlaceholder})) + require.False(t, IsPlaceholder(nil)) +} + +func TestRedactDoesNotLeakInputInErrors(t *testing.T) { + t.Parallel() + for _, in := range []string{ + "type: http\nurl: SECRETTOKEN-dup\nurl: SECRETTOKEN-dup2\n", + "type: http\nurl: *SECRETTOKEN-anchor\n", + "type: http\nurl: [SECRETTOKEN-open\n", + } { + _, err := RedactNotificationFile([]byte(in)) + require.ErrorIs(t, err, ErrRedactUnsafe) + require.NotContains(t, err.Error(), "SECRETTOKEN") + } +} + +// plantedChannels lists, per plugin type, the keys whose values are credentials. +var plantedChannels = map[string][]string{ + "http": {"url", "unknown_option"}, + "slack": {"webhook", "icon_url"}, + "splunk": {"url", "token"}, + "email": {"smtp_password", "smtp_username"}, + "sentinel": {"shared_key", "customer_id"}, + "file": {"extra"}, + "dummy": {"extra"}, + "other": {"url", "token"}, +} + +func TestRedactPlantedTokensNeverSurvive(t *testing.T) { + t.Parallel() + rng := &testRand{state: 1551} + boolLike := []string{"yes", "on", "null", "~", "off", "no"} + n := 0 + token := func() string { n++; return fmt.Sprintf("%s%04d-%d", redactTokenPrefix, n, rng.Intn(1000)) } + for plugin, keys := range plantedChannels { + for round := 0; round < 40; round++ { + var tokens []string + plant := func() string { v := token(); tokens = append(tokens, v); return v } + var b strings.Builder + fmt.Fprintf(&b, "# %s\n", plant()) + fmt.Fprintf(&b, "type: %s # %s\nname: n%d\n", plugin, plant(), round) + for _, k := range keys { + switch rng.Intn(6) { + case 0: + fmt.Fprintf(&b, "%s: %s\n", k, plant()) + case 1: + fmt.Fprintf(&b, "%s: &%s %s\n", k, plant(), plant()) + case 2: + fmt.Fprintf(&b, "%s: !%s %s\n", k, plant(), plant()) + case 3: + fmt.Fprintf(&b, "%s:\n %s: %s\n list: [%s, %s]\n", k, plant(), plant(), plant(), boolLike[rng.Intn(len(boolLike))]) + case 4: + fmt.Fprintf(&b, "%s: %s # %s\n", k, boolLike[rng.Intn(len(boolLike))], plant()) + default: + fmt.Fprintf(&b, "%s: |\n %s\n second line\n", k, plant()) + } + } + if plugin == "http" { + fmt.Fprintf(&b, "headers:\n Authorization: Bearer %s\n X-Api: %s # %s\n", plant(), plant(), plant()) + } + if plugin == "file" { + fmt.Fprintf(&b, "rotate:\n enabled: true\n other: %s\n", plant()) + } + fmt.Fprintf(&b, "# %s\n", plant()) + res, err := RedactNotificationFile([]byte(b.String())) + if err != nil { + require.ErrorIs(t, err, ErrRedactUnsafe) + continue + } + for _, tok := range tokens { + require.NotContains(t, string(res.Data), tok, "plugin %s round %d\n%s", plugin, round, b.String()) + } + _, perr := ParseYAMLDocuments(bytes.NewReader(res.Data), false) + require.NoError(t, perr) + } + } +} + +func FuzzRedactNotificationFile(f *testing.F) { + for _, in := range goldenInputsForFuzz(f) { + f.Add(in) + } + f.Add([]byte("type: slack\nunk: {inner: &S TOKEN}\nchannel: *S\n")) + f.Add([]byte("type: http\nheaders: &h\n A: SECRETTOKEN-x\n<<: *h\n")) + f.Add([]byte("a: &a [x]\nb: &b [*a,*a]\n")) + f.Fuzz(func(t *testing.T, src []byte) { + res, err := RedactNotificationFile(src) + if err != nil { + require.ErrorIs(t, err, ErrRedactUnsafe) + require.NotContains(t, err.Error(), redactTokenPrefix) + return + } + _, perr := ParseYAMLDocuments(bytes.NewReader(res.Data), false) + require.NoError(t, perr) + if !bytes.Contains(src, []byte(redactTokenPrefix)) { + return + } + // A planted token may only reach the output in a kept position; those hold the same text in + // the input under a key of the allowlist, never inside a comment, tag or anchor name. + docs, _ := ParseYAMLDocuments(bytes.NewReader(res.Data), false) + for _, tok := range plantedTokens(src) { + for _, d := range docs { + require.Empty(t, tokenLeak(d, false, tok), "planted token outside a kept position") + } + } + }) +} + +func goldenInputsForFuzz(f *testing.F) [][]byte { + f.Helper() + files, err := filepath.Glob(filepath.Join(redactGoldenDir, "*.in.txt")) + require.NoError(f, err) + var out [][]byte + for _, p := range files { + b, err := os.ReadFile(filepath.Clean(p)) + require.NoError(f, err) + out = append(out, b) + } + return out +} + +func plantedTokens(src []byte) []string { + var toks []string + for _, field := range strings.FieldsFunc(string(src), func(r rune) bool { + return r != '-' && (r < '0' || r > '9') && (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') + }) { + if strings.HasPrefix(field, redactTokenPrefix) { + toks = append(toks, field) + } + } + return toks +} + +// keptKeyNames are the keys whose values stay in a redacted file, over all plugin types. +var keptKeyNames = map[string]bool{ + "type": true, "name": true, "log_level": true, "group_wait": true, "group_threshold": true, + "max_retry": true, "timeout": true, "format": true, "unix_socket": true, "skip_tls_verification": true, + "method": true, "cert_path": true, "key_path": true, "ca_cert_path": true, "channel": true, + "username": true, "icon_emoji": true, "smtp_host": true, "smtp_port": true, "sender_email": true, + "sender_name": true, "receiver_emails": true, "email_subject": true, "encryption_type": true, + "auth_type": true, "helo_host": true, "connect_timeout": true, "send_timeout": true, "log_type": true, + "log_path": true, "output_file": true, "enabled": true, "max_size": true, "max_files": true, + "max_age": true, "compress": true, +} + +// tokenLeak returns a description of where tok appears in n other than in the text of a key or in the +// value of a kept key, or "" when it does not. Key text of unknown keys is kept by design. +func tokenLeak(n *yaml.Node, kept bool, tok string) string { + for _, s := range []string{n.Anchor, n.Tag, n.HeadComment, n.LineComment, n.FootComment} { + if strings.Contains(s, tok) { + return "metadata" + } + } + if n.Kind == yaml.ScalarNode && !kept && strings.Contains(n.Value, tok) { + return "value" + } + if n.Kind == yaml.MappingNode { + for i := 0; i+1 < len(n.Content); i += 2 { + key := n.Content[i] + for _, s := range []string{key.Anchor, key.Tag, key.HeadComment, key.LineComment, key.FootComment} { + if strings.Contains(s, tok) { + return "key metadata" + } + } + if leak := tokenLeak(n.Content[i+1], keptKeyNames[key.Value], tok); leak != "" { + return leak + } + } + return "" + } + for _, c := range n.Content { + if leak := tokenLeak(c, kept, tok); leak != "" { + return leak + } + } + return "" +} + +// testRand is a tiny deterministic generator so the planted-token test is reproducible. +type testRand struct{ state uint64 } + +func (r *testRand) Intn(n int) int { + r.state = r.state*6364136223846793005 + 1442695040888963407 + return int((r.state >> 33) % uint64(n)) //nolint:gosec // G115: n is a small positive slice length in tests +} diff --git a/backend/internal/crowdsec/safe_create.go b/backend/internal/crowdsec/safe_create.go index 545f6d99d..e8a7d1a2e 100644 --- a/backend/internal/crowdsec/safe_create.go +++ b/backend/internal/crowdsec/safe_create.go @@ -16,10 +16,27 @@ var ErrSymlinkInPath = errors.New("refusing to write through a symlink") // directories. It refuses to proceed if dest or any existing component between root and dest is a // symlink, so archive extraction over a live tree can never be redirected outside root. func CreateFileNoSymlink(root, dest string, parentMode, fileMode os.FileMode) (*os.File, error) { + rel, err := checkNoSymlinkBelow(root, dest) + if err != nil { + return nil, err + } + if err = os.MkdirAll(filepath.Dir(dest), parentMode); err != nil { //nolint:gosec // G703: dest verified contained under root and symlink-free above + return nil, fmt.Errorf("mkdir parent: %w", withoutPath(err)) + } + f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fileMode) //nolint:gosec // G304,G703: dest verified contained under root and symlink-free above + if err != nil { + return nil, fmt.Errorf("open %s: %w", rel, withoutPath(err)) + } + return f, nil +} + +// checkNoSymlinkBelow verifies that dest lies inside root and that no existing component between root and +// dest is a symlink. It returns dest relative to root. +func checkNoSymlinkBelow(root, dest string) (string, error) { root = filepath.Clean(root) rel, err := filepath.Rel(root, filepath.Clean(dest)) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { - return nil, fmt.Errorf("path escapes target: %s", rel) + return "", fmt.Errorf("path escapes target: %s", rel) } cur := root for _, part := range strings.Split(rel, string(os.PathSeparator)) { @@ -29,18 +46,11 @@ func CreateFileNoSymlink(root, dest string, parentMode, fileMode os.FileMode) (* break } if lerr != nil { - return nil, fmt.Errorf("inspect %s: %w", rel, withoutPath(lerr)) + return "", fmt.Errorf("inspect %s: %w", rel, withoutPath(lerr)) } if info.Mode()&os.ModeSymlink != 0 { - return nil, fmt.Errorf("%w: %s", ErrSymlinkInPath, rel) + return "", fmt.Errorf("%w: %s", ErrSymlinkInPath, rel) } } - if err = os.MkdirAll(filepath.Dir(dest), parentMode); err != nil { //nolint:gosec // G703: dest verified contained under root and symlink-free above - return nil, fmt.Errorf("mkdir parent: %w", withoutPath(err)) - } - f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fileMode) //nolint:gosec // G304,G703: dest verified contained under root and symlink-free above - if err != nil { - return nil, fmt.Errorf("open %s: %w", rel, withoutPath(err)) - } - return f, nil + return rel, nil } diff --git a/backend/internal/crowdsec/testdata/redact/anchors.golden.txt b/backend/internal/crowdsec/testdata/redact/anchors.golden.txt new file mode 100644 index 000000000..f0e885f99 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/anchors.golden.txt @@ -0,0 +1,10 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: anchors +url: &a1 +headers: &a2 + Authorization: &a3 + Other: +method: &a4 POST +username: diff --git a/backend/internal/crowdsec/testdata/redact/anchors.in.txt b/backend/internal/crowdsec/testdata/redact/anchors.in.txt new file mode 100644 index 000000000..bd2a3cb98 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/anchors.in.txt @@ -0,0 +1,8 @@ +type: http +name: anchors +url: &SECRETTOKEN-urlanchor https://hooks.example.invalid/SECRETTOKEN-url +headers: &SECRETTOKEN-hdr + Authorization: &SECRETTOKEN-auth Bearer SECRETTOKEN-bearer + Other: *SECRETTOKEN-auth +method: &SECRETTOKEN-m POST +username: *SECRETTOKEN-m diff --git a/backend/internal/crowdsec/testdata/redact/binary_tag.golden.txt b/backend/internal/crowdsec/testdata/redact/binary_tag.golden.txt new file mode 100644 index 000000000..2c49db5c5 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/binary_tag.golden.txt @@ -0,0 +1,7 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: bin +url: +headers: + A: diff --git a/backend/internal/crowdsec/testdata/redact/binary_tag.in.txt b/backend/internal/crowdsec/testdata/redact/binary_tag.in.txt new file mode 100644 index 000000000..3b9a77206 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/binary_tag.in.txt @@ -0,0 +1,5 @@ +type: http +name: bin +url: !!binary U0VDUkVUVE9LRU4tYmluYXJ5 +headers: + A: !!binary U0VDUkVUVE9LRU4tYmluYXJ5 diff --git a/backend/internal/crowdsec/testdata/redact/block_scalars.golden.txt b/backend/internal/crowdsec/testdata/redact/block_scalars.golden.txt new file mode 100644 index 000000000..fa149e02d --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/block_scalars.golden.txt @@ -0,0 +1,10 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: block +url: +headers: + A: +format: > + {{.|toJson}} kept + diff --git a/backend/internal/crowdsec/testdata/redact/block_scalars.in.txt b/backend/internal/crowdsec/testdata/redact/block_scalars.in.txt new file mode 100644 index 000000000..a87c67afa --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/block_scalars.in.txt @@ -0,0 +1,11 @@ +type: http +name: block +url: | + https://hooks.example.invalid/SECRETTOKEN-literal +headers: + A: > + folded SECRETTOKEN-folded + text +format: > + {{.|toJson}} + kept diff --git a/backend/internal/crowdsec/testdata/redact/bool_like.golden.txt b/backend/internal/crowdsec/testdata/redact/bool_like.golden.txt new file mode 100644 index 000000000..724777065 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/bool_like.golden.txt @@ -0,0 +1,14 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: bool_like +url: +method: POST +headers: + A: + B: + C: + D: + E: + F: + G: diff --git a/backend/internal/crowdsec/testdata/redact/bool_like.in.txt b/backend/internal/crowdsec/testdata/redact/bool_like.in.txt new file mode 100644 index 000000000..1f42d6053 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/bool_like.in.txt @@ -0,0 +1,12 @@ +type: http +name: bool_like +url: yes +method: POST +headers: + A: "on" + B: off + C: null + D: ~ + E: 12345678 + F: true + G: diff --git a/backend/internal/crowdsec/testdata/redact/collapsed_subtree.golden.txt b/backend/internal/crowdsec/testdata/redact/collapsed_subtree.golden.txt new file mode 100644 index 000000000..1066815ad --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/collapsed_subtree.golden.txt @@ -0,0 +1,8 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: sentinel +name: collapsed +shared_key: +customer_id: +future: +log_type: crowdsec diff --git a/backend/internal/crowdsec/testdata/redact/collapsed_subtree.in.txt b/backend/internal/crowdsec/testdata/redact/collapsed_subtree.in.txt new file mode 100644 index 000000000..9c48193be --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/collapsed_subtree.in.txt @@ -0,0 +1,9 @@ +type: sentinel +name: collapsed +shared_key: + nested: SECRETTOKEN-nested + list: [SECRETTOKEN-a, SECRETTOKEN-b] +customer_id: [SECRETTOKEN-cust] +future: + SECRETTOKEN-key: value +log_type: crowdsec diff --git a/backend/internal/crowdsec/testdata/redact/comment_only.golden.txt b/backend/internal/crowdsec/testdata/redact/comment_only.golden.txt new file mode 100644 index 000000000..e69de29bb diff --git a/backend/internal/crowdsec/testdata/redact/comment_only.in.txt b/backend/internal/crowdsec/testdata/redact/comment_only.in.txt new file mode 100644 index 000000000..d67eb95cc --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/comment_only.in.txt @@ -0,0 +1 @@ +# nothing but a comment SECRETTOKEN-comment diff --git a/backend/internal/crowdsec/testdata/redact/comments.golden.txt b/backend/internal/crowdsec/testdata/redact/comments.golden.txt new file mode 100644 index 000000000..fe364e8d8 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/comments.golden.txt @@ -0,0 +1,7 @@ +type: dummy +name: comments +output_file: /tmp/out.json +--- +type: file +name: comments2 +log_path: /tmp/x.log diff --git a/backend/internal/crowdsec/testdata/redact/comments.in.txt b/backend/internal/crowdsec/testdata/redact/comments.in.txt new file mode 100644 index 000000000..6a88b516d --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/comments.in.txt @@ -0,0 +1,11 @@ +# head SECRETTOKEN-head +type: dummy # line SECRETTOKEN-line +name: comments +# before key SECRETTOKEN-before +output_file: /tmp/out.json +# foot SECRETTOKEN-foot +--- +# second doc SECRETTOKEN-second +type: file +name: comments2 +log_path: /tmp/x.log diff --git a/backend/internal/crowdsec/testdata/redact/dummy.golden.txt b/backend/internal/crowdsec/testdata/redact/dummy.golden.txt new file mode 100644 index 000000000..eeab31137 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/dummy.golden.txt @@ -0,0 +1,11 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: dummy +name: dummy_default +log_level: info +format: | + {{range . -}} + {{.Value}} + {{end -}} +output_file: /tmp/crowdsec_alerts.json +extra_secret: diff --git a/backend/internal/crowdsec/testdata/redact/dummy.in.txt b/backend/internal/crowdsec/testdata/redact/dummy.in.txt new file mode 100644 index 000000000..9113fcc53 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/dummy.in.txt @@ -0,0 +1,10 @@ +# dummy sink +type: dummy +name: dummy_default +log_level: info +format: | + {{range . -}} + {{.Value}} + {{end -}} +output_file: /tmp/crowdsec_alerts.json # where to write +extra_secret: SECRETTOKEN-dummy-extra diff --git a/backend/internal/crowdsec/testdata/redact/email.golden.txt b/backend/internal/crowdsec/testdata/redact/email.golden.txt new file mode 100644 index 000000000..996b20a91 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/email.golden.txt @@ -0,0 +1,22 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: email +name: email_default +log_level: info +smtp_host: smtp.example.invalid +smtp_username: +smtp_password: +smtp_port: 587 +auth_type: login +sender_name: "Charon Alerts" +sender_email: alerts@example.invalid +email_subject: "CrowdSec alert" +encryption_type: starttls +helo_host: charon.example.invalid +connect_timeout: 10s +send_timeout: 10s +receiver_emails: + - admin@example.invalid + - ops@example.invalid +format: | +

Alert

diff --git a/backend/internal/crowdsec/testdata/redact/email.in.txt b/backend/internal/crowdsec/testdata/redact/email.in.txt new file mode 100644 index 000000000..1b9faf029 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/email.in.txt @@ -0,0 +1,20 @@ +type: email +name: email_default +log_level: info +smtp_host: smtp.example.invalid +smtp_username: alerts@example.invalid +smtp_password: SECRETTOKEN-smtp-pass +smtp_port: 587 +auth_type: login +sender_name: "Charon Alerts" +sender_email: alerts@example.invalid +email_subject: "CrowdSec alert" +encryption_type: starttls +helo_host: charon.example.invalid +connect_timeout: 10s +send_timeout: 10s +receiver_emails: + - admin@example.invalid + - ops@example.invalid +format: | +

Alert

diff --git a/backend/internal/crowdsec/testdata/redact/empty_file.golden.txt b/backend/internal/crowdsec/testdata/redact/empty_file.golden.txt new file mode 100644 index 000000000..e69de29bb diff --git a/backend/internal/crowdsec/testdata/redact/empty_file.in.txt b/backend/internal/crowdsec/testdata/redact/empty_file.in.txt new file mode 100644 index 000000000..e69de29bb diff --git a/backend/internal/crowdsec/testdata/redact/env_references.golden.txt b/backend/internal/crowdsec/testdata/redact/env_references.golden.txt new file mode 100644 index 000000000..4695cc64b --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/env_references.golden.txt @@ -0,0 +1,11 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: env_refs +url: ${HOOK_URL} +headers: + Authorization: Bearer ${API_TOKEN} + X-Basic: Basic $BASIC + X-Token: token ${T} + X-Mixed: + X-Spaced: diff --git a/backend/internal/crowdsec/testdata/redact/env_references.in.txt b/backend/internal/crowdsec/testdata/redact/env_references.in.txt new file mode 100644 index 000000000..fd6a89649 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/env_references.in.txt @@ -0,0 +1,9 @@ +type: http +name: env_refs +url: ${HOOK_URL} +headers: + Authorization: Bearer ${API_TOKEN} + X-Basic: Basic $BASIC + X-Token: token ${T} + X-Mixed: https://hooks.example.invalid/${PART}/SECRETTOKEN-mixed + X-Spaced: Bearer ${DOUBLE} diff --git a/backend/internal/crowdsec/testdata/redact/existing_placeholders.golden.txt b/backend/internal/crowdsec/testdata/redact/existing_placeholders.golden.txt new file mode 100644 index 000000000..2e0fef0f2 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/existing_placeholders.golden.txt @@ -0,0 +1,8 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: already +url: +headers: + Authorization: + X: ${KEEP} diff --git a/backend/internal/crowdsec/testdata/redact/existing_placeholders.in.txt b/backend/internal/crowdsec/testdata/redact/existing_placeholders.in.txt new file mode 100644 index 000000000..688d99b8e --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/existing_placeholders.in.txt @@ -0,0 +1,7 @@ +# Sensitive values were replaced by Charon when this configuration was exported. +type: http +name: already +url: +headers: + Authorization: + X: ${KEEP} diff --git a/backend/internal/crowdsec/testdata/redact/file.golden.txt b/backend/internal/crowdsec/testdata/redact/file.golden.txt new file mode 100644 index 000000000..db779ccd0 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/file.golden.txt @@ -0,0 +1,15 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: file +name: file_default +log_level: info +log_path: /var/log/crowdsec_alerts.log +rotate: + enabled: true + max_size: 500 + max_files: 5 + max_age: 5 + compress: true + hook: +format: | + {{range . -}}{{.Value}}{{end -}} diff --git a/backend/internal/crowdsec/testdata/redact/file.in.txt b/backend/internal/crowdsec/testdata/redact/file.in.txt new file mode 100644 index 000000000..6f37b6e30 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/file.in.txt @@ -0,0 +1,13 @@ +type: file +name: file_default +log_level: info +log_path: /var/log/crowdsec_alerts.log +rotate: + enabled: true + max_size: 500 + max_files: 5 + max_age: 5 + compress: true + hook: SECRETTOKEN-rotate-hook +format: | + {{range . -}}{{.Value}}{{end -}} diff --git a/backend/internal/crowdsec/testdata/redact/http.golden.txt b/backend/internal/crowdsec/testdata/redact/http.golden.txt new file mode 100644 index 000000000..2d6275736 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/http.golden.txt @@ -0,0 +1,19 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: http_default +log_level: info +url: +method: POST +unix_socket: /var/run/hook.sock +skip_tls_verification: false +ca_cert_path: /etc/crowdsec/ca.pem +cert_path: /etc/crowdsec/cert.pem +key_path: /etc/crowdsec/key.pem +headers: + Authorization: + X-Api-Token: + authorization: + Content-Type: +format: | + {{.|toJson}} diff --git a/backend/internal/crowdsec/testdata/redact/http.in.txt b/backend/internal/crowdsec/testdata/redact/http.in.txt new file mode 100644 index 000000000..4cb727fb7 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/http.in.txt @@ -0,0 +1,17 @@ +type: http +name: http_default +log_level: info +url: https://user:SECRETTOKEN-userinfo@hooks.example.invalid/path/SECRETTOKEN-path?key=SECRETTOKEN-query +method: POST +unix_socket: /var/run/hook.sock +skip_tls_verification: false +ca_cert_path: /etc/crowdsec/ca.pem +cert_path: /etc/crowdsec/cert.pem +key_path: /etc/crowdsec/key.pem +headers: + Authorization: Bearer SECRETTOKEN-bearer + X-Api-Token: SECRETTOKEN-apitoken + authorization: basic SECRETTOKEN-lower + Content-Type: application/json +format: | + {{.|toJson}} diff --git a/backend/internal/crowdsec/testdata/redact/merge_key.golden.txt b/backend/internal/crowdsec/testdata/redact/merge_key.golden.txt new file mode 100644 index 000000000..c2598c96d --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/merge_key.golden.txt @@ -0,0 +1,9 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: merged +headers: &a1 + Authorization: + X-Api-Token: +<<: *a1 +method: POST diff --git a/backend/internal/crowdsec/testdata/redact/merge_key.in.txt b/backend/internal/crowdsec/testdata/redact/merge_key.in.txt new file mode 100644 index 000000000..8a61954b2 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/merge_key.in.txt @@ -0,0 +1,7 @@ +type: http +name: merged +headers: &base + Authorization: Bearer SECRETTOKEN-base + X-Api-Token: SECRETTOKEN-api +<<: *base +method: POST diff --git a/backend/internal/crowdsec/testdata/redact/merge_key_list.golden.txt b/backend/internal/crowdsec/testdata/redact/merge_key_list.golden.txt new file mode 100644 index 000000000..9d388a1c7 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/merge_key_list.golden.txt @@ -0,0 +1,7 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: merged_list +headers: &a1 + A: +<<: [*a1, {url: , method: PUT}] diff --git a/backend/internal/crowdsec/testdata/redact/merge_key_list.in.txt b/backend/internal/crowdsec/testdata/redact/merge_key_list.in.txt new file mode 100644 index 000000000..7503f4627 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/merge_key_list.in.txt @@ -0,0 +1,5 @@ +type: http +name: merged_list +headers: &h1 + A: SECRETTOKEN-one +<<: [*h1, {url: "https://hooks.example.invalid/SECRETTOKEN-two", method: PUT}] diff --git a/backend/internal/crowdsec/testdata/redact/multi_alias_keep.golden.txt b/backend/internal/crowdsec/testdata/redact/multi_alias_keep.golden.txt new file mode 100644 index 000000000..ff4a49399 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/multi_alias_keep.golden.txt @@ -0,0 +1,8 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: slack +name: alias_in_kept +webhook: &a1 +channel: &a2 "#alerts" +username: *a2 +icon_emoji: *a1 diff --git a/backend/internal/crowdsec/testdata/redact/multi_alias_keep.in.txt b/backend/internal/crowdsec/testdata/redact/multi_alias_keep.in.txt new file mode 100644 index 000000000..5a333c6bf --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/multi_alias_keep.in.txt @@ -0,0 +1,6 @@ +type: slack +name: alias_in_kept +webhook: &W https://hooks.example.invalid/SECRETTOKEN-hook +channel: &C "#alerts" +username: *C +icon_emoji: *W diff --git a/backend/internal/crowdsec/testdata/redact/multi_document.golden.txt b/backend/internal/crowdsec/testdata/redact/multi_document.golden.txt new file mode 100644 index 000000000..4e9d996bd --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/multi_document.golden.txt @@ -0,0 +1,14 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: first +url: +--- +type: slack +name: second +webhook: +channel: "#two" +--- +type: dummy +name: third +output_file: /tmp/third.json diff --git a/backend/internal/crowdsec/testdata/redact/multi_document.in.txt b/backend/internal/crowdsec/testdata/redact/multi_document.in.txt new file mode 100644 index 000000000..0049d0e9f --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/multi_document.in.txt @@ -0,0 +1,12 @@ +type: http +name: first +url: https://one.example.invalid/SECRETTOKEN-one +--- +type: slack +name: second +webhook: https://two.example.invalid/SECRETTOKEN-two +channel: "#two" +--- +type: dummy +name: third +output_file: /tmp/third.json diff --git a/backend/internal/crowdsec/testdata/redact/no_secrets.golden.txt b/backend/internal/crowdsec/testdata/redact/no_secrets.golden.txt new file mode 100644 index 000000000..008406dc0 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/no_secrets.golden.txt @@ -0,0 +1,3 @@ +type: file +name: nothing +log_path: /var/log/a.log diff --git a/backend/internal/crowdsec/testdata/redact/no_secrets.in.txt b/backend/internal/crowdsec/testdata/redact/no_secrets.in.txt new file mode 100644 index 000000000..9c6559585 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/no_secrets.in.txt @@ -0,0 +1,4 @@ +# plain file +type: file +name: nothing +log_path: /var/log/a.log diff --git a/backend/internal/crowdsec/testdata/redact/non_mapping.golden.txt b/backend/internal/crowdsec/testdata/redact/non_mapping.golden.txt new file mode 100644 index 000000000..e14e4dee8 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/non_mapping.golden.txt @@ -0,0 +1,3 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + + diff --git a/backend/internal/crowdsec/testdata/redact/non_mapping.in.txt b/backend/internal/crowdsec/testdata/redact/non_mapping.in.txt new file mode 100644 index 000000000..f8d92d3c8 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/non_mapping.in.txt @@ -0,0 +1,2 @@ +- type: http + url: SECRETTOKEN-seq diff --git a/backend/internal/crowdsec/testdata/redact/quoted_numerics.golden.txt b/backend/internal/crowdsec/testdata/redact/quoted_numerics.golden.txt new file mode 100644 index 000000000..7b0cfad0c --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/quoted_numerics.golden.txt @@ -0,0 +1,10 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: email +name: "quoted" +smtp_host: "smtp.example.invalid" +smtp_port: "587" +smtp_username: +smtp_password: +receiver_emails: ["a@example.invalid"] +connect_timeout: 10 diff --git a/backend/internal/crowdsec/testdata/redact/quoted_numerics.in.txt b/backend/internal/crowdsec/testdata/redact/quoted_numerics.in.txt new file mode 100644 index 000000000..307da59f2 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/quoted_numerics.in.txt @@ -0,0 +1,8 @@ +type: email +name: "quoted" +smtp_host: "smtp.example.invalid" +smtp_port: "587" +smtp_username: "12345678" +smtp_password: 12345678 +receiver_emails: ["a@example.invalid"] +connect_timeout: 10 diff --git a/backend/internal/crowdsec/testdata/redact/scalar_root.golden.txt b/backend/internal/crowdsec/testdata/redact/scalar_root.golden.txt new file mode 100644 index 000000000..e14e4dee8 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/scalar_root.golden.txt @@ -0,0 +1,3 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + + diff --git a/backend/internal/crowdsec/testdata/redact/scalar_root.in.txt b/backend/internal/crowdsec/testdata/redact/scalar_root.in.txt new file mode 100644 index 000000000..c3f52f433 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/scalar_root.in.txt @@ -0,0 +1 @@ +SECRETTOKEN-scalar diff --git a/backend/internal/crowdsec/testdata/redact/sentinel.golden.txt b/backend/internal/crowdsec/testdata/redact/sentinel.golden.txt new file mode 100644 index 000000000..5a77acd9a --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/sentinel.golden.txt @@ -0,0 +1,10 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: sentinel +name: sentinel_default +log_level: info +customer_id: +shared_key: +log_type: crowdsec +format: | + {{.|toJson}} diff --git a/backend/internal/crowdsec/testdata/redact/sentinel.in.txt b/backend/internal/crowdsec/testdata/redact/sentinel.in.txt new file mode 100644 index 000000000..dbc7fb7d7 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/sentinel.in.txt @@ -0,0 +1,8 @@ +type: sentinel +name: sentinel_default +log_level: info +customer_id: SECRETTOKEN-customer +shared_key: SECRETTOKEN-shared +log_type: crowdsec +format: | + {{.|toJson}} diff --git a/backend/internal/crowdsec/testdata/redact/slack.golden.txt b/backend/internal/crowdsec/testdata/redact/slack.golden.txt new file mode 100644 index 000000000..bac9db5ec --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/slack.golden.txt @@ -0,0 +1,12 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: slack +name: slack_default +log_level: info +webhook: +channel: "#alerts" +username: charon +icon_emoji: ":shield:" +icon_url: +format: | + {{range . -}}{{.Message}}{{end -}} diff --git a/backend/internal/crowdsec/testdata/redact/slack.in.txt b/backend/internal/crowdsec/testdata/redact/slack.in.txt new file mode 100644 index 000000000..0c1f300fc --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/slack.in.txt @@ -0,0 +1,10 @@ +type: slack +name: slack_default +log_level: info +webhook: https://hooks.slack.example.invalid/services/SECRETTOKEN-slack +channel: "#alerts" +username: charon +icon_emoji: ":shield:" +icon_url: https://img.example.invalid/i.png?token=SECRETTOKEN-icon +format: | + {{range . -}}{{.Message}}{{end -}} diff --git a/backend/internal/crowdsec/testdata/redact/splunk.golden.txt b/backend/internal/crowdsec/testdata/redact/splunk.golden.txt new file mode 100644 index 000000000..cc0990117 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/splunk.golden.txt @@ -0,0 +1,9 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: splunk +name: splunk_default +log_level: info +url: +token: +format: | + {{.|toJson}} diff --git a/backend/internal/crowdsec/testdata/redact/splunk.in.txt b/backend/internal/crowdsec/testdata/redact/splunk.in.txt new file mode 100644 index 000000000..526d8a052 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/splunk.in.txt @@ -0,0 +1,7 @@ +type: splunk +name: splunk_default +log_level: info +url: https://splunk.example.invalid:8088/services/collector +token: SECRETTOKEN-splunk +format: | + {{.|toJson}} diff --git a/backend/internal/crowdsec/testdata/redact/tags.golden.txt b/backend/internal/crowdsec/testdata/redact/tags.golden.txt new file mode 100644 index 000000000..24d8539d3 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/tags.golden.txt @@ -0,0 +1,9 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: tags +url: +headers: + X-Token: +timeout: 5s +method: POST diff --git a/backend/internal/crowdsec/testdata/redact/tags.in.txt b/backend/internal/crowdsec/testdata/redact/tags.in.txt new file mode 100644 index 000000000..45b189480 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/tags.in.txt @@ -0,0 +1,7 @@ +type: http +name: !SECRETTOKEN-tag tags +url: !!str https://hooks.example.invalid/SECRETTOKEN-url +headers: !!map + X-Token: !SECRETTOKEN-custom SECRETTOKEN-value +timeout: !!str 5s +method: !!str POST diff --git a/backend/internal/crowdsec/testdata/redact/unknown_key.golden.txt b/backend/internal/crowdsec/testdata/redact/unknown_key.golden.txt new file mode 100644 index 000000000..1b3036569 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/unknown_key.golden.txt @@ -0,0 +1,8 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: splunk +name: unknown_key +url: +token: +future_option: +Token: diff --git a/backend/internal/crowdsec/testdata/redact/unknown_key.in.txt b/backend/internal/crowdsec/testdata/redact/unknown_key.in.txt new file mode 100644 index 000000000..e92298b60 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/unknown_key.in.txt @@ -0,0 +1,6 @@ +type: splunk +name: unknown_key +url: https://splunk.example.invalid/SECRETTOKEN-url +token: SECRETTOKEN-token +future_option: SECRETTOKEN-future +Token: SECRETTOKEN-case diff --git a/backend/internal/crowdsec/testdata/redact/unknown_type.golden.txt b/backend/internal/crowdsec/testdata/redact/unknown_type.golden.txt new file mode 100644 index 000000000..7816de5ad --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/unknown_type.golden.txt @@ -0,0 +1,7 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: carrier_pigeon +name: pigeon +url: +timeout: 5s +nested: diff --git a/backend/internal/crowdsec/testdata/redact/unknown_type.in.txt b/backend/internal/crowdsec/testdata/redact/unknown_type.in.txt new file mode 100644 index 000000000..ffc1d8ae0 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/unknown_type.in.txt @@ -0,0 +1,8 @@ +type: carrier_pigeon +name: pigeon +url: https://pigeon.example.invalid/SECRETTOKEN-pigeon +timeout: 5s +nested: + deep: + - SECRETTOKEN-list + - key: SECRETTOKEN-keyed diff --git a/backend/internal/crowdsec/testdata/redact/wrong_shapes.golden.txt b/backend/internal/crowdsec/testdata/redact/wrong_shapes.golden.txt new file mode 100644 index 000000000..a235f4458 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/wrong_shapes.golden.txt @@ -0,0 +1,7 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: email +name: shapes +smtp_host: +receiver_emails: +sender_email: diff --git a/backend/internal/crowdsec/testdata/redact/wrong_shapes.in.txt b/backend/internal/crowdsec/testdata/redact/wrong_shapes.in.txt new file mode 100644 index 000000000..a0d458b26 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/wrong_shapes.in.txt @@ -0,0 +1,8 @@ +type: email +name: shapes +smtp_host: [smtp.example.invalid, SECRETTOKEN-host] +receiver_emails: + - a@example.invalid + - nested: SECRETTOKEN-nested +sender_email: + key: SECRETTOKEN-sender diff --git a/backend/internal/crowdsec/testdata/redact/wrong_shapes_nested.golden.txt b/backend/internal/crowdsec/testdata/redact/wrong_shapes_nested.golden.txt new file mode 100644 index 000000000..453855491 --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/wrong_shapes_nested.golden.txt @@ -0,0 +1,18 @@ +# Sensitive values were replaced by Charon when this configuration was exported. + +type: http +name: shapes2 +headers: +method: +--- +type: file +name: shapes3 +log_path: /tmp/x +rotate: +--- +type: file +name: shapes4 +log_path: /tmp/x +rotate: + enabled: true + unknown_child: diff --git a/backend/internal/crowdsec/testdata/redact/wrong_shapes_nested.in.txt b/backend/internal/crowdsec/testdata/redact/wrong_shapes_nested.in.txt new file mode 100644 index 000000000..418c2f2cf --- /dev/null +++ b/backend/internal/crowdsec/testdata/redact/wrong_shapes_nested.in.txt @@ -0,0 +1,16 @@ +type: http +name: shapes2 +headers: SECRETTOKEN-notmap +method: [POST] +--- +type: file +name: shapes3 +log_path: /tmp/x +rotate: SECRETTOKEN-rotate +--- +type: file +name: shapes4 +log_path: /tmp/x +rotate: + enabled: true + unknown_child: SECRETTOKEN-child diff --git a/docs/features/crowdsec.md b/docs/features/crowdsec.md index 69edde128..402ceb1fd 100644 --- a/docs/features/crowdsec.md +++ b/docs/features/crowdsec.md @@ -66,7 +66,10 @@ Hub presets are applied on the server too, with the same safety net: before anyt - Only one change runs at a time, so two edits can't collide. - The config file editor only accepts common text and settings files (`.yaml`, `.yml`, `.json`, `.txt`, `.conf`) up to 1 MiB, and validates file type, size and YAML. - The config file editor does not list CrowdSec's account and connection credential files. They stay safely on disk; an operator can still reach them from the container shell if needed. -- An exported CrowdSec configuration is meant for moving settings, not for sharing. CrowdSec's account and connection details are not included, but other saved settings (for example notification and database settings) still can be, so treat the file as private. When you import one, your running system keeps its own. Its installed hub items and runtime state stay as they are, so an import cannot remove or change them. An exported configuration no longer contains the downloaded hub data; CrowdSec downloads it again when needed. CrowdSec re-creates its local connection details on its own. If you move to a new host and use community blocklists or the CrowdSec Console, re-enable console enrollment there. +- An exported CrowdSec configuration is meant for moving settings, not for sharing. CrowdSec's account and connection details are not included. In your notification files, passwords, tokens and webhook addresses are blanked and replaced with ``, while names, types and other plain settings stay readable. Settings written as `${NAME}` environment references are kept as they are, comments in notification files are not kept, and stray non-YAML files (such as `.bak` or `.txt`) in a notifications folder are left out. Database settings and anything else in the configuration are still included, so keep treating the file as private. When you import one, your running system keeps its own. Its installed hub items and runtime state stay as they are, so an import cannot remove or change them. An exported configuration no longer contains the downloaded hub data; CrowdSec downloads it again when needed. CrowdSec re-creates its local connection details on its own. If you move to a new host and use community blocklists or the CrowdSec Console, re-enable console enrollment there. +- When you import a blanked export, notifications that already exist on your system with the same type, name and target keep their current values. For any other notification file the placeholders stay, and the import card lists the files that still need values. Open each one in the config file editor and enter the real values. Importing a blanked export with an older version of Charon writes the placeholder into the file as plain text, so edit those values afterwards. +- A few limits to know about: the `format` message template is kept exactly as written (put secrets in `${NAME}` references, not in the template); the names of settings Charon does not recognize are kept, but their values are blanked; a notification folder you moved outside the data folder is not exported, and the `/etc/crowdsec` shortcut for a moved folder only works with the standard `config/` layout. Files that use YAML anchors, aliases or merge keys are never filled in from your current values, so they come back blanked and are listed on the card. The copy Charon makes of your current settings before an import (the `.backup.` folder next to your data folder) still has the original values. +- If Charon cannot safely blank a notification file, it exports that file as an empty one. If an alert profile uses a notification that only that file defined, importing the export is refused until you define the notification again. - An exported configuration also lists which hub items (collections, parsers, scenarios and so on) were installed on the system it came from. After an import, if some of those items are not installed on this system, the CrowdSec page offers an **Install missing hub items** button. It downloads and installs them from the official CrowdSec hub. This needs an internet connection and can take several minutes. If the hub cannot be reached, nothing is installed and you can simply try again later. If some items were not reached in time, an **Install remaining** button lets you retry just those. Items that are already installed are never removed or changed by an import. - Before changing anything, an import checks that every notification name used by your alert profiles exists in the imported configuration. If any are missing, the import is refused and the message lists the missing names. - If your CrowdSec already keeps its settings in the `config/` folder (every normal Charon install does), an import must keep them there too. An archive with its settings at the top level is refused with a clear message and nothing changes. Export a configuration from this page, or put the files in a `config/` folder inside the archive, then try again. diff --git a/docs/reports/qa_report.md b/docs/reports/qa_report.md index 95f4cffe9..89a8b5691 100644 --- a/docs/reports/qa_report.md +++ b/docs/reports/qa_report.md @@ -1,64 +1,62 @@ -# QA and Security Report: fix/crowdsec-followups-batch4 +# QA and Security Report: feat/crowdsec-export-redaction Date: 2026-10-11 -Scope: full diff `origin/development...HEAD` (11 branch commits plus the 1 QA commit listed below, 44 files) against `SECURITY.md`, `CLAUDE.md` (Definition of Done) and the approved batch-4 spec. The branch refuses a CrowdSec import whose configuration is not in the folder CrowdSec reads, keeps the current companion files an archive does not contain (and reports their names), makes the data lock abandonable when a request ends, classifies a missing hub item precisely, matches protected paths regardless of letter case, translates the remaining configuration page messages, and fixes two test-environment issues. +Scope: full diff `origin/development...HEAD` (8 branch commits plus the 2 QA commits listed below) against `SECURITY.md`, `CLAUDE.md` (Definition of Done) and the approved spec for blanking sensitive values in exported CrowdSec notification files (#1551). The branch rewrites notification files inside the export so they keep their names, types and structure but not their sensitive values, leaves out stray non-YAML files in notification folders, exports a file it cannot rewrite safely as an empty file, and makes the import fill blanked values from the files being replaced when the notification has the same type, name and target. The import card lists the files that still need values. ## Verdict: PASS (no blocking findings) -No critical, high or medium findings. One test defect was found and fixed in the branch (F1); two informational observations (F2, F3) are listed. No model, migration or GORM query file is in the diff, and no security-sensitive defect was found in the import path. +No critical, high or medium findings. Two low findings (L1, L2) and three informational notes are listed; none blocks the merge. No model, migration or GORM query file is in the diff, so the GORM scan was not required. ## Definition of Done evidence | Check | Command | Result | |---|---|---| -| Build | `cd backend && go build ./...` | pass | -| Backend tests and coverage | `scripts/go-test-coverage.sh` | first run: one failing test (F1); after the fix: all packages pass, statements 91.4%, lines 90.4% (gate 87%) | +| Build | `cd backend && go build ./...` and `cd frontend && npm run build` | pass | +| Backend tests and coverage | `scripts/go-test-coverage.sh` | all packages pass, 0 failing tests; statements 91.5%, lines 90.5% (gate 87%) | +| Race run | `go test -race -timeout 60m ./internal/crowdsec/... ./internal/api/handlers/...` | pass (crowdsec 109.3 s, handlers 381.7 s) | | golangci-lint | `cd backend && golangci-lint run --config .golangci.yml ./...` (v2.14.0) | 0 issues | -| Frontend type-check and build | `cd frontend && npm run type-check && npm run build` | pass | -| Frontend coverage | `scripts/frontend-test-coverage.sh` | all tests pass; statements 90.38%, branches 83.92%, functions 88.34%, lines 91.53% (gate 87%) | -| Patch coverage | `bash scripts/local-patch-report.sh` (strict) | overall 91.6% (416/454), backend 92.0% (379/412), frontend 88.1% (37/42), agent n/a; all above the 87% gate; artifacts `test-results/local-patch-report.{md,json}` produced | -| CodeQL | `lefthook run codeql` (CLI 2.27.2) | Go: 4 results, all 4 pre-existing suppressed entries in files outside this diff; JS: 0 results; 0 blocking; parity check passed | -| Trivy | `trivy fs --scanners vuln,misconfig --severity HIGH,CRITICAL` on go.mod/go.sum and the frontend package files (copied under `$HOME` because the snap trivy cannot read `/projects`, removed afterwards) | no HIGH/CRITICAL | -| GORM scan | not run | no `backend/internal/models/**`, migration or GORM query file in the diff | -| E2E | rebuilt `charon-e2e` with `docker-rebuild-e2e`; `npx playwright test tests/security/crowdsec-import.spec.ts tests/security/crowdsec-diagnostics.spec.ts --project=security-tests` | 72 passed, 0 failed, 0 fixme or skipped (46.7 s) | +| Frontend type-check | `cd frontend && npm run type-check` | pass | +| Frontend tests and coverage | `scripts/frontend-test-coverage.sh` | 298 files, 3683 passed, 4 skipped, 2 todo, 0 failed; statements 90.36%, branches 83.83%, functions 88.34%, lines 91.5% (gate 87%) | +| Patch coverage | `bash scripts/local-patch-report.sh` (strict, run after the coverage runs) | overall 96.5% (737/764), backend 96.4% (731/758), frontend 100% (6/6), agent n/a; artifacts `test-results/local-patch-report.{md,json}` produced | +| Fuzzing | `go test ./internal/crowdsec/ -run '^$' -fuzz FuzzRedactNotificationFile -fuzztime=120s` | 536,037 executions, 463 corpus entries, no failure | +| CodeQL | `scripts/pre-commit-hooks/codeql-go-scan.sh`, `codeql-js-scan.sh`, then `codeql-check-findings.sh` (CLI 2.27.2) | Go: 4 results, all 4 pre-existing, in files outside this diff, covered by `codeql-suppressions.yml`; JS: 0 results; 0 blocking | +| Trivy | `trivy fs --scanners vuln,misconfig --severity HIGH,CRITICAL` on the Go and frontend dependency files (copied under `$HOME` because the snap trivy cannot read `/projects`, removed afterwards); the diff does not touch `go.mod`, `go.sum` or `package.json` | no HIGH or CRITICAL | +| GORM scan | not run | no model, migration or GORM query file in the diff | +| E2E | rebuilt `charon-e2e` with `docker-rebuild-e2e` (the user's `charon` container was not touched); `npx playwright test tests/security/crowdsec-export-redaction.spec.ts tests/security/crowdsec-import.spec.ts tests/security/crowdsec-diagnostics.spec.ts tests/security-enforcement/zzz-security-ui/crowdsec-import.spec.ts --project=security-tests` | 93 passed, 0 failed, 0 skipped or fixme (1.3 min) | -Patch coverage detail (lines still uncovered in the patch): `companions.go` 89.0% (the remaining branches are write and close failures on a file just created, and the lost-race `O_EXCL` case, which need fault injection), `archive_inspect.go` 88.2% (defensive read errors), `crowdsec_handler.go` 92.1% (the 500 branch when keeping the current files fails, lines 824-828), `CrowdSecConfig.tsx` 86.5% (error toasts). +Patch coverage detail (changed lines still uncovered): `export.go` 84.3% (16 lines: error branches on a file just opened or stat-ed, which need fault injection), `redact.go` 96.4% (8 lines: defensive encode and walk errors), `crowdsec_handler.go` 96.6% (1 line), `redact_merge.go` 99.3% (2 lines). ## Security review -### 1. Import path: nothing destructive before validation +Probes were run as throwaway Go tests (about 60 hostile or odd notification files, a 19-file export tree, merge round trips of 26 awkward values, size and node-count limits); they were deleted afterwards, and the permanent suite already covers the same ground with goldens, a property test and the fuzzer. -Order in `ImportConfig`: upload saved to a private temp folder, archive validator (size, ratio, required file, traversal), archive inspection and notification check, then the data lock, then the layout check, and only then the snapshot (a copy) and the clearing of the old tree. Every refusal before the snapshot changes nothing. After the snapshot, every failure calls `rollback()`: clearing failed, extraction failed, the flat-archive re-check after extraction, failure to keep the current files, and the kept-profile verification. The flat-archive refusal happens both before the snapshot (from the inspection) and after extraction (when the inspection was skipped), and returns a fixed message. +### 1. No sensitive value survives in the export -### 2. Archive entry names +- Values: every value outside the per-type allowlist is replaced, whatever its YAML spelling (plain, quoted, block, folded, number, bool-like, null, binary, timestamp, URL with credentials, path or query parts, header values of any shape). Unknown plugin types, wrong-cased or quoted types, a type that arrives through an alias, and keys that differ only by case or by look-alike letters all get the strictest treatment (common keys only). +- Text around values: comments of every kind (head, line, foot, trailing document comments), tags (custom, verbatim, directive-defined, binary) and anchor names never reach the output; anchors are renumbered and aliases follow. Merge keys (single, list, inline mapping, into `headers`) are walked in the merging context; when the shared definition has been blanked away the output no longer parses and the file is exported empty. +- Fail closed: duplicate keys, complex keys, expanding alias chains, recursive anchors, unparsable or non-UTF-8 input, more than 50,000 nodes, a file over 1 MiB and a residual-value match all give the empty-file result; the original bytes are never a fallback. Redacting an already redacted file is byte-identical (idempotent), checked for every probe. +- Archive level: files in notification folders at any depth, with any letter case of the folder or the extension, a folder moved by the configuration inside the data folder, flat and nested layouts, look-alike folder names, files with an oversize or binary body, and links (files and folders) were exported without a planted value. Non-YAML files and editor leftovers inside a notification folder are left out; a name with a trailing space is not a YAML name and is left out. +- Documented limits confirmed by probe (unchanged from the spec): key text of unknown top-level keys, unknown children of `rotate` and header names stays readable; `format` templates and other kept plain settings are exported as written; other files in the configuration (for example `config.yaml`) are not rewritten. -`ArchiveEntryRel` is the single normalisation shared by the validator (`requireArchiveFile`), the inspector and the extractor. Absolute names are re-rooted below the destination (as the extractor does), `..` sequences that would leave it are reported as invalid, `./` and doubled slashes collapse, and the required-file check only accepts the top level or `config/`, so a copy such as `parsers/config.yaml` does not count. Case variants are different names on the case-sensitive live filesystem and are never treated as the real configuration; the protected-path predicates (`IsHubPath`, `IsEngineOwnedPath`, `isLiveDBFile`) now lower-case their input, so a differently cased spelling of a protected path is skipped too (conservative direction). +### 2. Import merge -### 3. Links and file writes +Fills only placeholders, only when type and name match exactly one current definition and every target key (URL, host, port, transport and certificate settings; every non-common key for unknown types) is absent on both sides or unchanged, with an exact comparison. Documents holding aliases or merge keys are never merged and are listed. Values are copied into nodes (26 awkward live values, including line breaks, document separators, directives, anchors and tags, round-tripped to the same value in a single document). Rewrites go to a new exclusive temp file renamed over the old one, mode capped at 0600 and at the current file's mode; links in the path are refused. The merge runs after extraction and before the companion restore; an error rolls the import back with a fixed message. The current files are located from the snapshot's own configuration, never from the archive's. -- Presence of a companion in the live tree is decided with `Lstat`, so a link counts as present and is never replaced or written through. Snapshot entries that are links or have the wrong type are skipped and logged with a fixed message. -- Notification files are restored with `O_WRONLY|O_CREATE|O_EXCL` and mode 0600, after an `Lstat` of the destination; an existing path of any kind (regular file, link, dangling link) is skipped, never written through. The notifications folder is only created when absent, and refused when it is a link or a file. The new unit tests pin each of these (regular, linked and dangling destinations, linked folder, file in the way, read-only folder). -- Only direct children of `notifications/` with a lowercase `.yaml` or `.yml` name are considered, and only files that define a name the kept profile needs are copied back; a restore that would leave a name defined twice is refused (422) and rolled back. +### 3. Logs, errors and responses -### 4. Secrets and responses +Rewriter errors carry fixed reasons only; log lines carry sanitised relative names or counts; `needs_values` carries sanitised file names only (capped at 50, sorted); a failed scan omits the list and does not undo the import. Tests assert that planted values never appear in captured logs. -`kept_files` carries file names only (fixed companion names plus `notifications/`), never contents. Error messages are fixed strings or sanitised, length-bounded name lists; I/O errors are wrapped through `withoutPath` so paths are not returned or logged; log fields pass through `SanitizeForLog`. No token or key is logged in the new code. +### 4. Resource bounds -### 5. Locking (`ctxMutex`) +Size (1 MiB), node count (50,000), the residual scan (bounded work, otherwise the empty-file result) and per-tree file count (256) are enforced. Measured: a 400 KB file of 45,000 list items redacts in 55 ms; a 9,000-level nested file in 14 ms; a file of 20,000 distinct blanked values returns the empty-file result in 0.95 s. See L2 for the one super-linear case. -`LockContext` returns an error without taking the lock when the context ends first (an already ended context never takes a free lock); `Unlock` panics when not held. All call sites register `defer Unlock()` only after a successful lock (`lockData` returns false with the lock not held and the callers return before the defer; `writeExportLocked`, `hubUpdate`, `installedHubSet` and `installHubItem` return before the defer on error). No double unlock and no leaked lock was found. The restore single-flight flag is a never-waited compare-and-swap taken before any `dataMu` section and the import never takes it, so the lock order `restore flag -> dataMu` cannot invert. A cancelled waiter releasing nothing is covered by the new cancel tests, and the earlier observation about a non-context-aware lock no longer applies. +### 5. Frontend -### 6. Hub item classification +The import card renders names as React text after `sanitizeKeptFileName` (control, format and separator characters stripped, length capped), lists at most 10 names plus a count, adds no toast, banner or button, and clears the line when another file is chosen. All five locales carry the new keys (parity test passes). -`hubItemNotFound` now matches only cscli's sentence for the requested type and name; any other wording is `install_failed` (safe direction). A failed second read of the installed list now returns 500 with what already happened instead of claiming items were not attempted. +### 6. Public wording -### 7. Frontend - -Archive-derived names are rendered as React text (no `dangerouslySetInnerHTML` or `innerHTML` in the page or the helper); `sanitizeKeptFileName` strips control, format (including direction override) and line/paragraph separator characters and caps the length at 80. The translation key sets for the page are identical in en, de, es, fr and zh (checked by the i18n key test). - -### 8. Commits and docs - -Subjects are generic; none uses the `(security)` scope; no session id or link in any message. Docs additions describe behaviour only. +Commit subjects and bodies, docs, code comments and test names were checked for vulnerability classes, attack scenarios and the phrases "tampered archive" and "destination binding": none found. One test variable was named after an attack pattern and was renamed (commit below). The only `(security)` commit subject is `feat(security): protect sensitive values in CrowdSec exports`, which is generic. No session id or link appears in any message. ## Findings @@ -66,18 +64,20 @@ No critical, high or medium findings. ### Low -- **F1: low (test defect, fixed in this review).** `backend/internal/api/handlers/crowdsec_export_import_test.go` `TestEditorHidesPidBouncersAndHub` still asserted that `config/Hub/x.yaml` is writable, which contradicts the case-insensitive hub matching added by the protected-path commit. It failed deterministically in the handlers package (`go test` exit 1, although the coverage gate message said "met"). The assertion now expects the path to be protected. Evidence: the first `scripts/go-test-coverage.sh` run reported this single failure; after the fix the whole run passes. +- **L1: low (hardening, spec-conformant).** The second-layer check that blanks a notification-shaped YAML file found outside the recognised notification folders looks only at the first document and only at a plain (unquoted, non-aliased) type value. This matters only when a notification folder cannot be located from the configuration (for example a folder setting that is built from an environment reference), because inside a located folder every YAML file is blanked regardless of its content. Recommended: treat a quoted type and every document of the file the same way as a plain first-document type (this only adds blanking). Tracked as a follow-up; the docs already say other files in the configuration are exported as written. +- **L2: low (availability, administrator only, pre-existing root cause).** `ParseYAMLDocuments` takes super-linear time on files with very many aliases: measured 1.3 s for 208 KB, 4.6 s for 430 KB and 36.7 s for 1 MiB (then rejected as excessively aliased). The export, the import merge and the import report now parse YAML files too, so this cost is paid in more places, while holding the data lock. Needs write access to the configuration (administrator). Recommended: reject a file early when its alias count is high (a cheap pre-scan), and skip the notification-shape probe for files that do not mention a `type` key. ### Informational -- **F2: info.** `InspectTree` reads the live tree and the pre-import snapshot following links (as CrowdSec does) to parse configuration and notification names. The snapshot keeps links as links, so a linked notifications folder in the live tree is read through, but the contents are only parsed for names and a file is copied back only when it defines a needed name and the file itself is a regular file. This needs an administrator-created link on the server; no action needed. -- **F3: info.** The 500 branch "failed to keep the current CrowdSec files" in `ImportConfig` (rollback after a companion restore error) has no handler-level test; the underlying function's error paths are covered in the package tests. Reaching it from the handler needs a failure injected into a copy, so it was not added. +- **I1.** Merge and report stop examining after 256 notification files; any beyond that are left as extracted and not listed. Unreachable in practice (the image ships no notification plugins). +- **I2.** Charon's full backups (the backup archive of the database and the CrowdSec folder) and the pre-import snapshot copy the CrowdSec folder unchanged by design (host-local, full fidelity). This feature covers the CrowdSec configuration export only. +- **I3.** The unquoted-type and first-document behaviour of L1 and the key-text, `format` and `config.yaml` limits are the same documented limits as in the spec and the feature guide. ## Known limitations (not re-opened) -Redacting sensitive values in notification files on export is tracked separately (#1551); the image ships no notification plugins. +Database settings in the exported `config.yaml` and other private files in the configuration tree are tracked separately (spec follow-ups F1 and F2); the image ships no notification plugins. ## Commits added by this review -- `test: cover the defensive branches of the CrowdSec notification file restore and keep hub paths protected in the editor test` (new tests for `prepareNotificationsDir` 77.8% -> 100% and `writeNotificationFile` 62.5% -> 75%, plus the F1 assertion fix). -- `docs: update QA report for the CrowdSec follow-up batch` (this report). +- `test: tidy a variable name in the CrowdSec notification rewriter tests` (renames a local test variable; no behaviour change). +- `docs: update QA report for the CrowdSec export protection` (this report). diff --git a/docs/troubleshooting/crowdsec.md b/docs/troubleshooting/crowdsec.md index d213d9cb8..941b85fd1 100644 --- a/docs/troubleshooting/crowdsec.md +++ b/docs/troubleshooting/crowdsec.md @@ -197,6 +197,7 @@ charon-local-machine 127.0.0.1 password v1.x.x - **Import refused because the archive keeps its configuration at the top level**: this CrowdSec reads its settings from the `config/` folder. Export a configuration from the CrowdSec page, or put your files in a `config/` folder inside the archive. Nothing was changed. See the [CrowdSec feature guide](../features/crowdsec.md). - **Import refused after saying the current profile was kept**: your archive has no `profiles.yaml`, so Charon kept the current one, but it names notifications your archive has no files for. Add those notification files to the archive, or include your own `profiles.yaml`. Nothing was changed. - **Import refused because a profile names notifications that have no matching file**: your imported alert profiles refer to notification names that are not in the archive. Add the missing notification files to the configuration, or remove those names from the profile, then import again. Nothing was changed by the refused import. +- **An imported notification does not send, or the import card says files "still need values"**: exports blank passwords, tokens and webhook addresses in notification files, and only notifications that already exist on this system with the same type, name and target get their values back. Open each listed file in the config file editor and enter the real values, or import again after adding them. If a file used YAML anchors or aliases and came back blanked on the same system, your earlier values are in the `.backup.` copy Charon made next to your data folder before the import. See the [CrowdSec feature guide](../features/crowdsec.md). - Apply failed: review the apply response and restore from the backup that was taken automatically, then retry after fixing the underlying issue. - Apply not supported (501): use curated/offline presets; Hub apply will be re-enabled when supported in your environment. - **Security Engine Offline**: If your dashboard says "Offline", it means CrowdSec LAPI is not running. diff --git a/frontend/src/api/crowdsec.ts b/frontend/src/api/crowdsec.ts index 86373d1f7..86c682a0a 100644 --- a/frontend/src/api/crowdsec.ts +++ b/frontend/src/api/crowdsec.ts @@ -75,6 +75,8 @@ export interface ImportCrowdsecResult { hub_items?: ImportHubItems /** Names of the current files kept because the archive did not contain them. */ kept_files?: string[] + /** Imported notification files whose blanked values still need to be entered. */ + needs_values?: string[] } /** Result of a hub item restore. */ diff --git a/frontend/src/locales/de/translation.json b/frontend/src/locales/de/translation.json index f3582f8c1..08f5421c7 100644 --- a/frontend/src/locales/de/translation.json +++ b/frontend/src/locales/de/translation.json @@ -1186,6 +1186,8 @@ "importSuccess": "CrowdSec-Konfiguration importiert (Sicherung erstellt)", "importFailed": "Import fehlgeschlagen", "importKeptFiles": "Diese Dateien waren nicht im Archiv, daher wurden die aktuellen beibehalten: {{files}}", + "importNeedsValues": "Einige Werte in diesen Benachrichtigungsdateien wurden beim Export der Konfiguration geleert und müssen eingetragen werden, bevor sie funktionieren: {{files}}", + "importNeedsValuesMore": "und {{count}} weitere", "exportPrompt": "Name für das CrowdSec-Exportarchiv", "title": "CrowdSec-Konfiguration", "checkNow": "Jetzt prüfen", @@ -1265,7 +1267,7 @@ }, "packages": { "title": "Konfigurationspakete", - "description": "Exportieren oder importieren Sie CrowdSec-Konfigurationspakete (tar.gz). Vor dem Import wird ein Backup erstellt.", + "description": "Exportieren oder importieren Sie CrowdSec-Konfigurationspakete (tar.gz). Vor dem Import wird ein Backup erstellt. Passwörter, Tokens und Webhook-Adressen in Benachrichtigungsdateien werden beim Export geleert.", "export": "Exportieren", "import": "Importieren", "importing": "Importiere...", diff --git a/frontend/src/locales/en/translation.json b/frontend/src/locales/en/translation.json index f1864d444..0102b3b4d 100644 --- a/frontend/src/locales/en/translation.json +++ b/frontend/src/locales/en/translation.json @@ -1507,6 +1507,8 @@ "importSuccess": "CrowdSec config imported (backup created)", "importFailed": "Failed to import", "importKeptFiles": "These files were not in the archive, so the current ones were kept: {{files}}", + "importNeedsValues": "Some values in these notification files were blanked when the configuration was exported, so they need to be entered before they work: {{files}}", + "importNeedsValuesMore": "and {{count}} more", "exportPrompt": "Name your CrowdSec export archive", "title": "CrowdSec Configuration", "checkNow": "Check Now", @@ -1588,7 +1590,7 @@ }, "packages": { "title": "Configuration Packages", - "description": "Export or import CrowdSec configuration packages (tar.gz). A backup is created before importing.", + "description": "Export or import CrowdSec configuration packages (tar.gz). A backup is created before importing. Passwords, tokens and webhook addresses in notification files are blanked in exports.", "export": "Export", "import": "Import", "importing": "Importing...", diff --git a/frontend/src/locales/es/translation.json b/frontend/src/locales/es/translation.json index a3d9d8fff..81a7f7654 100644 --- a/frontend/src/locales/es/translation.json +++ b/frontend/src/locales/es/translation.json @@ -1188,6 +1188,8 @@ "importSuccess": "Configuración de CrowdSec importada (copia de seguridad creada)", "importFailed": "Error al importar", "importKeptFiles": "Estos archivos no estaban en el archivo comprimido, por lo que se conservaron los actuales: {{files}}", + "importNeedsValues": "Algunos valores de estos archivos de notificación se vaciaron al exportar la configuración, por lo que deben introducirse para que funcionen: {{files}}", + "importNeedsValuesMore": "y {{count}} más", "exportPrompt": "Nombre del archivo de exportación de CrowdSec", "title": "Configuración de CrowdSec", "checkNow": "Comprobar ahora", @@ -1267,7 +1269,7 @@ }, "packages": { "title": "Paquetes de configuración", - "description": "Exporte o importe paquetes de configuración de CrowdSec (tar.gz). Se crea una copia de seguridad antes de importar.", + "description": "Exporte o importe paquetes de configuración de CrowdSec (tar.gz). Se crea una copia de seguridad antes de importar. Las contraseñas, los tokens y las direcciones de webhook de los archivos de notificación se vacían en las exportaciones.", "export": "Exportar", "import": "Importar", "importing": "Importando...", diff --git a/frontend/src/locales/fr/translation.json b/frontend/src/locales/fr/translation.json index 2dfe5c15a..fe2b0674f 100644 --- a/frontend/src/locales/fr/translation.json +++ b/frontend/src/locales/fr/translation.json @@ -1188,6 +1188,8 @@ "importSuccess": "Configuration CrowdSec importée (sauvegarde créée)", "importFailed": "Échec de l'importation", "importKeptFiles": "Ces fichiers n'étaient pas dans l'archive, les fichiers actuels ont donc été conservés : {{files}}", + "importNeedsValues": "Certaines valeurs de ces fichiers de notification ont été vidées lors de l'export de la configuration ; il faut les saisir pour qu'elles fonctionnent : {{files}}", + "importNeedsValuesMore": "et {{count}} de plus", "exportPrompt": "Nom de l'archive d'exportation CrowdSec", "title": "Configuration CrowdSec", "checkNow": "Vérifier maintenant", @@ -1267,7 +1269,7 @@ }, "packages": { "title": "Paquets de configuration", - "description": "Exportez ou importez des paquets de configuration CrowdSec (tar.gz). Une sauvegarde est créée avant l'importation.", + "description": "Exportez ou importez des paquets de configuration CrowdSec (tar.gz). Une sauvegarde est créée avant l'importation. Les mots de passe, jetons et adresses de webhook des fichiers de notification sont vidés dans les exports.", "export": "Exporter", "import": "Importer", "importing": "Importation...", diff --git a/frontend/src/locales/zh/translation.json b/frontend/src/locales/zh/translation.json index 7f3b96fdb..d2009b4ac 100644 --- a/frontend/src/locales/zh/translation.json +++ b/frontend/src/locales/zh/translation.json @@ -1187,6 +1187,8 @@ "importSuccess": "CrowdSec 配置已导入（已创建备份）", "importFailed": "导入失败", "importKeptFiles": "归档中没有这些文件，因此保留了当前的文件：{{files}}", + "importNeedsValues": "这些通知文件中的部分值在导出配置时已被清空，需要填写后才能生效：{{files}}", + "importNeedsValuesMore": "以及另外 {{count}} 个", "exportPrompt": "为 CrowdSec 导出文件命名", "title": "CrowdSec 配置", "checkNow": "立即检查", @@ -1266,7 +1268,7 @@ }, "packages": { "title": "配置包", - "description": "导出或导入 CrowdSec 配置包（tar.gz）。导入前会创建备份。", + "description": "导出或导入 CrowdSec 配置包（tar.gz）。导入前会创建备份。导出时，通知文件中的密码、令牌和 Webhook 地址会被清空。", "export": "导出", "import": "导入", "importing": "正在导入...", diff --git a/frontend/src/pages/CrowdSecConfig.tsx b/frontend/src/pages/CrowdSecConfig.tsx index c41a4220f..02c9164fe 100644 --- a/frontend/src/pages/CrowdSecConfig.tsx +++ b/frontend/src/pages/CrowdSecConfig.tsx @@ -36,6 +36,8 @@ function getServerErrorMessage(err: unknown, fallback: string): string { return fallback } +const MAX_LISTED_NEEDS_VALUES = 10 + const CrowdSecDashboard = lazy(() => import('../components/crowdsec/CrowdSecDashboard')) export default function CrowdSecConfig() { @@ -47,6 +49,9 @@ export default function CrowdSecConfig() { // Hub items the last import reported as missing; the key restarts the panel per import. const [importHubItems, setImportHubItems] = useState<{ key: number; items: ImportHubItems } | null>(null) const [keptFiles, setKeptFiles] = useState([]) + const [needsValues, setNeedsValues] = useState([]) + const listedNeedsValues = needsValues.slice(0, MAX_LISTED_NEEDS_VALUES).map(sanitizeKeptFileName) + const hiddenNeedsValues = needsValues.length - listedNeedsValues.length const [selectedPath, setSelectedPath] = useState(null) const [fileContent, setFileContent] = useState(null) const [selectedPresetSlug, setSelectedPresetSlug] = useState('') @@ -129,6 +134,7 @@ export default function CrowdSecConfig() { const items = result.hub_items setImportHubItems(items && items.missing.length > 0 ? { key: Date.now(), items } : null) setKeptFiles(result.kept_files ?? []) + setNeedsValues(result.needs_values ?? []) queryClient.invalidateQueries({ queryKey: ['security-status'] }) }, onError: (err: unknown) => { @@ -989,6 +995,7 @@ export default function CrowdSecConfig() { setFile(e.target.files?.[0] ?? null) setImportHubItems(null) setKeptFiles([]) + setNeedsValues([]) }} data-testid="import-file" accept=".tar.gz" @@ -999,6 +1006,15 @@ export default function CrowdSecConfig() { {t('crowdsecConfig.importKeptFiles', { files: keptFiles.map(sanitizeKeptFileName).join(', ') })}

)} + {needsValues.length > 0 && ( +

+ {t('crowdsecConfig.importNeedsValues', { + files: hiddenNeedsValues > 0 + ? [...listedNeedsValues, t('crowdsecConfig.importNeedsValuesMore', { count: hiddenNeedsValues })].join(', ') + : listedNeedsValues.join(', '), + })} +

+ )} {importHubItems && } diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx index 9108dcb15..5f855b5ea 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx +++ b/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx @@ -9,6 +9,7 @@ import * as featureFlagsApi from '../../api/featureFlags' import * as presetsApi from '../../api/presets' import * as securityApi from '../../api/security' import { renderWithQueryClient } from '../../test-utils/renderWithQueryClient' +import { toast } from '../../utils/toast' import CrowdSecConfig from '../CrowdSecConfig' vi.mock('../../api/security') @@ -151,3 +152,79 @@ describe('CrowdSecConfig kept files note', () => { expect(screen.queryByTestId('import-kept-files')).not.toBeInTheDocument() }) }) + +describe('CrowdSecConfig needs-values note', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(securityApi.getSecurityStatus).mockResolvedValue({ + cerberus: { enabled: true }, + crowdsec: { enabled: true, mode: 'local', api_url: '' }, + waf: { enabled: true, mode: 'enabled' }, + rate_limit: { enabled: true }, + acl: { enabled: true }, + }) + vi.mocked(crowdsecApi.statusCrowdsec).mockResolvedValue({ running: true, pid: 1, lapi_ready: true }) + vi.mocked(crowdsecApi.listCrowdsecFiles).mockResolvedValue({ files: [] }) + vi.mocked(crowdsecApi.listCrowdsecDecisions).mockResolvedValue({ decisions: [] }) + vi.mocked(presetsApi.listCrowdsecPresets).mockResolvedValue({ presets: [] }) + vi.mocked(backupsApi.createBackup).mockResolvedValue({ job_id: 'j', type: 'create', status: 'pending' }) + vi.mocked(featureFlagsApi.getFeatureFlags).mockResolvedValue({ 'feature.crowdsec.console_enrollment': false }) + }) + + it('lists the files that still need values as one inline line without a new toast', async () => { + await importWith({ status: 'imported', needs_values: ['notifications/http_default.yaml', 'notifications/slack.yaml'] }) + + const note = await screen.findByTestId('import-needs-values') + expect(note).toHaveTextContent( + 'Some values in these notification files were blanked when the configuration was exported, so they need to be entered before they work: notifications/http_default.yaml, notifications/slack.yaml', + ) + expect(toast.success).toHaveBeenCalledTimes(1) + expect(toast.error).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() + }) + + it('shows the first 10 names and the number of the remaining ones', async () => { + const names = Array.from({ length: 13 }, (_, i) => `notifications/n${i + 1}.yaml`) + await importWith({ status: 'imported', needs_values: names }) + + const note = await screen.findByTestId('import-needs-values') + expect(note).toHaveTextContent('notifications/n10.yaml, and 3 more') + expect(note).not.toHaveTextContent('notifications/n11.yaml') + }) + + it('lists exactly 10 names without a remainder', async () => { + const names = Array.from({ length: 10 }, (_, i) => `notifications/n${i + 1}.yaml`) + await importWith({ status: 'imported', needs_values: names }) + + const note = await screen.findByTestId('import-needs-values') + expect(note).toHaveTextContent('notifications/n10.yaml') + expect(note).not.toHaveTextContent('more') + }) + + it('sanitises archive-derived names and does not render markup', async () => { + await importWith({ status: 'imported', needs_values: ['notifications/x\u202e\n.yaml'] }) + + const note = await screen.findByTestId('import-needs-values') + expect(note).toHaveTextContent('notifications/x.yaml') + expect(within(note).queryByText('x')).not.toBeInTheDocument() + }) + + it.each([ + ['no needs_values field', { status: 'imported' }], + ['an empty needs_values list', { status: 'imported', needs_values: [] }], + ])('shows no line with %s', async (_label, response) => { + await importWith(response) + + await waitFor(() => expect(crowdsecApi.importCrowdsecConfig).toHaveBeenCalled()) + expect(screen.queryByTestId('import-needs-values')).not.toBeInTheDocument() + }) + + it('clears the line when another file is selected', async () => { + await importWith({ status: 'imported', needs_values: ['notifications/http_default.yaml'] }) + await screen.findByTestId('import-needs-values') + + await userEvent.upload(screen.getByTestId('import-file'), new File(['other'], 'other.tar.gz')) + + expect(screen.queryByTestId('import-needs-values')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts b/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts index bc17e136b..124e32603 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts +++ b/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts @@ -79,6 +79,19 @@ describe('CrowdSec config i18n keys', () => { } }) + it.each(LOCALES)('%s defines the import notice keys and the updated export description', (locale) => { + const tree = loadLocale(locale) + for (const key of [ + 'crowdsecConfig.importNeedsValues', + 'crowdsecConfig.importNeedsValuesMore', + 'crowdsecConfig.packages.description', + ]) { + expect(typeof lookup(tree, key), `${locale}.${key}`).toBe('string') + } + expect(lookup(tree, 'crowdsecConfig.importNeedsValues') as string).toContain('{{files}}') + expect(lookup(tree, 'crowdsecConfig.importNeedsValuesMore') as string).toContain('{{count}}') + }) + it('CrowdSecConfig.tsx has no hard-coded user-visible strings in toasts, status messages or errors', () => { const [source] = Object.entries(sourceModules).filter(([path]) => path.endsWith('/CrowdSecConfig.tsx')).map(([, src]) => src) expect(source).toBeDefined() diff --git a/lefthook.yml b/lefthook.yml index e41216afe..ca427468a 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -115,6 +115,12 @@ pre-commit: exclude: 'frontend/(coverage|dist|node_modules|\.vite)/' run: scripts/pre-commit-hooks/semgrep-scan.sh {staged_files} + # --- Markdown --- + # Staged .md files only; generated/ignored paths are skipped inside the script. + markdownlint: + glob: "*.md" + run: scripts/pre-commit-hooks/markdownlint-staged.sh {staged_files} + # ============================================================ # MANUAL: security-full diff --git a/scripts/pre-commit-hooks/markdownlint-staged.sh b/scripts/pre-commit-hooks/markdownlint-staged.sh new file mode 100755 index 000000000..891b6a2d9 --- /dev/null +++ b/scripts/pre-commit-hooks/markdownlint-staged.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Lint the given markdown files (lefthook passes the staged ones) with markdownlint-cli2. +# +# Rules: .markdownlint.jsonc (a few noisy rules are off until the repo-wide cleanup). +# Skipped paths: docs-site/docs (generated), node_modules, CHANGELOG*, plus everything listed +# in .markdownlintignore. Exits 0 without running anything when no lintable files remain. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +readonly REPO_ROOT + +cd "${REPO_ROOT}" + +# Lint the lefthook-supplied files; with no args, lint every tracked markdown file (make lint-markdown). +if [ "$#" -gt 0 ]; then + CANDIDATES=("$@") +else + mapfile -t CANDIDATES < <(git ls-files '*.md') +fi + +# Build ignore patterns: fixed generated paths plus .markdownlintignore (gitignore-style subset: +# "dir/" matches the whole directory, anything else matches by glob at any depth). +IGNORES=('docs-site/docs/**' '**/node_modules/**' '**/CHANGELOG*') +if [ -f .markdownlintignore ]; then + while IFS= read -r line; do + case "${line}" in + '' | '#'*) continue ;; + */) IGNORES+=("${line}**") ;; + *) IGNORES+=("**/${line}") ;; + esac + done < .markdownlintignore +fi + +# Drop deleted files and ignored paths (extglob-free matching via bash [[ == ]] globs). +FILES=() +for f in "${CANDIDATES[@]}"; do + [ -f "${f}" ] || continue + skip=0 + for pat in "${IGNORES[@]}"; do + # shellcheck disable=SC2053 # intentional glob match against the pattern + if [[ "${f}" == ${pat} || "/${f}" == ${pat} || "${f}" == ${pat#\*\*/} ]]; then + skip=1 + break + fi + done + [ "${skip}" -eq 0 ] && FILES+=("${f}") +done + +if [ "${#FILES[@]}" -eq 0 ]; then + echo "markdownlint: no lintable markdown files, skipping" + exit 0 +fi + +if [ ! -x node_modules/.bin/markdownlint-cli2 ]; then + echo "Error: markdownlint-cli2 not found. Run 'npm ci' at the repo root." >&2 + exit 127 +fi + +exec node_modules/.bin/markdownlint-cli2 "${FILES[@]}" diff --git a/tests/security-enforcement/zzz-security-ui/crowdsec-import.spec.ts b/tests/security-enforcement/zzz-security-ui/crowdsec-import.spec.ts index 9aeb2dfa8..36551da3b 100644 --- a/tests/security-enforcement/zzz-security-ui/crowdsec-import.spec.ts +++ b/tests/security-enforcement/zzz-security-ui/crowdsec-import.spec.ts @@ -10,6 +10,7 @@ * - Import Execution (3 tests): import success, error handling, already exists */ +import type { Page } from '@playwright/test'; import { test, expect, loginUser } from '../../fixtures/auth-fixtures'; import { cleanupBackupsCreatedSince } from '../../utils/backup-cleanup'; import { waitForLoadingComplete } from '../../utils/wait-helpers'; @@ -297,4 +298,79 @@ test.describe('Import CrowdSec Configuration', () => { await expect(importButton).toBeEnabled(); }); }); + + // ========================================================================= + // Blanked notification values + // ========================================================================= + test.describe('Notification files that need values', () => { + const NEEDS_VALUES_LINE = '[data-testid="import-needs-values"]'; + + /** Mocks the backup and import answers, then imports a file through the CrowdSec page. */ + async function importWithResult(page: Page, result: Record) { + await page.route('**/api/v1/backups', async (route) => { + if (route.request().method() === 'POST') { + await route.fulfill({ + status: 201, + json: { filename: 'pre-import-backup.tar.gz', size: 1000, time: new Date().toISOString() }, + }); + } else { + await route.continue(); + } + }); + await page.route('**/api/v1/admin/crowdsec/import', (route) => + route.fulfill({ status: 200, json: { status: 'imported', backup: 'backup-1', ...result } }), + ); + await page.goto('/security/crowdsec'); + await waitForLoadingComplete(page); + await page.getByTestId('import-file').setInputFiles({ + name: 'crowdsec-config.tar.gz', + mimeType: 'application/gzip', + buffer: createMockTarGzBuffer(), + }); + await page.getByTestId('import-btn').click(); + } + + test('shows the files that need values inline in the import card', async ({ page, adminUser }) => { + await loginUser(page, adminUser); + + await importWithResult(page, { needs_values: ['notifications/http_default.yaml'] }); + + const line = page.locator(NEEDS_VALUES_LINE); + await expect(line).toBeVisible(); + await expect(line).toContainText('notifications/http_default.yaml'); + }); + + test('adds no notification beyond the existing success toast', async ({ page, adminUser }) => { + await loginUser(page, adminUser); + + await importWithResult(page, { needs_values: ['notifications/http_default.yaml'] }); + + await expect(page.locator(NEEDS_VALUES_LINE)).toBeVisible(); + await expect(page.locator(SELECTORS.successToast)).toHaveCount(1); + await expect(page.getByRole('alert')).toHaveCount(0); + }); + + test('shows no line when no file needs values', async ({ page, adminUser }) => { + await loginUser(page, adminUser); + + await importWithResult(page, {}); + + await expect(page.locator(SELECTORS.successToast)).toBeVisible(); + await expect(page.locator(NEEDS_VALUES_LINE)).toHaveCount(0); + }); + + test('removes the line when another file is chosen', async ({ page, adminUser }) => { + await loginUser(page, adminUser); + await importWithResult(page, { needs_values: ['notifications/http_default.yaml'] }); + await expect(page.locator(NEEDS_VALUES_LINE)).toBeVisible(); + + await page.getByTestId('import-file').setInputFiles({ + name: 'other-config.tar.gz', + mimeType: 'application/gzip', + buffer: createMockTarGzBuffer(), + }); + + await expect(page.locator(NEEDS_VALUES_LINE)).toHaveCount(0); + }); + }); }); diff --git a/tests/security/crowdsec-export-redaction.spec.ts b/tests/security/crowdsec-export-redaction.spec.ts new file mode 100644 index 000000000..329956ce7 --- /dev/null +++ b/tests/security/crowdsec-export-redaction.spec.ts @@ -0,0 +1,223 @@ +/** + * CrowdSec export: blanked notification values - E2E (API level) + * + * An export blanks the passwords, tokens, addresses and header values of notification files and + * keeps the structure a profile needs. An import keeps the current values when the notification + * already exists, and lists the files that still need values when it does not. + */ + +import { promises as fs } from 'fs'; +import type { APIRequestContext } from '@playwright/test'; +import { test, expect } from '@playwright/test'; +import { + BLANKED_VALUE, + createTarGz, + listTarGzEntries, + notificationYamlWithValues, + profilesYaml, + slackNotificationYamlWithAlias, + readTarGzEntry, +} from '../utils/archive-helpers'; + +const ADMIN = '/api/v1/admin/crowdsec'; +const NOTIFICATION = 'http_default'; +const NOTIFICATION_PATH = `notifications/${NOTIFICATION}.yaml`; +const ALIASED = 'slack_alias'; +const ALIASED_PATH = `notifications/${ALIASED}.yaml`; + +const VALID_CONFIG = `api: + server: + listen_uri: 0.0.0.0:8080 + log_level: info +`; + +/** Fake values planted in the live notification file. */ +const PLANTED = { + url: 'https://notify.example.invalid/hook/SECRETTOKEN-url', + authorization: 'Bearer SECRETTOKEN-auth', + apiToken: 'SECRETTOKEN-header', +}; +const PLANTED_TOKENS = ['SECRETTOKEN-url', 'SECRETTOKEN-auth', 'SECRETTOKEN-header']; + +/** The path of a configuration file as the file API lists and reads it. */ +const cfg = (name: string) => `config/${name}`; + +async function upload(request: APIRequestContext, archivePath: string, uploadName: string) { + return request.post(`${ADMIN}/import`, { + multipart: { file: { name: uploadName, mimeType: 'application/gzip', buffer: await fs.readFile(archivePath) } }, + timeout: 120_000, + }); +} + +async function readConfigFile(request: APIRequestContext, name: string) { + const response = await request.get(`${ADMIN}/file?path=${encodeURIComponent(name)}`); + expect(response.status()).toBe(200); + return (await response.json()).content as string; +} + +async function exportArchive(request: APIRequestContext) { + const response = await request.get(`${ADMIN}/export`, { timeout: 120_000 }); + expect(response.status()).toBe(200); + return response.body(); +} + +/** Imports an archive with the notification file (and a profile naming it) and expects success. */ +async function seedNotification( + request: APIRequestContext, + outputPath: string, + notification: string, + extraFiles: Record = {}, +) { + const archive = await createTarGz( + { + 'config/config.yaml': VALID_CONFIG, + 'config/profiles.yaml': profilesYaml([NOTIFICATION]), + [cfg(NOTIFICATION_PATH)]: notification, + ...extraFiles, + }, + outputPath, + ); + const response = await upload(request, archive, 'seed.tar.gz'); + expect(response.status()).toBe(200); + return response; +} + +test.describe('CrowdSec export: blanked notification values', () => { + test.beforeEach(async ({ request }, testInfo) => { + await seedNotification(request, testInfo.outputPath('seed.tar.gz'), notificationYamlWithValues(NOTIFICATION)); + }); + + test('export blanks the address and header values of a notification file', async ({ request }) => { + const entry = await readTarGzEntry(await exportArchive(request), cfg(NOTIFICATION_PATH)); + + expect(entry).not.toBeNull(); + expect(entry).toContain(BLANKED_VALUE); + for (const token of PLANTED_TOKENS) { + expect(entry).not.toContain(token); + } + }); + + test('export keeps no planted value anywhere in the archive', async ({ request }) => { + const archive = await exportArchive(request); + + const names = await listTarGzEntries(archive); + for (const name of names) { + const content = (await readTarGzEntry(archive, name)) ?? ''; + for (const token of PLANTED_TOKENS) { + expect(content, `${name} must not hold ${token}`).not.toContain(token); + expect(name).not.toContain(token); + } + } + }); + + test('export keeps the keys a profile and the plugin need readable', async ({ request }) => { + const entry = (await readTarGzEntry(await exportArchive(request), cfg(NOTIFICATION_PATH))) as string; + + expect(entry).toContain('type: http'); + expect(entry).toContain(`name: ${NOTIFICATION}`); + expect(entry).toContain('log_level: info'); + expect(entry).toContain('method: POST'); + // The profile still names a notification the exported archive defines. + const profiles = (await readTarGzEntry(await exportArchive(request), cfg('profiles.yaml'))) as string; + expect(profiles).toContain(NOTIFICATION); + }); + + test('export drops comments from a notification file', async ({ request }, testInfo) => { + await seedNotification( + request, + testInfo.outputPath('seed-comments.tar.gz'), + notificationYamlWithValues(NOTIFICATION, { comments: true }), + ); + + const entry = (await readTarGzEntry(await exportArchive(request), cfg(NOTIFICATION_PATH))) as string; + + expect(entry).not.toContain('SECRETTOKEN-comment'); + }); + + test('export leaves out stray non-YAML files from the notifications folder', async ({ request }, testInfo) => { + await seedNotification(request, testInfo.outputPath('seed-stray.tar.gz'), notificationYamlWithValues(NOTIFICATION), { + [cfg(`${NOTIFICATION_PATH}.bak`)]: notificationYamlWithValues(NOTIFICATION), + }); + + const names = await listTarGzEntries(await exportArchive(request)); + + expect(names).toContain(cfg(NOTIFICATION_PATH)); + expect(names).not.toContain(cfg(`${NOTIFICATION_PATH}.bak`)); + }); + + test('importing an export on the same system keeps the current values', async ({ request }, testInfo) => { + const archivePath = testInfo.outputPath('roundtrip.tar.gz'); + await fs.writeFile(archivePath, await exportArchive(request)); + + const response = await upload(request, archivePath, 'roundtrip.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).needs_values).toBeUndefined(); + const content = await readConfigFile(request, cfg(NOTIFICATION_PATH)); + expect(content).toContain(PLANTED.url); + expect(content).toContain(PLANTED.authorization); + expect(content).toContain(PLANTED.apiToken); + expect(content).not.toContain(BLANKED_VALUE); + }); + + test('a file that uses aliases is listed as needing values and stays as exported', async ({ request }, testInfo) => { + await seedNotification(request, testInfo.outputPath('seed-aliased.tar.gz'), notificationYamlWithValues(NOTIFICATION), { + 'config/profiles.yaml': profilesYaml([NOTIFICATION, ALIASED]), + [cfg(ALIASED_PATH)]: slackNotificationYamlWithAlias(ALIASED), + }); + const exported = await exportArchive(request); + const exportedEntry = (await readTarGzEntry(exported, cfg(ALIASED_PATH))) as string; + const archivePath = testInfo.outputPath('roundtrip-aliased.tar.gz'); + await fs.writeFile(archivePath, exported); + + const response = await upload(request, archivePath, 'roundtrip-aliased.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).needs_values).toEqual([ALIASED_PATH]); + // Excluded from the "values unchanged" comparison: the file comes back exactly as exported. + expect(await readConfigFile(request, cfg(ALIASED_PATH))).toBe(exportedEntry); + }); + + test('a file with an anchor that nothing refers to merges like any other file', async ({ request }, testInfo) => { + await seedNotification( + request, + testInfo.outputPath('seed-anchored.tar.gz'), + notificationYamlWithValues(NOTIFICATION, { anchored: true }), + ); + const archivePath = testInfo.outputPath('roundtrip-anchored.tar.gz'); + await fs.writeFile(archivePath, await exportArchive(request)); + + const response = await upload(request, archivePath, 'roundtrip-anchored.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).needs_values).toBeUndefined(); + const content = await readConfigFile(request, cfg(NOTIFICATION_PATH)); + expect(content).toContain(PLANTED.authorization); + expect(content).not.toContain(BLANKED_VALUE); + }); + + test('importing a blanked file where the notification does not exist keeps the placeholders and lists it', async ({ + request, + }, testInfo) => { + const exported = await exportArchive(request); + // A fresh host: replace the live notification with a different one first. + await seedNotification( + request, + testInfo.outputPath('seed-other.tar.gz'), + notificationYamlWithValues('http_other'), + { 'config/profiles.yaml': profilesYaml(['http_other']) }, + ); + const archivePath = testInfo.outputPath('fresh.tar.gz'); + await fs.writeFile(archivePath, exported); + + const response = await upload(request, archivePath, 'fresh.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).needs_values).toEqual([NOTIFICATION_PATH]); + const content = await readConfigFile(request, cfg(NOTIFICATION_PATH)); + expect(content).toContain(BLANKED_VALUE); + for (const token of PLANTED_TOKENS) { + expect(content).not.toContain(token); + } + }); +}); diff --git a/tests/utils/archive-helpers.ts b/tests/utils/archive-helpers.ts index a3ec3cf22..5fa0bb023 100644 --- a/tests/utils/archive-helpers.ts +++ b/tests/utils/archive-helpers.ts @@ -331,3 +331,60 @@ method: POST ) .join('---\n'); } + +/** The text a CrowdSec export writes in place of a blanked notification value. */ +export const BLANKED_VALUE = ''; + +export interface NotificationValues { + /** Address the notification is sent to. */ + url?: string; + /** Value of the Authorization header. */ + authorization?: string; + /** Value of a second custom header. */ + apiToken?: string; + /** Write the headers under a YAML anchor that nothing refers to (the export keeps no alias). */ + anchored?: boolean; + /** Add comments to the document. */ + comments?: boolean; +} + +/** + * A notification plugin config document that carries values a CrowdSec export blanks + * (the address, header values) next to values it keeps (type, name, log level, method). + * Use obviously fake `SECRETTOKEN-` strings for the values. + */ +export function notificationYamlWithValues(name: string, values: NotificationValues = {}): string { + const url = values.url ?? 'https://notify.example.invalid/hook/SECRETTOKEN-url'; + const authorization = values.authorization ?? 'Bearer SECRETTOKEN-auth'; + const apiToken = values.apiToken ?? 'SECRETTOKEN-header'; + const headers = values.anchored + ? `headers: &shared + Authorization: ${authorization} + X-Api-Token: ${apiToken} +` + : `headers: + Authorization: ${authorization} + X-Api-Token: ${apiToken} +`; + const comments = values.comments ? '# SECRETTOKEN-comment\n' : ''; + return `${comments}type: http +name: ${name} +log_level: info +format: "" +url: ${url} +method: POST +${headers}`; +} + +/** + * A Slack notification file that reuses a kept value (the channel) through an anchor and an alias, + * so an export of it still holds an alias. + */ +export function slackNotificationYamlWithAlias(name: string): string { + return `type: slack +name: ${name} +webhook: https://hooks.example.invalid/SECRETTOKEN-url +channel: &c ops +username: *c +`; +}