diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 57aa76621..2929cdaff 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -888,7 +888,13 @@ backups (`.filebackup.`, newest 10 kept). Import and rollbac server-local state untouched: the installed hub tree and its links, runtime state, and the proxy/bouncer registration. Export is built in a temporary file under the data lock and sent only after it is complete and within the -importable limits; otherwise a clean error is returned. One handler-level +importable limits; otherwise a clean error is returned. The export also carries +a small manifest listing the installed hub items by name, and an import verifies +that the profiles' notification references resolve before it changes anything. +Installing missing hub items is a separate action: it takes the data lock for +one hub operation at a time, so it never overlaps an import or preset apply, a +single-flight guard allows only one restore to run at a time, and the whole +action has a bounded overall time. One handler-level lock serializes apply, import, and file saves. The file editor validates file type, size and YAML, and the frontend no longer has a local-write fallback for presets; failures are shown as errors. diff --git a/backend/internal/api/handlers/crowdsec_export_import_test.go b/backend/internal/api/handlers/crowdsec_export_import_test.go index c44b1a815..f3dceaced 100644 --- a/backend/internal/api/handlers/crowdsec_export_import_test.go +++ b/backend/internal/api/handlers/crowdsec_export_import_test.go @@ -106,9 +106,11 @@ func TestExportConfigRealLayoutEntryList(t *testing.T) { seedRealLayout(t, dir) _, r := newImportRouter(t, dir) + // The fixture's hub links are not exported themselves; they are listed in the generated manifest. require.Equal(t, []string{ "bouncer_key.bak", "caddy.yaml", + crowdsec.ManifestEntryName, "config/acquis.yaml", "config/collections/placeholder.yaml", "config/config.yaml", diff --git a/backend/internal/api/handlers/crowdsec_files.go b/backend/internal/api/handlers/crowdsec_files.go index 56cf63d3a..35a967973 100644 --- a/backend/internal/api/handlers/crowdsec_files.go +++ b/backend/internal/api/handlers/crowdsec_files.go @@ -199,31 +199,14 @@ func checkSize(content string) *fileError { // document's node, or io.EOF when the stream has no document. With firstOnly set, later documents are // never read; otherwise every document must parse. func parseYAMLDocuments(r io.Reader, firstOnly bool) (*yaml.Node, error) { - dec := yaml.NewDecoder(r) - var first *yaml.Node - for { - var node yaml.Node - err := dec.Decode(&node) - if errors.Is(err, io.EOF) { - if first == nil { - return nil, io.EOF - } - return first, nil - } - if err != nil { - return nil, fmt.Errorf("decode yaml document: %w", err) - } - var doc any - if err := node.Decode(&doc); err != nil { - return nil, fmt.Errorf("decode yaml value: %w", err) - } - if first == nil { - first = &node - } - if firstOnly { - return first, nil - } + docs, err := crowdsec.ParseYAMLDocuments(r, firstOnly) + if err != nil { + return nil, err + } + if len(docs) == 0 { + return nil, io.EOF } + return docs[0], nil } // checkYAML verifies that YAML files parse; other allowed types are not inspected. diff --git a/backend/internal/api/handlers/crowdsec_handler.go b/backend/internal/api/handlers/crowdsec_handler.go index 9f6e0c7a4..378609aeb 100644 --- a/backend/internal/api/handlers/crowdsec_handler.go +++ b/backend/internal/api/handlers/crowdsec_handler.go @@ -87,6 +87,9 @@ type CrowdsecHandler struct { // Lock order is always dataMu first, then HubService.mu. dataMu sync.Mutex + // restore holds the state of the hub item restore action (crowdsec_hub_items.go). + restore hubRestoreState + // registrationMutex protects concurrent bouncer registration attempts registrationMutex sync.Mutex @@ -716,6 +719,19 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { return } + // One pass over the upload feeds the pre-import checks and the hub item report. Refuse an archive + // CrowdSec could not start with before taking the lock or changing anything. + info, inspectErr := crowdsec.InspectArchive(dst) + if inspectErr != nil { + logger.Log().WithField("error", sanitizeForLog(inspectErr.Error())).Warn("crowdsec import: archive inspection skipped") + } else if err = crowdsec.VerifyInspectedNotifications(info); err != nil { + var undefined *crowdsec.UndefinedNotificationsError + if errors.As(err, &undefined) { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "config validation failed: " + undefined.Error()}) + return + } + } + // Serialize with preset applies and file writes, then snapshot (copy, never rename) the config. h.dataMu.Lock() defer h.dataMu.Unlock() @@ -764,7 +780,9 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { return } - c.JSON(http.StatusOK, gin.H{"status": "imported", "backup": crowdsec.BackupID(backupDir)}) + resp := gin.H{"status": "imported", "backup": crowdsec.BackupID(backupDir)} + h.addHubItemsReport(resp, info.HubItems) + c.JSON(http.StatusOK, resp) } // pruneSnapshots bounds the full-tree snapshots kept next to DataDir; failures are logged only. @@ -805,15 +823,14 @@ func (h *CrowdsecHandler) extractArchive(archivePath, destDir string, keep func( return fmt.Errorf("failed to read tar header: %w", err) } - // Path traversal protection - // #nosec G305 -- Path traversal is explicitly checked below - target := filepath.Join(destDir, header.Name) - if !strings.HasPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) { + // Path traversal protection, shared with the pre-import inspection so both see the same entry. + rel, ok := crowdsec.ArchiveEntryRel(header.Name) + if !ok { return fmt.Errorf("invalid file path: %s", header.Name) } + target := filepath.Join(destDir, rel) // Server-local state (engine-owned data, stored account details, the hub tree, live symlinks) is // never taken from an upload. This runs only after the containment check above. - rel := strings.TrimPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) if skipArchiveEntry(liveDirs, rel, header.Typeflag, keep) { continue } @@ -2382,6 +2399,7 @@ func (h *CrowdsecHandler) RegisterRoutes(rg *gin.RouterGroup) { rg.POST("/admin/crowdsec/stop", h.Stop) rg.GET("/admin/crowdsec/status", h.Status) rg.POST("/admin/crowdsec/import", h.ImportConfig) + rg.POST("/admin/crowdsec/hub-items/restore", h.RestoreHubItems) rg.GET("/admin/crowdsec/export", h.ExportConfig) rg.GET("/admin/crowdsec/files", h.ListFiles) rg.GET("/admin/crowdsec/file", h.ReadFile) @@ -2472,6 +2490,9 @@ func (h *CrowdsecHandler) reloadAfterWhitelistChange(ctx context.Context, action // (after the clear step such a directory remains only because it holds kept entries). rel has passed // the containment check. func skipArchiveEntry(liveDirs map[string]struct{}, rel string, typeflag byte, keep func(rel string) bool) bool { + if crowdsec.IsManifestPath(rel) { + return true // generated hub item list, read by the pre-import inspection, never written to disk + } if crowdsec.KeptOrBeneathKept(keep, rel) { return true } diff --git a/backend/internal/api/handlers/crowdsec_hub_items.go b/backend/internal/api/handlers/crowdsec_hub_items.go new file mode 100644 index 000000000..087fa2c70 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_hub_items.go @@ -0,0 +1,331 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "sync/atomic" + "time" + + "github.com/gin-gonic/gin" + + "github.com/Wikid82/charon/backend/internal/crowdsec" + "github.com/Wikid82/charon/backend/internal/logger" +) + +// addHubItemsReport adds the hub_items field to an import response when the archive carried a usable +// manifest. It diffs against the items installed right now, so the caller must hold dataMu. If the +// installed items cannot be read the field is left out rather than reporting everything as missing. +func (h *CrowdsecHandler) addHubItemsReport(resp map[string]any, manifest *crowdsec.ArchiveHubItems) { + if manifest == nil { + return + } + installed, err := crowdsec.InstalledHubItems(h.DataDir) + if err != nil { + logger.Log().WithError(err).Warn("crowdsec import: installed hub items could not be read") + return + } + resp["hub_items"] = manifest.Report(installed) +} + +// Limits of one restore action; see hubRestoreState.limits for the test override. +const ( + hubRestoreMaxBodyBytes = 1 << 20 + hubRestoreOverall = 5 * time.Minute + hubRestoreItemTimeout = 120 * time.Second + hubRestoreUpdateTimeout = 60 * time.Second + hubRestoreReloadTimeout = 10 * time.Second + hubRestoreLogLimit = 300 +) + +// Fixed per-item reasons of a restore. They are the only failure detail a client ever receives. +const ( + hubReasonNotFound = "not_found" + hubReasonInstallFail = "install_failed" + hubReasonVerifyFail = "verify_failed" + hubReasonNotAttempted = "not_attempted" +) + +// Fixed messages of the restore endpoint. +const ( + msgRestoreInvalidItem = "invalid hub item" + msgRestoreNoItems = "no hub items given" + msgRestoreBadRequest = "invalid request" + msgRestoreTooLarge = "request too large" + msgRestoreRunning = "a restore is already running" + msgRestoreHubDown = "the CrowdSec hub could not be reached; nothing was installed" + msgRestoreReadFailed = "the installed hub items could not be read; nothing was installed" +) + +// hubRestoreLimits are the time budgets of one restore. The zero value means the production budgets. +type hubRestoreLimits struct { + overall time.Duration + item time.Duration + update time.Duration +} + +func (l hubRestoreLimits) withDefaults() hubRestoreLimits { + if l.overall <= 0 { + l.overall = hubRestoreOverall + } + if l.item <= 0 { + l.item = hubRestoreItemTimeout + } + if l.update <= 0 { + l.update = hubRestoreUpdateTimeout + } + return l +} + +// hubRestoreState is the handler-wide state of the restore action. running is compare-and-swapped and +// never waited on, so a second restore is refused instead of queued. Lock order is always +// running -> CrowdsecHandler.dataMu -> HubService.mu. +type hubRestoreState struct { + running atomic.Bool + limits hubRestoreLimits +} + +type hubRestoreRequest struct { + Items []crowdsec.HubItem `json:"items"` +} + +type hubRestoreFailure struct { + Type string `json:"type"` + Name string `json:"name"` + Reason string `json:"reason"` +} + +type hubRestoreResponse struct { + Installed []crowdsec.HubItem `json:"installed"` + AlreadyInstalled []crowdsec.HubItem `json:"already_installed"` + Failed []hubRestoreFailure `json:"failed"` + Remaining int `json:"remaining"` + SkippedInvalid int `json:"skipped_invalid"` + Reloaded bool `json:"reloaded"` +} + +// RestoreHubItems installs the requested hub items that are not installed yet: POST +// /admin/crowdsec/hub-items/restore. Every hub write takes dataMu for exactly one cscli section, so an +// import, export, file save or preset apply waits for at most one bounded call and never overlaps an +// install. +func (h *CrowdsecHandler) RestoreHubItems(c *gin.Context) { + items, status, msg := parseHubRestoreRequest(c) + if status != 0 { + c.JSON(status, gin.H{"error": msg}) + return + } + + if !h.restore.running.CompareAndSwap(false, true) { + c.JSON(http.StatusConflict, gin.H{"error": msgRestoreRunning}) + return + } + defer h.restore.running.Store(false) + + limits := h.restore.limits.withDefaults() + ctx, cancel := context.WithTimeout(c.Request.Context(), limits.overall) + defer cancel() + + if err := h.hubUpdate(ctx, limits.update); err != nil { + logger.Log().WithField("error", hubLogDetail(err.Error())).Warn("crowdsec hub items restore: hub update failed") + c.JSON(http.StatusBadGateway, gin.H{"error": msgRestoreHubDown}) + return + } + + res := &hubRestoreResponse{ + Installed: []crowdsec.HubItem{}, + AlreadyInstalled: []crowdsec.HubItem{}, + Failed: []hubRestoreFailure{}, + } + var collections, others []crowdsec.HubItem + for _, it := range items { + if it.Type == "collections" { + collections = append(collections, it) + } else { + others = append(others, it) + } + } + + // Collections first: they bring most scenarios and parsers with them, so the rest is re-diffed + // against a fresh locked read. + if !h.restorePass(ctx, limits.item, collections, res) { + c.JSON(http.StatusInternalServerError, gin.H{"error": msgRestoreReadFailed}) + return + } + if !h.restorePass(ctx, limits.item, others, res) { + for _, it := range others { + res.Failed = append(res.Failed, hubRestoreFailure{it.Type, it.Name, hubReasonNotAttempted}) + } + } + for _, f := range res.Failed { + if f.Reason == hubReasonNotAttempted { + res.Remaining++ + } + } + + if len(res.Installed) > 0 { + res.Reloaded = h.reloadAfterRestore(ctx) + } + c.JSON(http.StatusOK, res) +} + +// parseHubRestoreRequest reads and validates the body. A non-zero status carries the error to send. +func parseHubRestoreRequest(c *gin.Context) (items []crowdsec.HubItem, status int, msg string) { + body := http.MaxBytesReader(c.Writer, c.Request.Body, hubRestoreMaxBodyBytes) + var req hubRestoreRequest + if err := json.NewDecoder(body).Decode(&req); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return nil, http.StatusRequestEntityTooLarge, msgRestoreTooLarge + } + return nil, http.StatusBadRequest, msgRestoreBadRequest + } + if len(req.Items) == 0 { + return nil, http.StatusBadRequest, msgRestoreNoItems + } + if len(req.Items) > crowdsec.MaxHubItems { + return nil, http.StatusBadRequest, msgRestoreInvalidItem + } + seen := make(map[crowdsec.HubItem]struct{}, len(req.Items)) + for _, it := range req.Items { + if !it.Valid() { + return nil, http.StatusBadRequest, msgRestoreInvalidItem + } + if _, dup := seen[it]; dup { + continue + } + seen[it] = struct{}{} + items = append(items, it) + } + return items, 0, "" +} + +// hubUpdate refreshes the hub index in its own locked section. +func (h *CrowdsecHandler) hubUpdate(ctx context.Context, limit time.Duration) error { + h.dataMu.Lock() + defer h.dataMu.Unlock() + uctx, cancel := context.WithTimeout(ctx, limit) + defer cancel() + if out, err := h.CmdExec.Execute(uctx, "cscli", "hub", "update"); err != nil { + return fmt.Errorf("hub update: %w (%s)", err, hubLogDetail(string(out))) + } + return nil +} + +// installedHubSet reads the installed items under dataMu, so it never observes a tree an import is +// clearing. +func (h *CrowdsecHandler) installedHubSet() (map[crowdsec.HubItem]struct{}, error) { + h.dataMu.Lock() + defer h.dataMu.Unlock() + installed, err := crowdsec.InstalledHubItems(h.DataDir) + if err != nil { + return nil, fmt.Errorf("read installed hub items: %w", err) + } + set := make(map[crowdsec.HubItem]struct{}, len(installed)) + for _, it := range installed { + set[it] = struct{}{} + } + return set, nil +} + +// restorePass installs the items of one pass that are not installed, recording every outcome in res. +// It returns false, having recorded nothing, when the installed items cannot be read. +func (h *CrowdsecHandler) restorePass(ctx context.Context, itemLimit time.Duration, items []crowdsec.HubItem, res *hubRestoreResponse) bool { + if len(items) == 0 { + return true + } + installed, err := h.installedHubSet() + if err != nil { + logger.Log().WithField("error", hubLogDetail(err.Error())).Warn("crowdsec hub items restore: cannot read installed items") + return false + } + for _, it := range items { + if _, ok := installed[it]; ok { + res.AlreadyInstalled = append(res.AlreadyInstalled, it) + continue + } + if reason := h.installHubItem(ctx, itemLimit, it); reason != "" { + res.Failed = append(res.Failed, hubRestoreFailure{it.Type, it.Name, reason}) + continue + } + res.Installed = append(res.Installed, it) + } + return true +} + +// installHubItem installs and verifies one item as a single critical section under dataMu, bounded by +// min(remaining overall deadline, itemLimit). It returns "" on success or a fixed reason. +func (h *CrowdsecHandler) installHubItem(ctx context.Context, itemLimit time.Duration, it crowdsec.HubItem) string { + if ctx.Err() != nil { + return hubReasonNotAttempted + } + h.dataMu.Lock() + defer h.dataMu.Unlock() + + // The lock wait may have used up the budget or the client may have left meanwhile. + limit := itemLimit + if dl, ok := ctx.Deadline(); ok { + limit = min(limit, time.Until(dl)) + } + if ctx.Err() != nil || limit <= 0 { + return hubReasonNotAttempted + } + ictx, cancel := context.WithTimeout(ctx, limit) + defer cancel() + + // Literal binary, no shell, one item per call; type and name passed ValidHubItem checks and cannot + // read as a flag. + out, err := h.CmdExec.Execute(ictx, "cscli", it.Type, "install", it.Name) + if err != nil { + logger.Log(). + WithField("type", hubLogDetail(it.Type)).WithField("name", hubLogDetail(it.Name)). + WithField("error", hubLogDetail(err.Error())).WithField("output", hubLogDetail(string(out))). + Warn("crowdsec hub items restore: install failed") + if hubItemNotFound(out) { + return hubReasonNotFound + } + return hubReasonInstallFail + } + if err := crowdsec.VerifyHubItem(ictx, h.CmdExec, it.Type, it.Name); err != nil { + logger.Log(). + WithField("type", hubLogDetail(it.Type)).WithField("name", hubLogDetail(it.Name)). + WithField("error", hubLogDetail(err.Error())). + Warn("crowdsec hub items restore: verification failed") + return hubReasonVerifyFail + } + return "" +} + +// hubItemNotFound reports whether cscli's output says the hub does not know the item. This is a +// best-effort match on cscli's wording; anything else is install_failed. +func hubItemNotFound(out []byte) bool { + s := strings.ToLower(string(out)) + return strings.Contains(s, "can't find") || strings.Contains(s, "not found") +} + +// reloadAfterRestore asks CrowdSec to reload on a detached, bounded context so an install that succeeded +// just before a client disconnect or the deadline is still picked up. Best-effort: it reports whether a +// reload was performed. +func (h *CrowdsecHandler) reloadAfterRestore(ctx context.Context) bool { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), hubRestoreReloadTimeout) + defer cancel() + switch err := h.reloadCrowdSec(rctx); { + case err == nil: + return true + case errors.Is(err, crowdsec.ErrCrowdSecNotRunning): + default: + logger.Log().WithField("error", hubLogDetail(err.Error())).Warn("crowdsec hub items restore: reload failed (non-fatal)") + } + return false +} + +// hubLogDetail sanitizes and truncates cscli detail for the log; it is never sent to a client. +func hubLogDetail(s string) string { + s = sanitizeForLog(s) + if len(s) > hubRestoreLogLimit { + s = s[:hubRestoreLogLimit] + } + return s +} diff --git a/backend/internal/api/handlers/crowdsec_hub_items_test.go b/backend/internal/api/handlers/crowdsec_hub_items_test.go new file mode 100644 index 000000000..a630287e2 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_hub_items_test.go @@ -0,0 +1,233 @@ +package handlers + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Wikid82/charon/backend/internal/crowdsec" + "github.com/stretchr/testify/require" +) + +// hubLink creates the symlink an installed hub item leaves behind. +func hubLink(t *testing.T, dir, rel, target string) { + t.Helper() + p := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(t, os.Symlink(target, p)) +} + +type rawEntry struct { + name string + body string + flag byte +} + +// rawArchive builds a tar.gz with exactly the given entry names, in order. +func rawArchive(t *testing.T, entries []rawEntry) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for _, e := range entries { + flag := e.flag + if flag == 0 { + flag = tar.TypeReg + } + hdr := &tar.Header{Name: e.name, Mode: 0o600, Typeflag: flag} + if flag == tar.TypeReg { + hdr.Size = int64(len(e.body)) + } + require.NoError(t, tw.WriteHeader(hdr)) + if flag == tar.TypeReg { + _, err := tw.Write([]byte(e.body)) + require.NoError(t, err) + } + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func importHubResponse(t *testing.T, r http.Handler, archive []byte) map[string]json.RawMessage { + t.Helper() + w := httptest.NewRecorder() + r.ServeHTTP(w, importBytesRequest(t, archive)) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var body map[string]json.RawMessage + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + require.JSONEq(t, `"imported"`, string(body["status"])) + return body +} + +func TestExportThenImportReportsMissingHubItems(t *testing.T) { + t.Parallel() + const hub = "/etc/crowdsec/hub/" + src := t.TempDir() + seedRealLayout(t, src) + hubLink(t, src, "config/collections/sftpgo.yaml", hub+"collections/Azlaroc/sftpgo.yaml") + hubLink(t, src, "config/scenarios/CVE-2022-26134.yaml", hub+"scenarios/crowdsecurity/CVE-2022-26134.yaml") + hubLink(t, src, "config/parsers/s02-enrich/geoip-enrich.yaml", hub+"parsers/s02-enrich/crowdsecurity/geoip-enrich.yaml") + _, srcRouter := newImportRouter(t, src) + exported := exportBytes(t, srcRouter) + require.Contains(t, archiveNames(t, exported), crowdsec.ManifestEntryName) + + // The target only has the geoip parser (and a link of its own that is not in the export). + dst := t.TempDir() + seedRealLayout(t, dst) + hubLink(t, dst, "config/parsers/s02-enrich/geoip-enrich.yaml", hub+"parsers/s02-enrich/crowdsecurity/geoip-enrich.yaml") + hubLink(t, dst, "config/scenarios/only-here.yaml", hub+"scenarios/target/only-here.yaml") + _, dstRouter := newImportRouter(t, dst) + + body := importHubResponse(t, dstRouter, exported) + require.JSONEq(t, `{"in_export":5,"skipped_invalid":0,"missing":[`+ + `{"type":"collections","name":"Azlaroc/sftpgo"},`+ + `{"type":"scenarios","name":"crowdsecurity/CVE-2022-26134"}]}`, string(body["hub_items"])) + require.NoFileExists(t, filepath.Join(dst, crowdsec.ManifestEntryName)) + require.FileExists(t, filepath.Join(dst, "config", "config.yaml")) + assertTargetLinkKept(t, dst) +} + +func assertTargetLinkKept(t *testing.T, dst string) { + t.Helper() + target, err := os.Readlink(filepath.Join(dst, "config", "scenarios", "only-here.yaml")) + require.NoError(t, err) + require.Equal(t, "/etc/crowdsec/hub/scenarios/target/only-here.yaml", target) +} + +func TestImportConfigNothingMissingReportsEmptyList(t *testing.T) { + t.Parallel() + dst := t.TempDir() + seedRealLayout(t, dst) + hubLink(t, dst, "config/collections/a.yaml", "/etc/crowdsec/hub/collections/o/a.yaml") + _, r := newImportRouter(t, dst) + body := importHubResponse(t, r, rawArchive(t, []rawEntry{ + {name: "config/config.yaml", body: importableConfig}, + {name: crowdsec.ManifestEntryName, body: `{"version":1,"items":[{"type":"collections","name":"o/a"}]}`}, + })) + require.JSONEq(t, `{"in_export":1,"skipped_invalid":0,"missing":[]}`, string(body["hub_items"])) +} + +func TestImportConfigLegacyArchiveHasNoHubItems(t *testing.T) { + t.Parallel() + dst := t.TempDir() + seedRealLayout(t, dst) + _, r := newImportRouter(t, dst) + body := importHubResponse(t, r, rawArchive(t, []rawEntry{{name: "config/config.yaml", body: importableConfig}})) + require.NotContains(t, body, "hub_items") + require.Contains(t, body, "backup") +} + +func TestImportConfigUnusableManifestIsIgnored(t *testing.T) { + t.Parallel() + good := `{"version":1,"items":[{"type":"collections","name":"o/a"}]}` + cfg := rawEntry{name: "config/config.yaml", body: importableConfig} + overCap := make([]string, 0, crowdsec.MaxHubItems+1) + for i := 0; i <= crowdsec.MaxHubItems; i++ { + overCap = append(overCap, fmt.Sprintf(`{"type":"collections","name":"o%x/%x"}`, i*7919, i*104729)) + } + cases := map[string][]rawEntry{ + "malformed": {cfg, {name: crowdsec.ManifestEntryName, body: `{"version":`}}, + "unknown version": {cfg, {name: crowdsec.ManifestEntryName, body: `{"version":2,"items":[]}`}}, + "over the cap": {cfg, {name: crowdsec.ManifestEntryName, body: `{"version":1,"items":[` + strings.Join(overCap, ",") + `]}`}}, + "duplicate": {cfg, {name: crowdsec.ManifestEntryName, body: good}, {name: "./" + crowdsec.ManifestEntryName, body: good}}, + "symlink entry": {cfg, {name: crowdsec.ManifestEntryName, flag: tar.TypeSymlink}}, + } + for name, entries := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + dst := t.TempDir() + seedRealLayout(t, dst) + _, r := newImportRouter(t, dst) + body := importHubResponse(t, r, rawArchive(t, entries)) + require.NotContains(t, body, "hub_items") + require.NoFileExists(t, filepath.Join(dst, crowdsec.ManifestEntryName)) + }) + } +} + +func TestImportConfigCountsInvalidManifestItems(t *testing.T) { + t.Parallel() + dst := t.TempDir() + seedRealLayout(t, dst) + _, r := newImportRouter(t, dst) + body := importHubResponse(t, r, rawArchive(t, []rawEntry{ + {name: "config/config.yaml", body: importableConfig}, + {name: crowdsec.ManifestEntryName, body: `{"version":1,"items":[` + + `{"type":"collections","name":"crowdsecurity/CVE-2022-26134"},{"type":"collections","name":"--help"},` + + `{"type":"appsec-rules","name":"a/b"},{"type":"parsers","name":"a/../b"}]}`}, + })) + require.JSONEq(t, `{"in_export":1,"skipped_invalid":3,"missing":[{"type":"collections","name":"crowdsecurity/CVE-2022-26134"}]}`, string(body["hub_items"])) +} + +func TestImportConfigManifestSpellingsNeverReachDisk(t *testing.T) { + t.Parallel() + good := `{"version":1,"items":[{"type":"collections","name":"o/a"}]}` + for _, spelling := range []string{crowdsec.ManifestEntryName, "./" + crowdsec.ManifestEntryName, "a/../" + crowdsec.ManifestEntryName, "config/../" + crowdsec.ManifestEntryName} { + t.Run(spelling, func(t *testing.T) { + t.Parallel() + dst := t.TempDir() + seedRealLayout(t, dst) + writeTree(t, dst, map[string]string{crowdsec.ManifestEntryName: "stale live file"}) + _, r := newImportRouter(t, dst) + body := importHubResponse(t, r, rawArchive(t, []rawEntry{ + {name: "config/config.yaml", body: importableConfig}, + {name: spelling, body: good}, + })) + require.JSONEq(t, `{"in_export":1,"skipped_invalid":0,"missing":[{"type":"collections","name":"o/a"}]}`, string(body["hub_items"])) + require.NoFileExists(t, filepath.Join(dst, crowdsec.ManifestEntryName)) + require.NoDirExists(t, filepath.Join(dst, "a")) + }) + } +} + +func TestImportConfigNestedManifestNameIsAnOrdinaryFile(t *testing.T) { + t.Parallel() + dst := t.TempDir() + seedRealLayout(t, dst) + _, r := newImportRouter(t, dst) + body := importHubResponse(t, r, rawArchive(t, []rawEntry{ + {name: "config/config.yaml", body: importableConfig}, + {name: "config/" + crowdsec.ManifestEntryName, body: "ordinary"}, + })) + require.NotContains(t, body, "hub_items") + require.Equal(t, "ordinary", readTreeFile(t, filepath.Join(dst, "config", crowdsec.ManifestEntryName))) + require.NoFileExists(t, filepath.Join(dst, crowdsec.ManifestEntryName)) +} + +func TestSkipArchiveEntrySkipsOnlyTheRootManifest(t *testing.T) { + t.Parallel() + keep := func(string) bool { return false } + for _, typeflag := range []byte{tar.TypeReg, tar.TypeDir, tar.TypeSymlink} { + require.True(t, skipArchiveEntry(map[string]struct{}{}, crowdsec.ManifestEntryName, typeflag, keep)) + } + require.False(t, skipArchiveEntry(map[string]struct{}{}, "config/"+crowdsec.ManifestEntryName, tar.TypeReg, keep)) +} + +func TestAddHubItemsReportOmitsFieldWhenScanFails(t *testing.T) { + t.Parallel() + if os.Getuid() == 0 { + t.Skip("permission checks do not apply to root") + } + dir := t.TempDir() + bad := filepath.Join(dir, "config", "scenarios") + require.NoError(t, os.MkdirAll(bad, 0o750)) + require.NoError(t, os.Chmod(bad, 0o000)) //nolint:gosec // G302: test makes a directory unreadable + t.Cleanup(func() { _ = os.Chmod(bad, 0o700) }) //nolint:gosec // G302: restore so TempDir cleanup works + h, _ := newImportRouter(t, dir) + + resp := map[string]any{} + h.addHubItemsReport(resp, &crowdsec.ArchiveHubItems{Items: []crowdsec.HubItem{{Type: "collections", Name: "o/a"}}}) + require.NotContains(t, resp, "hub_items") + + h.addHubItemsReport(resp, nil) + require.Empty(t, resp) +} diff --git a/backend/internal/api/handlers/crowdsec_hub_restore_test.go b/backend/internal/api/handlers/crowdsec_hub_restore_test.go new file mode 100644 index 000000000..6edf6e8b4 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_hub_restore_test.go @@ -0,0 +1,730 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/Wikid82/charon/backend/internal/crowdsec" +) + +const restorePath = "/api/v1/admin/crowdsec/hub-items/restore" + +// hubFakeExec is a scripted cscli. A successful install creates the link a real one would, so the +// installed-item diff sees it. +type hubFakeExec struct { + mu sync.Mutex + calls []string + dataDir string + + update func(ctx context.Context) ([]byte, error) + install func(ctx context.Context, typ, name string) ([]byte, error) + inspect func(ctx context.Context, typ, name string) ([]byte, error) + deps map[string][]crowdsec.HubItem // links created together with an install + noLinkFor map[string]bool +} + +func (f *hubFakeExec) Execute(ctx context.Context, name string, args ...string) ([]byte, error) { + f.mu.Lock() + f.calls = append(f.calls, strings.Join(append([]string{name}, args...), " ")) + f.mu.Unlock() + switch { + case name != "cscli": + return nil, fmt.Errorf("unexpected binary %q", name) + case len(args) == 2 && args[0] == "hub" && args[1] == "update": + if f.update != nil { + return f.update(ctx) + } + return []byte("ok"), nil + case len(args) == 3 && args[1] == "install": + if f.install != nil { + if out, err := f.install(ctx, args[0], args[2]); err != nil { + return out, err + } + } + it := crowdsec.HubItem{Type: args[0], Name: args[2]} + if !f.noLinkFor[it.Name] { + linkHubItem(f.dataDir, it) + } + for _, dep := range f.deps[it.Name] { + linkHubItem(f.dataDir, dep) + } + return []byte("ok"), nil + case len(args) == 5 && args[1] == "inspect": + if f.inspect != nil { + return f.inspect(ctx, args[0], args[2]) + } + return []byte(`{"installed":true,"tainted":false}`), nil + } + return nil, fmt.Errorf("unexpected call %v", args) +} + +func (f *hubFakeExec) callLog() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.calls...) +} + +func (f *hubFakeExec) installCalls() []string { + var out []string + for _, c := range f.callLog() { + if strings.Contains(c, " install ") { + out = append(out, c) + } + } + return out +} + +// linkHubItem creates the symlink an installed item is made of. +func linkHubItem(dataDir string, it crowdsec.HubItem) { + owner, base, _ := strings.Cut(it.Name, "/") + stage := "" + if it.Type == "parsers" || it.Type == "postoverflows" { + stage = "s01-parse" + } + dir := filepath.Join(dataDir, "config", it.Type, stage) + if err := os.MkdirAll(dir, 0o750); err != nil { + panic(err) + } + target := path.Join("/etc/crowdsec/hub", it.Type, stage, owner, base+".yaml") + if err := os.Symlink(target, filepath.Join(dir, owner+"-"+base+".yaml")); err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } +} + +type hubHarness struct { + h *CrowdsecHandler + r *gin.Engine + exec *hubFakeExec + dir string + signal atomic.Int32 +} + +func newHubHarness(t *testing.T) *hubHarness { + t.Helper() + dir := t.TempDir() + h := newTestCrowdsecHandler(t, OpenTestDB(t), &fakeExec{started: true}, "/bin/false", dir) + ex := &hubFakeExec{dataDir: dir} + h.CmdExec = ex + hh := &hubHarness{h: h, exec: ex, dir: dir} + h.signalProcess = func(int, syscall.Signal) error { + hh.signal.Add(1) + return nil + } + hh.r = gin.New() + h.RegisterRoutes(hh.r.Group("/api/v1")) + return hh +} + +func hubItems(pairs ...string) []crowdsec.HubItem { + var out []crowdsec.HubItem + for i := 0; i+1 < len(pairs); i += 2 { + out = append(out, crowdsec.HubItem{Type: pairs[i], Name: pairs[i+1]}) + } + return out +} + +func restoreBody(items ...crowdsec.HubItem) []byte { + b, _ := json.Marshal(map[string]any{"items": items}) + return b +} + +func (hh *hubHarness) post(body []byte) *httptest.ResponseRecorder { + return hh.postCtx(context.Background(), body) +} + +func (hh *hubHarness) postCtx(ctx context.Context, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, restorePath, bytes.NewReader(body)).WithContext(ctx) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + hh.r.ServeHTTP(w, req) + return w +} + +func decodeRestore(t *testing.T, w *httptest.ResponseRecorder) hubRestoreResponse { + t.Helper() + var out hubRestoreResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out), w.Body.String()) + return out +} + +func restoreErr(t *testing.T, w *httptest.ResponseRecorder) string { + t.Helper() + var out map[string]string + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out), w.Body.String()) + return out["error"] +} + +func TestRestoreHubItemsRejectsInvalidInput(t *testing.T) { + t.Parallel() + valid := crowdsec.HubItem{Type: "collections", Name: "crowdsecurity/sshd"} + long := func(n int) string { return "a/" + strings.Repeat("b", n) } + bad := map[string]crowdsec.HubItem{ + "flag": {Type: "collections", Name: "--help"}, + "flag in name": {Type: "collections", Name: "-x/y"}, + "traversal": {Type: "collections", Name: "../x"}, + "dotdot middle": {Type: "collections", Name: "a/../b"}, + "metachar": {Type: "collections", Name: "a;b/c"}, + "trailing newline": {Type: "collections", Name: "a/b\n"}, + "space": {Type: "collections", Name: "a/b c"}, + "empty owner": {Type: "collections", Name: "/b"}, + "empty name": {Type: "collections", Name: "a/"}, + "empty": {Type: "collections", Name: ""}, + "no slash": {Type: "collections", Name: "sshd"}, + "two slashes": {Type: "collections", Name: "a/b/c"}, + "unicode": {Type: "collections", Name: "a/bé"}, + "over-long name": {Type: "collections", Name: long(129)}, + "wrong type": {Type: "bouncers", Name: "a/b"}, + "empty type": {Type: "", Name: "a/b"}, + "flag-like type": {Type: "--help", Name: "a/b"}, + "type with space": {Type: "collections install", Name: "a/b"}, + "upper-case type": {Type: "Collections", Name: "a/b"}, + "command separator": {Type: "collections", Name: "a/b&&c"}, + } + for name, item := range bad { + t.Run(name, func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + w := hh.post(restoreBody(valid, item)) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Equal(t, "invalid hub item", restoreErr(t, w)) + require.Empty(t, hh.exec.callLog(), "a rejected request runs nothing, valid items included") + }) + } + + t.Run("more than 1000 items", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + items := make([]crowdsec.HubItem, 0, crowdsec.MaxHubItems+1) + for i := 0; i <= crowdsec.MaxHubItems; i++ { + items = append(items, crowdsec.HubItem{Type: "scenarios", Name: fmt.Sprintf("o/n%d", i)}) + } + w := hh.post(restoreBody(items...)) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Equal(t, "invalid hub item", restoreErr(t, w)) + require.Empty(t, hh.exec.callLog()) + }) + + t.Run("empty list", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + w := hh.post([]byte(`{"items":[]}`)) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Equal(t, "no hub items given", restoreErr(t, w)) + require.Empty(t, hh.exec.callLog()) + }) + + t.Run("malformed JSON", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + w := hh.post([]byte(`{"items":`)) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Equal(t, "invalid request", restoreErr(t, w)) + require.Empty(t, hh.exec.callLog()) + }) + + t.Run("oversized body", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + body := append([]byte(`{"items":[],"pad":"`), bytes.Repeat([]byte("a"), 2<<20)...) + body = append(body, []byte(`"}`)...) + w := hh.post(body) + require.Equal(t, http.StatusRequestEntityTooLarge, w.Code) + require.Equal(t, "request too large", restoreErr(t, w)) + require.Empty(t, hh.exec.callLog()) + }) +} + +func TestRestoreHubItemsInstallsAndVerifies(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + items := hubItems("collections", "Azlaroc/sftpgo", "parsers", "crowdsecurity/CVE-2022-26134", "scenarios", "Dominic-Wagner/vaultwarden") + + // Duplicates collapse to one install. + w := hh.post(restoreBody(append(items, items[0])...)) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + res := decodeRestore(t, w) + require.Equal(t, items, res.Installed) + require.Empty(t, res.Failed) + require.Empty(t, res.AlreadyInstalled) + require.Zero(t, res.Remaining) + require.Zero(t, res.SkippedInvalid) + require.True(t, res.Reloaded) + require.EqualValues(t, 1, hh.signal.Load()) + + require.Equal(t, []string{ + "cscli hub update", + "cscli collections install Azlaroc/sftpgo", + "cscli collections inspect Azlaroc/sftpgo -o json", + "cscli parsers install crowdsecurity/CVE-2022-26134", + "cscli parsers inspect crowdsecurity/CVE-2022-26134 -o json", + "cscli scenarios install Dominic-Wagner/vaultwarden", + "cscli scenarios inspect Dominic-Wagner/vaultwarden -o json", + }, hh.exec.callLog()) + + // Arrays are present (not null) even when empty. + var raw map[string]json.RawMessage + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &raw)) + require.JSONEq(t, `[]`, string(raw["failed"])) + require.JSONEq(t, `[]`, string(raw["already_installed"])) +} + +func TestRestoreHubItemsSkipsInstalled(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + have := crowdsec.HubItem{Type: "scenarios", Name: "crowdsecurity/ssh-bf"} + linkHubItem(hh.dir, have) + + w := hh.post(restoreBody(have, crowdsec.HubItem{Type: "scenarios", Name: "crowdsecurity/other"})) + require.Equal(t, http.StatusOK, w.Code) + res := decodeRestore(t, w) + require.Equal(t, []crowdsec.HubItem{have}, res.AlreadyInstalled) + require.Equal(t, hubItems("scenarios", "crowdsecurity/other"), res.Installed) + require.Equal(t, []string{"cscli scenarios install crowdsecurity/other"}, hh.exec.installCalls()) +} + +func TestRestoreHubItemsCollectionsFirstThenReDiff(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + dep := crowdsec.HubItem{Type: "scenarios", Name: "crowdsecurity/dep"} + hh.exec.deps = map[string][]crowdsec.HubItem{"crowdsecurity/coll": {dep}} + + // The scenario is listed first but only diffed after the collection pass pulled it in. + other := crowdsec.HubItem{Type: "parsers", Name: "crowdsecurity/solo"} + coll := crowdsec.HubItem{Type: "collections", Name: "crowdsecurity/coll"} + w := hh.post(restoreBody(dep, other, coll)) + require.Equal(t, http.StatusOK, w.Code) + res := decodeRestore(t, w) + require.Equal(t, []crowdsec.HubItem{coll, other}, res.Installed) + require.Equal(t, []crowdsec.HubItem{dep}, res.AlreadyInstalled) + require.Equal(t, []string{ + "cscli collections install crowdsecurity/coll", + "cscli parsers install crowdsecurity/solo", + }, hh.exec.installCalls()) +} + +func TestRestoreHubItemsReasons(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.exec.noLinkFor = map[string]bool{"o/linkless": true} + hh.exec.install = func(_ context.Context, _, name string) ([]byte, error) { + switch name { + case "o/missing": + return []byte("Error: can't find 'o/missing' in scenarios at /secret/path"), errors.New("exit status 1") + case "o/broken": + return []byte("Error: download failed /secret/path token=abc"), errors.New("exit status 1") + } + return nil, nil + } + hh.exec.inspect = func(_ context.Context, _, name string) ([]byte, error) { + switch name { + case "o/tainted": + return []byte(`{"installed":true,"tainted":true}`), nil + case "o/notinstalled": + return []byte(`{"installed":false}`), nil + case "o/garbage": + return []byte(`not json /secret/path`), nil + } + return []byte(`{"installed":true,"tainted":false}`), nil + } + + w := hh.post(restoreBody(hubItems("scenarios", "o/missing", "scenarios", "o/broken", "scenarios", "o/tainted", + "scenarios", "o/notinstalled", "scenarios", "o/garbage", "scenarios", "o/fine")...)) + require.Equal(t, http.StatusOK, w.Code) + require.NotContains(t, w.Body.String(), "secret") + require.NotContains(t, w.Body.String(), "token") + res := decodeRestore(t, w) + require.Equal(t, hubItems("scenarios", "o/fine"), res.Installed) + require.Equal(t, []hubRestoreFailure{ + {"scenarios", "o/missing", "not_found"}, + {"scenarios", "o/broken", "install_failed"}, + {"scenarios", "o/tainted", "verify_failed"}, + {"scenarios", "o/notinstalled", "verify_failed"}, + {"scenarios", "o/garbage", "verify_failed"}, + }, res.Failed) + require.Zero(t, res.Remaining, "permanent failures are not retryable") +} + +func TestRestoreHubItemsHubUpdateFailure(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.exec.update = func(context.Context) ([]byte, error) { + return []byte("Error: Get https://hub.example/secret: no route"), errors.New("exit status 1") + } + w := hh.post(restoreBody(hubItems("collections", "a/b")...)) + require.Equal(t, http.StatusBadGateway, w.Code) + require.Equal(t, "the CrowdSec hub could not be reached; nothing was installed", restoreErr(t, w)) + require.NotContains(t, w.Body.String(), "secret") + require.Equal(t, []string{"cscli hub update"}, hh.exec.callLog()) + require.Zero(t, hh.signal.Load()) + require.False(t, hh.h.restore.running.Load(), "the running flag is cleared after a failure") +} + +func TestRestoreHubItemsDeadlineLeavesNotAttempted(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.h.restore.limits.overall = 100 * time.Millisecond + hh.exec.update = func(context.Context) ([]byte, error) { + time.Sleep(200 * time.Millisecond) // the budget is gone before the first item + return []byte("ok"), nil + } + items := hubItems("collections", "a/one", "scenarios", "a/two", "parsers", "a/three") + w := hh.post(restoreBody(items...)) + require.Equal(t, http.StatusOK, w.Code) + res := decodeRestore(t, w) + require.Empty(t, res.Installed) + require.Equal(t, 3, res.Remaining) + require.Len(t, res.Failed, 3) + for _, f := range res.Failed { + require.Equal(t, "not_attempted", f.Reason) + } + require.Empty(t, hh.exec.installCalls()) + require.False(t, res.Reloaded) +} + +func TestRestoreHubItemsClientCancelBetweenItems(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // The client leaves while the first item is being verified. + hh.exec.inspect = func(context.Context, string, string) ([]byte, error) { + cancel() + return []byte(`{"installed":true,"tainted":false}`), nil + } + w := hh.postCtx(ctx, restoreBody(hubItems("scenarios", "a/one", "scenarios", "a/two", "scenarios", "a/three")...)) + require.Equal(t, http.StatusOK, w.Code) + res := decodeRestore(t, w) + require.Equal(t, hubItems("scenarios", "a/one"), res.Installed) + require.Equal(t, []hubRestoreFailure{{"scenarios", "a/two", "not_attempted"}, {"scenarios", "a/three", "not_attempted"}}, res.Failed) + require.Equal(t, 2, res.Remaining) + require.Len(t, hh.exec.installCalls(), 1) + // The install that did succeed is still reloaded although the request context is cancelled. + require.True(t, res.Reloaded) + require.EqualValues(t, 1, hh.signal.Load()) +} + +func TestRestoreHubItemsReloadOutcomesAreIgnored(t *testing.T) { + t.Parallel() + items := restoreBody(hubItems("scenarios", "a/one")...) + + t.Run("not running", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.h.Executor = &fakeExec{} + w := hh.post(items) + require.Equal(t, http.StatusOK, w.Code) + res := decodeRestore(t, w) + require.Len(t, res.Installed, 1) + require.False(t, res.Reloaded) + }) + t.Run("status error", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.h.Executor = &fakeExecWithError{statusError: errors.New("boom")} + w := hh.post(items) + require.Equal(t, http.StatusOK, w.Code) + res := decodeRestore(t, w) + require.Len(t, res.Installed, 1) + require.False(t, res.Reloaded) + }) + t.Run("signal error", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.h.signalProcess = func(int, syscall.Signal) error { return errors.New("denied") } + w := hh.post(items) + require.Equal(t, http.StatusOK, w.Code) + require.False(t, decodeRestore(t, w).Reloaded) + }) + t.Run("nothing installed means no reload", func(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + linkHubItem(hh.dir, crowdsec.HubItem{Type: "scenarios", Name: "a/one"}) + w := hh.post(items) + require.Equal(t, http.StatusOK, w.Code) + require.False(t, decodeRestore(t, w).Reloaded) + require.Zero(t, hh.signal.Load()) + }) +} + +// blockingInstall makes install calls wait for a release, reporting each start. +type blockingInstall struct { + started chan string + release chan struct{} +} + +func newBlockingInstall() *blockingInstall { + return &blockingInstall{started: make(chan string, 16), release: make(chan struct{}, 16)} +} + +func (b *blockingInstall) fn(ctx context.Context, _, name string) ([]byte, error) { + b.started <- name + select { + case <-b.release: + time.Sleep(20 * time.Millisecond) // the install keeps working a moment, so queued writers get scheduled + return nil, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func awaitInstall(t *testing.T, ch <-chan string) string { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the install to start") + return "" + } +} + +func TestRestoreHubItemsConcurrentCallIsRefused(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + b := newBlockingInstall() + hh.exec.install = b.fn + + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- hh.post(restoreBody(hubItems("scenarios", "a/one")...)) }() + awaitInstall(t, b.started) + + w := hh.post(restoreBody(hubItems("scenarios", "a/two")...)) + require.Equal(t, http.StatusConflict, w.Code) + require.Equal(t, "a restore is already running", restoreErr(t, w)) + + b.release <- struct{}{} + require.Equal(t, http.StatusOK, (<-done).Code) + require.False(t, hh.h.restore.running.Load()) + + // The flag is cleared: the next call is served. + b.release <- struct{}{} + require.Equal(t, http.StatusOK, hh.post(restoreBody(hubItems("scenarios", "a/two")...)).Code) +} + +func TestRestoreHubItemsHungInstallFreesTheLockWithinTheItemTimeout(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + hh.h.restore.limits.item = 150 * time.Millisecond + b := newBlockingInstall() + hh.exec.install = func(ctx context.Context, typ, name string) ([]byte, error) { + if name == "a/hung" { // never released: only its context ends the call + return b.fn(ctx, typ, name) + } + return nil, nil + } + + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- hh.post(restoreBody(hubItems("scenarios", "a/hung", "scenarios", "a/next")...)) }() + awaitInstall(t, b.started) + + acquired := make(chan struct{}) + start := time.Now() + go func() { + hh.h.dataMu.Lock() + close(acquired) + hh.h.dataMu.Unlock() + }() + select { + case <-acquired: + require.Less(t, time.Since(start), 3*time.Second) + case <-time.After(5 * time.Second): + t.Fatal("dataMu was not released within the per-item timeout") + } + + w := <-done + res := decodeRestore(t, w) + require.Equal(t, []hubRestoreFailure{{"scenarios", "a/hung", "install_failed"}}, res.Failed) + require.Equal(t, hubItems("scenarios", "a/next"), res.Installed) +} + +// TestRestoreHubItemsExcludesImport proves an import started during an install waits for that one +// item, never overlaps it, and cannot remove the link the install created. +func TestRestoreHubItemsExcludesImport(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + seedRealLayout(t, hh.dir) + b := newBlockingInstall() + var seq, installDone, importDone atomic.Int64 + hh.exec.install = func(ctx context.Context, typ, name string) ([]byte, error) { + out, err := b.fn(ctx, typ, name) + installDone.Store(seq.Add(1)) // before the link is written and the lock released + return out, err + } + item := crowdsec.HubItem{Type: "collections", Name: "crowdsecurity/coll"} + + restoreDone := make(chan *httptest.ResponseRecorder, 1) + go func() { restoreDone <- hh.post(restoreBody(item)) }() + awaitInstall(t, b.started) + + importResult := make(chan int, 1) + go func() { + w := httptest.NewRecorder() + hh.r.ServeHTTP(w, importRequest(t, map[string]string{"config/config.yaml": importableConfig})) + importDone.Store(seq.Add(1)) + importResult <- w.Code + }() + + select { + case <-importResult: + t.Fatal("the import ran while an install held the data lock") + case <-time.After(300 * time.Millisecond): + } + + b.release <- struct{}{} + require.Equal(t, http.StatusOK, (<-restoreDone).Code) + require.Equal(t, http.StatusOK, <-importResult) + require.Less(t, installDone.Load(), importDone.Load(), "the import completed after the install") + + installed, err := crowdsec.InstalledHubItems(hh.dir) + require.NoError(t, err) + require.Contains(t, installed, item, "the link the install created survived the import") +} + +// TestRestoreHubItemsReleasesTheLockBetweenItems proves another writer gets in after every item. +func TestRestoreHubItemsReleasesTheLockBetweenItems(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + b := newBlockingInstall() + hh.exec.install = b.fn + + items := hubItems("scenarios", "a/one", "scenarios", "a/two", "scenarios", "a/three") + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- hh.post(restoreBody(items...)) }() + + awaitInstall(t, b.started) + var startedAtAcquire atomic.Int64 + acquired := make(chan struct{}) + go func() { // another writer queues behind the first item + hh.h.dataMu.Lock() + startedAtAcquire.Store(int64(len(hh.exec.installCalls()))) + close(acquired) + hh.h.dataMu.Unlock() + }() + time.Sleep(50 * time.Millisecond) + b.release <- struct{}{} + + // It gets in no later than the end of the next item, long before the action is over. + deadline := time.After(5 * time.Second) + for got := false; !got; { + select { + case <-acquired: + got = true + case <-b.started: + b.release <- struct{}{} + case <-deadline: + t.Fatal("another writer was starved by the restore") + } + } + require.Less(t, int(startedAtAcquire.Load()), len(items), "the writer only got in after the last item started") + + // Let the remaining items finish. + var w *httptest.ResponseRecorder + for w == nil { + select { + case <-b.started: + b.release <- struct{}{} + case w = <-done: + case <-time.After(5 * time.Second): + t.Fatal("restore stalled") + } + } + res := decodeRestore(t, w) + require.Equal(t, items, res.Installed) +} + +// TestRestoreHubItemsSerializesWithPresetApply proves ApplyCurated cannot overlap an install. +func TestRestoreHubItemsSerializesWithPresetApply(t *testing.T) { + c := newCuratedHarness(t, nil) + ex := &hubFakeExec{dataDir: t.TempDir()} + b := newBlockingInstall() + ex.install = b.fn + c.handler.CmdExec = ex + c.handler.DataDir = ex.dataDir + + restoreDone := make(chan *httptest.ResponseRecorder, 1) + go func() { + req := httptest.NewRequest(http.MethodPost, restorePath, bytes.NewReader(restoreBody(hubItems("scenarios", "a/one")...))) + w := httptest.NewRecorder() + c.router.ServeHTTP(w, req) + restoreDone <- w + }() + awaitInstall(t, b.started) + + applyDone := make(chan int, 1) + go func() { + code, _ := c.apply(t, "honeypot-friendly-defaults") + applyDone <- code + }() + select { + case <-applyDone: + t.Fatal("the preset apply ran during an install") + case <-time.After(300 * time.Millisecond): + } + c.exec.mu.Lock() + require.Empty(t, c.exec.calls, "no hub call of the preset apply while the install runs") + c.exec.mu.Unlock() + + b.release <- struct{}{} + require.Equal(t, http.StatusOK, (<-restoreDone).Code) + require.Equal(t, http.StatusOK, <-applyDone) +} + +func TestRestoreHubItemsMaximumRequest(t *testing.T) { + t.Parallel() + hh := newHubHarness(t) + owner := "o" + strings.Repeat("w", 63) + items := make([]crowdsec.HubItem, 0, crowdsec.MaxHubItems) + for i := 0; i < crowdsec.MaxHubItems; i++ { + name := fmt.Sprintf("n%04d", i) + items = append(items, crowdsec.HubItem{Type: "postoverflows", Name: owner + "/" + name + strings.Repeat("x", 128-len(name))}) + } + body := restoreBody(items...) + require.Less(t, len(body), 1<<20) + + w := hh.post(body) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + res := decodeRestore(t, w) + require.Len(t, res.Installed, crowdsec.MaxHubItems) + require.Empty(t, res.Failed) +} + +func TestHubItemNotFound(t *testing.T) { + t.Parallel() + require.True(t, hubItemNotFound([]byte("Error: can't find 'a/b' in collections"))) + require.True(t, hubItemNotFound([]byte("item NOT FOUND"))) + require.False(t, hubItemNotFound([]byte("download failed"))) + require.False(t, hubItemNotFound(nil)) +} + +func TestHubRestoreLimitsDefaults(t *testing.T) { + t.Parallel() + l := hubRestoreLimits{}.withDefaults() + require.Equal(t, hubRestoreOverall, l.overall) + require.Equal(t, hubRestoreItemTimeout, l.item) + require.Equal(t, hubRestoreUpdateTimeout, l.update) + require.Equal(t, time.Second, hubRestoreLimits{overall: time.Second}.withDefaults().overall) +} + +func TestHubLogDetailTruncatesAndSanitizes(t *testing.T) { + t.Parallel() + got := hubLogDetail("a\nb" + strings.Repeat("x", 1000)) + require.Len(t, got, hubRestoreLogLimit) + require.NotContains(t, got, "\n") +} diff --git a/backend/internal/api/handlers/crowdsec_import_notifications_test.go b/backend/internal/api/handlers/crowdsec_import_notifications_test.go new file mode 100644 index 000000000..83c060380 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_import_notifications_test.go @@ -0,0 +1,133 @@ +package handlers + +import ( + "encoding/json" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + importProfilesNaming = "name: p\nfilters:\n - Alert.Remediation == true\ndecisions:\n - type: ban\n duration: 4h\nnotifications:\n - %s\n" + importNotification = "type: http\nname: %s\nurl: https://example.invalid/\nmethod: POST\n" +) + +// treeState captures every entry below root (kind, mode, content, link target) so a test can assert +// that a refused import left the tree byte-identical. +func treeState(t *testing.T, root string) map[string]string { + t.Helper() + state := map[string]string{} + require.NoError(t, filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + require.NoError(t, err) + rel, relErr := filepath.Rel(root, p) + require.NoError(t, relErr) + info, infoErr := d.Info() + require.NoError(t, infoErr) + entry := info.Mode().String() + switch { + case info.Mode()&os.ModeSymlink != 0: + target, linkErr := os.Readlink(p) + require.NoError(t, linkErr) + entry += " -> " + target + case info.Mode().IsRegular(): + entry += " " + readTreeFile(t, p) + } + state[rel] = entry + return nil + })) + return state +} + +func importNotificationBody(t *testing.T, w *httptest.ResponseRecorder) string { + t.Helper() + var body map[string]string + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + return body["error"] +} + +func TestImportConfigRefusesUndefinedNotificationsBeforeAnyChange(t *testing.T) { + t.Parallel() + parent := t.TempDir() + dir := filepath.Join(parent, "crowdsec") + require.NoError(t, os.MkdirAll(dir, 0o750)) + seedRealLayout(t, dir) + writeTree(t, dir, map[string]string{ + "config/profiles.yaml": "live profiles\n", + "config/notifications/l.yaml": "live notification\n", + "config/notifications/l2.yaml": "live notification 2\n", + }) + h, r := newImportRouter(t, dir) + before := treeState(t, dir) + parentBefore := treeState(t, parent) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{ + "config/config.yaml": importableConfig, + "config/profiles.yaml": strings.Replace(importProfilesNaming, "%s", "x", 1), + })) + + require.Equal(t, http.StatusUnprocessableEntity, w.Code, w.Body.String()) + require.Equal(t, "config validation failed: profiles.yaml names notifications that have no matching file: x", importNotificationBody(t, w)) + require.Equal(t, before, treeState(t, dir), "the live tree must be byte-identical") + require.Equal(t, parentBefore, treeState(t, parent), "no snapshot may be created") + require.True(t, h.dataMu.TryLock(), "the data lock must not be held") + h.dataMu.Unlock() +} + +func TestImportConfigNotificationReferences(t *testing.T) { + t.Parallel() + profiles := strings.Replace(importProfilesNaming, "%s", "x", 1) + cases := []struct { + name string + files map[string]string + code int + }{ + {"satisfied by a notification file", map[string]string{"config/config.yaml": importableConfig, "config/profiles.yaml": profiles, "config/notifications/x.yaml": strings.Replace(importNotification, "%s", "x", 1)}, http.StatusOK}, + {"satisfied by name in a differently named file", map[string]string{"config/config.yaml": importableConfig, "config/profiles.yaml": profiles, "config/notifications/other.yml": strings.Replace(importNotification, "%s", "x", 1)}, http.StatusOK}, + {"defined only by file name", map[string]string{"config/config.yaml": importableConfig, "config/profiles.yaml": profiles, "config/notifications/x.yaml": strings.Replace(importNotification, "%s", "y", 1)}, http.StatusUnprocessableEntity}, + {"legacy archive without profiles", map[string]string{"config/config.yaml": importableConfig}, http.StatusOK}, + {"root layout refused", map[string]string{"config.yaml": importableConfig, "profiles.yaml": profiles}, http.StatusUnprocessableEntity}, + {"redirected paths pass", map[string]string{"config/config.yaml": "api:\n server:\n profiles_path: /elsewhere/profiles.yaml\n", "config/profiles.yaml": profiles}, http.StatusOK}, + {"unparsable profiles pass the check", map[string]string{"config/config.yaml": importableConfig, "config/profiles.yaml": "name: [unclosed\n"}, http.StatusOK}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + _, r := newImportRouter(t, dir) + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, tc.files)) + require.Equal(t, tc.code, w.Code, w.Body.String()) + if tc.code == http.StatusOK { + for rel, content := range tc.files { + require.Equal(t, content, readTreeFile(t, filepath.Join(dir, rel))) + } + } + }) + } +} + +func TestImportConfigNotificationMessageIsBoundedAndSanitized(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedRealLayout(t, dir) + _, r := newImportRouter(t, dir) + var names []string + for _, n := range []string{"n01", "n02", "n03", "n04", "n05", "n06", "n07", "n08", "n09", "n10", "n11", "n12"} { + names = append(names, "- "+n) + } + profiles := "name: p\nnotifications:\n " + strings.Join(names, "\n ") + "\n" + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config/config.yaml": importableConfig, "config/profiles.yaml": profiles})) + require.Equal(t, http.StatusUnprocessableEntity, w.Code, w.Body.String()) + msg := importNotificationBody(t, w) + require.Equal(t, "config validation failed: profiles.yaml names notifications that have no matching file: n01, n02, n03, n04, n05, n06, n07, n08, n09, n10, and 2 more", msg) + require.NotContains(t, msg, dir) +} diff --git a/backend/internal/api/routes/crowdsec_hub_items_auth_test.go b/backend/internal/api/routes/crowdsec_hub_items_auth_test.go new file mode 100644 index 000000000..8109b1f0f --- /dev/null +++ b/backend/internal/api/routes/crowdsec_hub_items_auth_test.go @@ -0,0 +1,40 @@ +package routes + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/Wikid82/charon/backend/internal/config" +) + +// The hub item restore sits in the same admin group as the other CrowdSec routes, so an +// unauthenticated call is refused before any handler logic runs. +func TestRegister_CrowdsecHubItemsRestoreRequiresAuthentication(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + + db, err := gorm.Open(sqlite.Open(isolatedMemoryDSN(t)), &gorm.Config{}) + require.NoError(t, err) + + cfg := config.Config{ + JWTSecret: "test-secret", + EncryptionKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + } + require.NoError(t, Register(context.Background(), router, db, cfg)) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/hub-items/restore", + strings.NewReader(`{"items":[{"type":"collections","name":"crowdsecurity/sshd"}]}`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(w, req) + assert.Equal(t, http.StatusUnauthorized, w.Code) +} diff --git a/backend/internal/crowdsec/archive_inspect.go b/backend/internal/crowdsec/archive_inspect.go new file mode 100644 index 000000000..2ac732549 --- /dev/null +++ b/backend/internal/crowdsec/archive_inspect.go @@ -0,0 +1,172 @@ +package crowdsec + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "github.com/Wikid82/charon/backend/internal/logger" +) + +// ManifestEntryName is the archive entry reserved for the hub item manifest. Exports generate it, an +// import reads it through InspectArchive and never writes it to disk. +const ManifestEntryName = "charon-hub-items.json" + +// Bounds on what InspectArchive keeps in memory. The archive is untrusted, and the import validator +// stays the authority on overall size; an entry over a bound only makes the inspection incomplete. +const ( + maxInspectEntryBytes int64 = 1 << 20 + maxInspectTotalBytes int64 = 8 << 20 + maxInspectEntries = 256 +) + +// ArchiveInfo holds the few entries of an uploaded archive that the pre-import checks read. +type ArchiveInfo struct { + // Files maps the normalized, slash-separated entry name to its content. + Files map[string][]byte + // Incomplete is set when a wanted entry exceeded a bound, so Files is not the whole picture. + Incomplete bool + // HubItems is the usable hub item manifest, nil when the archive has none or it was rejected. + HubItems *ArchiveHubItems +} + +// ArchiveEntryRel normalizes a tar entry name exactly as the extractor places it below the destination +// directory ("./a", "a/../b" and "b" are the same entry). It reports false for a name that would leave +// the destination or names the destination itself. +func ArchiveEntryRel(name string) (string, bool) { + root := string(os.PathSeparator) + "root" + target := filepath.Join(root, name) // #nosec G305 -- containment is checked on the next line + prefix := root + string(os.PathSeparator) + if !strings.HasPrefix(target, prefix) { + return "", false + } + return strings.TrimPrefix(target, prefix), true +} + +// inspectWanted reports whether rel is an entry the notification check needs, in either layout: the +// config file, the profiles file, or a direct child of notifications/ that CrowdSec loads (lowercase +// .yaml or .yml only, no subdirectories). +func inspectWanted(rel string) bool { + for _, prefix := range []string{"", "config/"} { + switch rel { + case prefix + "config.yaml", prefix + "profiles.yaml": + return true + } + if path.Dir(rel) == prefix+"notifications" { + ext := path.Ext(rel) + return (ext == ".yaml" || ext == ".yml") && path.Base(rel) != ext + } + } + return false +} + +// InspectArchive reads the entries the pre-import checks need from a tar.gz in one pass. Bodies of +// every other entry are skipped unread, and nothing is written to disk. +func InspectArchive(archivePath string) (ArchiveInfo, error) { + info := ArchiveInfo{Files: map[string][]byte{}} + f, err := os.Open(filepath.Clean(archivePath)) + if err != nil { + var pathErr *fs.PathError + if errors.As(err, &pathErr) { + err = pathErr.Err // the path stays out of the message + } + return info, fmt.Errorf("open archive: %w", err) + } + defer func() { _ = f.Close() }() + gr, err := gzip.NewReader(f) + if err != nil { + return info, fmt.Errorf("create gzip reader: %w", err) + } + defer func() { _ = gr.Close() }() + + tr := tar.NewReader(gr) + var total int64 + manifest := manifestSlot{} + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + info.HubItems = manifest.parse() + return info, nil + } + if err != nil { + return info, fmt.Errorf("read tar header: %w", err) + } + if hdr.Typeflag != tar.TypeReg { + continue + } + rel, ok := ArchiveEntryRel(hdr.Name) + if ok && IsManifestPath(rel) { + if readErr := manifest.read(tr, hdr.Size); readErr != nil { + return info, readErr + } + continue + } + if !ok || !inspectWanted(filepath.ToSlash(rel)) { + continue + } + rel = filepath.ToSlash(rel) + if hdr.Size > maxInspectEntryBytes || total+hdr.Size > maxInspectTotalBytes || (len(info.Files) >= maxInspectEntries && info.Files[rel] == nil) { + info.Incomplete = true + continue + } + data, err := io.ReadAll(io.LimitReader(tr, maxInspectEntryBytes+1)) + if err != nil { + return info, fmt.Errorf("read archive entry: %w", err) + } + if int64(len(data)) > maxInspectEntryBytes { + info.Incomplete = true + continue + } + total += int64(len(data)) + info.Files[rel] = data + } +} + +// manifestSlot collects the single manifest entry of an archive. +type manifestSlot struct { + entries int + data []byte + tooBig bool +} + +// read takes the body of a regular manifest entry; its size is bounded by maxManifestBytes. +func (m *manifestSlot) read(r io.Reader, size int64) error { + m.entries++ + if m.entries > 1 || size > maxManifestBytes { + m.tooBig = m.tooBig || size > maxManifestBytes + return nil + } + data, err := io.ReadAll(io.LimitReader(r, maxManifestBytes+1)) + if err != nil { + return fmt.Errorf("read archive entry: %w", err) + } + m.data = data + return nil +} + +// parse returns the usable manifest, or nil when there is none: a duplicate entry, an oversized body, +// malformed JSON and an unknown version all ignore the whole manifest. +func (m *manifestSlot) parse() *ArchiveHubItems { + switch { + case m.entries == 0: + return nil + case m.entries > 1: + logger.Log().Warn("hub item manifest ignored: the archive holds it more than once") + return nil + case m.tooBig || int64(len(m.data)) > maxManifestBytes: + logger.Log().Warn("hub item manifest ignored: it is too large") + return nil + } + items := parseHubManifest(m.data) + if items == nil { + logger.Log().Warn("hub item manifest ignored: it is not valid version 1 content") + } + return items +} diff --git a/backend/internal/crowdsec/curated_apply.go b/backend/internal/crowdsec/curated_apply.go index 48607827c..5c5496557 100644 --- a/backend/internal/crowdsec/curated_apply.go +++ b/backend/internal/crowdsec/curated_apply.go @@ -2,7 +2,6 @@ package crowdsec import ( "context" - "encoding/json" "errors" "fmt" @@ -75,42 +74,13 @@ func (s *HubService) installAndVerify(ctx context.Context, preset Preset) error if _, err := s.Exec.Execute(ctx, "cscli", item.Type, "install", item.Name); err != nil { return fmt.Errorf("install %s %s: %w", item.Type, item.Name, err) } - if err := s.verifyItem(ctx, item); err != nil { + if err := VerifyHubItem(ctx, s.Exec, item.Type, item.Name); err != nil { return fmt.Errorf("verify %s %s: %w", item.Type, item.Name, err) } } return nil } -// verifyItem requires `cscli inspect -o json` to report the item as installed and untainted. -func (s *HubService) verifyItem(ctx context.Context, item PresetItem) error { - if err := ctx.Err(); err != nil { - return err - } - out, err := s.Exec.Execute(ctx, "cscli", item.Type, "inspect", item.Name, "-o", "json") - if err != nil { - return err - } - // Field names follow cscli's `-o json` inspect output (installed, tainted). - var state struct { - Installed *bool `json:"installed"` - Tainted bool `json:"tainted"` - } - if err := json.Unmarshal(out, &state); err != nil { - return fmt.Errorf("parse inspect output: %w", err) - } - if state.Installed == nil { - return errors.New("inspect output has no installed field") - } - if !*state.Installed { - return errors.New("item reported as not installed") - } - if state.Tainted { - return errors.New("item reported as tainted") - } - return nil -} - // reloadCrowdSec asks the managed CrowdSec process to reload (best-effort) and reports // whether the reload was performed. A missing reloader or a stopped process is not an error: // CrowdSec loads the installed items itself on its next start. diff --git a/backend/internal/crowdsec/curated_apply_test.go b/backend/internal/crowdsec/curated_apply_test.go index 22656d690..594914122 100644 --- a/backend/internal/crowdsec/curated_apply_test.go +++ b/backend/internal/crowdsec/curated_apply_test.go @@ -511,7 +511,7 @@ func TestVerifyItemNilContextError(t *testing.T) { hub, _ := newCuratedHub(t, &curatedExec{}) ctx, cancel := context.WithCancel(context.Background()) cancel() - require.ErrorIs(t, hub.verifyItem(ctx, PresetItem{Type: "collections", Name: "crowdsecurity/sshd"}), context.Canceled) + require.ErrorIs(t, VerifyHubItem(ctx, hub.Exec, "collections", "crowdsecurity/sshd"), context.Canceled) } func TestApplyCuratedRollbackLeavesLiveDatabaseUntouched(t *testing.T) { diff --git a/backend/internal/crowdsec/export.go b/backend/internal/crowdsec/export.go index e482bac09..57a74ee9d 100644 --- a/backend/internal/crowdsec/export.go +++ b/backend/internal/crowdsec/export.go @@ -77,7 +77,8 @@ func checkExportImportable(compressed, uncompressed, maxCompressed int64, maxRat // returns the uncompressed byte total. Excluded paths (IsExportExcluded) are left out, directories are // skipped with fs.SkipDir only when excluded, and non-regular entries (symlinks, pipes, sockets, // devices) are skipped so that nothing is followed or blocks on open. A file that vanishes is skipped; -// an unreadable directory is logged and skipped; an unreadable regular file, or one whose size changes +// an unreadable directory is logged and skipped; the installed hub items are listed in a generated +// ManifestEntryName entry (only when there are any); an unreadable regular file, or one whose size changes // while it is copied, fails the export with ErrExportFileUnreadable. func WriteExportArchive(ctx context.Context, dataDir string, w io.Writer) (int64, error) { return writeExportArchive(ctx, dataDir, w, MaxImportUncompressedBytes, exportHooks{}) @@ -112,6 +113,11 @@ func writeExportArchive(ctx context.Context, dataDir string, w io.Writer, maxByt if d.IsDir() { return nil } + if IsManifestPath(rel) { + // The name is reserved for the generated hub item list; a live file never takes its place. + logger.Log().Warn("skipping a file with the reserved hub item manifest name during export") + return nil + } if !d.Type().IsRegular() { logger.Log().Warnf("skipping non-regular entry %s during export", util.SanitizeForLog(rel)) return nil @@ -123,6 +129,11 @@ func writeExportArchive(ctx context.Context, dataDir string, w io.Writer, maxByt if walkErr != nil { return total, fmt.Errorf("export walk: %w", walkErr) } + n, manifestErr := writeHubManifest(tw, dataDir, total, maxBytes) + total += n + if manifestErr != nil { + return total, fmt.Errorf("export hub item manifest: %w", manifestErr) + } if err := tw.Close(); err != nil { return total, fmt.Errorf("close tar writer: %w", err) } diff --git a/backend/internal/crowdsec/hubitem.go b/backend/internal/crowdsec/hubitem.go new file mode 100644 index 000000000..93b0ee4ee --- /dev/null +++ b/backend/internal/crowdsec/hubitem.go @@ -0,0 +1,90 @@ +package crowdsec + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "slices" + "strings" +) + +// HubItemTypes are the hub item types an export records and a restore can install. It is wider than the +// types Charon's own presets use (allowedPresetItemTypes). +var HubItemTypes = []string{"collections", "contexts", "parsers", "postoverflows", "scenarios"} + +var ( + // hubItemNamePattern accepts every real hub item name, mixed case included (for example + // "Azlaroc/sftpgo", "crowdsecurity/CVE-2022-26134"). ASCII only, a single slash, no leading dash. + hubItemNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + // presetItemNamePattern is the stricter lowercase form Charon uses for the presets it authors. + presetItemNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*/[a-z0-9][a-z0-9_.-]*$`) +) + +// IsHubItemType reports whether t is one of HubItemTypes. +func IsHubItemType(t string) bool { + return slices.Contains(HubItemTypes, t) +} + +// ValidHubItemName reports whether name is a well-formed hub item name in owner/name form. A name that +// passes can never be read as a flag, contains no whitespace or shell metacharacters and cannot +// traverse a path. +func ValidHubItemName(name string) bool { + return hubItemNamePattern.MatchString(name) && !strings.Contains(name, "..") +} + +// ValidPresetItemName is ValidHubItemName restricted to the lowercase names Charon's presets use. +func ValidPresetItemName(name string) bool { + return ValidHubItemName(name) && presetItemNamePattern.MatchString(name) +} + +// HubItem identifies one installed hub item by type and name. Versions are not recorded: they cannot be +// recovered from a link and cscli cannot pin them. +type HubItem struct { + Type string `json:"type"` + Name string `json:"name"` +} + +// Valid reports whether the item has a supported type and a well-formed name. +func (i HubItem) Valid() bool { + return IsHubItemType(i.Type) && ValidHubItemName(i.Name) +} + +// compareHubItems orders items by type, then name. +func compareHubItems(a, b HubItem) int { + if c := strings.Compare(a.Type, b.Type); c != 0 { + return c + } + return strings.Compare(a.Name, b.Name) +} + +// VerifyHubItem requires `cscli inspect -o json` to report the item as installed and +// untainted. Callers validate itemType and name first. +func VerifyHubItem(ctx context.Context, exec CommandExecutor, itemType, name string) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("verify %s %s: %w", itemType, name, err) + } + out, err := exec.Execute(ctx, "cscli", itemType, "inspect", name, "-o", "json") + if err != nil { + return err + } + // Field names follow cscli's `-o json` inspect output (installed, tainted). + var state struct { + Installed *bool `json:"installed"` + Tainted bool `json:"tainted"` + } + if err := json.Unmarshal(out, &state); err != nil { + return fmt.Errorf("parse inspect output: %w", err) + } + if state.Installed == nil { + return errors.New("inspect output has no installed field") + } + if !*state.Installed { + return errors.New("item reported as not installed") + } + if state.Tainted { + return errors.New("item reported as tainted") + } + return nil +} diff --git a/backend/internal/crowdsec/hubitem_test.go b/backend/internal/crowdsec/hubitem_test.go new file mode 100644 index 000000000..0e5b5701b --- /dev/null +++ b/backend/internal/crowdsec/hubitem_test.go @@ -0,0 +1,101 @@ +package crowdsec + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidHubItemName(t *testing.T) { + t.Parallel() + good := []string{ + "crowdsecurity/sshd", "Azlaroc/sftpgo", "Dominic-Wagner/vaultwarden", "crowdsecurity/CVE-2022-26134", + "a/b", "0/0", "o.w_n-er/na.me_1-x", strings.Repeat("a", 64) + "/" + strings.Repeat("b", 128), + } + for _, n := range good { + require.True(t, ValidHubItemName(n), n) + } + bad := []string{ + "", "/", "a/", "/b", "-a/b", "a/-b", "--help", "-h/x", "a/b/c", "a/../b", "../x", "a..b/c", "a/b..c", "a;b/c", "a/b c", + "a/b\n", "a\n/b", "ä/b", "a/ö", ".a/b", "a/.b", "a/b$", "a/`b`", "a b/c", "nofolder", + strings.Repeat("a", 65) + "/b", "a/" + strings.Repeat("b", 129), + } + for _, n := range bad { + require.False(t, ValidHubItemName(n), "%q", n) + } +} + +func TestValidPresetItemNameIsStricter(t *testing.T) { + t.Parallel() + require.True(t, ValidPresetItemName("crowdsecurity/sshd")) + require.True(t, ValidPresetItemName("a_b/c.d-e")) + require.False(t, ValidPresetItemName("CrowdSecurity/sshd")) + require.False(t, ValidPresetItemName("crowdsecurity/CVE-2022-26134")) + require.False(t, ValidPresetItemName("a/../b")) + require.False(t, ValidPresetItemName("a/b\n")) +} + +func TestHubItemTypes(t *testing.T) { + t.Parallel() + for _, typ := range []string{"collections", "parsers", "scenarios", "postoverflows", "contexts"} { + require.True(t, IsHubItemType(typ), typ) + require.True(t, HubItem{Type: typ, Name: "a/b"}.Valid()) + } + for _, typ := range []string{"", "appsec-rules", "appsec-configs", "Collections", "collection", "parsers/"} { + require.False(t, IsHubItemType(typ), typ) + require.False(t, HubItem{Type: typ, Name: "a/b"}.Valid()) + } + require.False(t, HubItem{Type: "parsers", Name: "bad"}.Valid()) + // The restore's set is wider than the preset set; presets stay unchanged. + require.NotContains(t, allowedPresetItemTypes, "contexts") + require.NotContains(t, allowedPresetItemTypes, "appsec-rules") +} + +type verifyExec struct { + out string + err error + args []string +} + +func (v *verifyExec) Execute(_ context.Context, name string, args ...string) ([]byte, error) { + v.args = append([]string{name}, args...) + return []byte(v.out), v.err +} + +func TestVerifyHubItem(t *testing.T) { + t.Parallel() + cases := []struct { + name string + out string + execErr error + wantErr string + }{ + {"installed", `{"installed":true,"tainted":false}`, nil, ""}, + {"not installed", `{"installed":false}`, nil, "not installed"}, + {"tainted", `{"installed":true,"tainted":true}`, nil, "tainted"}, + {"no field", `{}`, nil, "no installed field"}, + {"bad json", `nope`, nil, "parse inspect output"}, + {"exec error", ``, errors.New("boom"), "boom"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + exec := &verifyExec{out: tc.out, err: tc.execErr} + err := VerifyHubItem(t.Context(), exec, "collections", "crowdsecurity/sshd") + if tc.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.wantErr) + } + require.Equal(t, []string{"cscli", "collections", "inspect", "crowdsecurity/sshd", "-o", "json"}, exec.args) + }) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + exec := &verifyExec{} + require.ErrorIs(t, VerifyHubItem(ctx, exec, "collections", "a/b"), context.Canceled) + require.Nil(t, exec.args, "a cancelled context never reaches cscli") +} diff --git a/backend/internal/crowdsec/hubitems.go b/backend/internal/crowdsec/hubitems.go new file mode 100644 index 000000000..5a14f176a --- /dev/null +++ b/backend/internal/crowdsec/hubitems.go @@ -0,0 +1,138 @@ +package crowdsec + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/Wikid82/charon/backend/internal/logger" + "github.com/Wikid82/charon/backend/internal/util" +) + +const ( + // MaxHubItems bounds the items one manifest, scan or restore request carries. + MaxHubItems = 1000 + // maxHubScanEntries bounds how many directory entries one scan visits. + maxHubScanEntries = 10 * MaxHubItems +) + +// stagedHubTypes keep their installed links one directory deeper, below a processing stage such as +// s01-parse. +var stagedHubTypes = map[string]bool{"parsers": true, "postoverflows": true} + +// InstalledHubItems lists the hub items installed below dataDir, sorted by type then name. An item is +// installed when a symlink under / (or config//) points into the hub tree at +// hub//[stage/]/.yaml. Only the link text is read: no cscli, no network, and the +// target is not required to exist. Links that do not fit that shape (relative, pointing elsewhere, a +// name that fails ValidHubItemName) are skipped, real files are not hub items, and at most MaxHubItems +// are returned. +func InstalledHubItems(dataDir string) ([]HubItem, error) { + items, _, err := scanHubItems(dataDir) + return items, err +} + +// scanHubItems is InstalledHubItems plus the number of links that were skipped. +func scanHubItems(dataDir string) (items []HubItem, skipped int, err error) { + seen := map[HubItem]struct{}{} + visited := 0 + for _, base := range []string{"config", ""} { + for _, typ := range HubItemTypes { + root := filepath.Join(dataDir, base, typ) + walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return handleHubScanError(root, p, d, werr) + } + visited++ + if visited > maxHubScanEntries { + return fs.SkipAll + } + if d.Type()&fs.ModeSymlink == 0 { + return nil + } + rel, relErr := filepath.Rel(root, p) + if relErr != nil { + return fmt.Errorf("hub item relative path: %w", relErr) + } + item, ok := hubItemFromLink(typ, rel, p) + if !ok { + skipped++ + return nil + } + seen[item] = struct{}{} + return nil + }) + if walkErr != nil { + return nil, skipped, fmt.Errorf("scan %s: %w", typ, walkErr) + } + } + } + items = make([]HubItem, 0, len(seen)) + for item := range seen { + items = append(items, item) + } + slices.SortFunc(items, compareHubItems) + if len(items) > MaxHubItems { + logger.Log().Warnf("hub item scan found %d items; keeping the first %d", len(items), MaxHubItems) + items = items[:MaxHubItems] + } + if skipped > 0 { + logger.Log().Debugf("hub item scan skipped %d links that are not installed hub items", skipped) + } + return items, skipped, nil +} + +// handleHubScanError ignores a type directory that does not exist and an unreadable subdirectory; +// anything else (including an unreadable type directory) fails the scan. +func handleHubScanError(root, p string, d fs.DirEntry, err error) error { + switch { + case errors.Is(err, fs.ErrNotExist): + return nil + case d != nil && d.IsDir() && p != root && errors.Is(err, fs.ErrPermission): + logger.Log().WithError(err).Warnf("skipping unreadable directory %s during hub item scan", util.SanitizeForLog(p)) + return fs.SkipDir + default: + return err + } +} + +// hubItemFromLink derives the item behind the symlink at p (rel below its type directory). It reports +// false for anything that is not a link into the hub tree of the same type. +func hubItemFromLink(typ, rel, p string) (HubItem, bool) { + wantDepth := 1 + if stagedHubTypes[typ] { + wantDepth = 2 + } + if strings.Count(filepath.ToSlash(rel), "/")+1 != wantDepth { + return HubItem{}, false + } + target, err := os.Readlink(p) + if err != nil || !filepath.IsAbs(target) { + return HubItem{}, false + } + item, ok := hubItemFromTarget(typ, filepath.ToSlash(filepath.Clean(target))) + return item, ok && item.Valid() +} + +// hubItemFromTarget parses a cleaned absolute link target ending in +// hub//[stage/]/.yaml. +func hubItemFromTarget(typ, target string) (HubItem, bool) { + parts := strings.Split(strings.TrimPrefix(target, "/"), "/") + tail := 2 // owner/name.yaml + if stagedHubTypes[typ] { + tail = 3 // stage/owner/name.yaml + } + hubAt := len(parts) - tail - 2 + if hubAt < 0 || parts[hubAt] != "hub" || parts[hubAt+1] != typ { + return HubItem{}, false + } + owner, file := parts[len(parts)-2], parts[len(parts)-1] + name, found := strings.CutSuffix(file, ".yaml") + if !found { + return HubItem{}, false + } + return HubItem{Type: typ, Name: owner + "/" + name}, true +} diff --git a/backend/internal/crowdsec/hubitems_test.go b/backend/internal/crowdsec/hubitems_test.go new file mode 100644 index 000000000..93d307588 --- /dev/null +++ b/backend/internal/crowdsec/hubitems_test.go @@ -0,0 +1,153 @@ +package crowdsec + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// link creates a symlink at dir/rel pointing to target, creating parent directories. +func link(t *testing.T, dir, rel, target string) { + t.Helper() + p := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(t, os.Symlink(target, p)) +} + +func TestInstalledHubItemsLinkTargets(t *testing.T) { + t.Parallel() + const hub = "/etc/crowdsec/hub/" + cases := []struct { + name string + rel string + target string + want *HubItem + }{ + {"collection", "config/collections/caddy.yaml", hub + "collections/crowdsecurity/caddy.yaml", &HubItem{"collections", "crowdsecurity/caddy"}}, + {"context", "config/contexts/http_base.yaml", hub + "contexts/crowdsecurity/http_base.yaml", &HubItem{"contexts", "crowdsecurity/http_base"}}, + {"scenario", "config/scenarios/CVE-2017-9841.yaml", hub + "scenarios/crowdsecurity/CVE-2017-9841.yaml", &HubItem{"scenarios", "crowdsecurity/CVE-2017-9841"}}, + {"parser with stage", "config/parsers/s01-parse/caddy-logs.yaml", hub + "parsers/s01-parse/crowdsecurity/caddy-logs.yaml", &HubItem{"parsers", "crowdsecurity/caddy-logs"}}, + {"postoverflow with stage", "config/postoverflows/s01-whitelist/rdns.yaml", hub + "postoverflows/s01-whitelist/crowdsecurity/rdns.yaml", &HubItem{"postoverflows", "crowdsecurity/rdns"}}, + {"mixed case owner", "config/collections/sftpgo.yaml", hub + "collections/Azlaroc/sftpgo.yaml", &HubItem{"collections", "Azlaroc/sftpgo"}}, + {"root layout", "collections/caddy.yaml", hub + "collections/crowdsecurity/caddy.yaml", &HubItem{"collections", "crowdsecurity/caddy"}}, + {"hub below the data dir", "config/collections/c.yaml", "/app/data/crowdsec/config/hub/collections/o/c.yaml", &HubItem{"collections", "o/c"}}, + {"unclean target", "config/collections/c.yaml", hub + "collections/./x/../o/c.yaml", &HubItem{"collections", "o/c"}}, + {"dangling target is still listed", "config/collections/c.yaml", "/nonexistent/hub/collections/o/c.yaml", &HubItem{"collections", "o/c"}}, + {"relative link", "config/collections/c.yaml", "../hub/collections/o/c.yaml", nil}, + {"user link elsewhere", "config/collections/c.yaml", "/home/user/mine/c.yaml", nil}, + {"wrong type directory", "config/collections/c.yaml", hub + "scenarios/o/c.yaml", nil}, + {"stage missing for parser", "config/parsers/p.yaml", hub + "parsers/o/p.yaml", nil}, + {"parser link not in a stage directory", "config/parsers/p.yaml", hub + "parsers/s01-parse/o/p.yaml", nil}, + {"unexpected stage for collection", "config/collections/c.yaml", hub + "collections/s01/o/c.yaml", nil}, + {"link too deep", "config/collections/sub/c.yaml", hub + "collections/o/c.yaml", nil}, + {"staged link too deep", "config/parsers/s01/x/p.yaml", hub + "parsers/s01/o/p.yaml", nil}, + {"not yaml", "config/collections/c.yaml", hub + "collections/o/c.json", nil}, + {"upper case extension", "config/collections/c.yaml", hub + "collections/o/c.YAML", nil}, + {"hub is the last component", "config/collections/c.yaml", hub, nil}, + {"hub missing", "config/collections/c.yaml", "/etc/crowdsec/collections/o/c.yaml", nil}, + {"invalid name", "config/collections/c.yaml", hub + "collections/o/-c.yaml", nil}, + {"too long owner", "config/collections/c.yaml", hub + "collections/" + fmt.Sprintf("%065d", 0) + "/c.yaml", nil}, + {"too short", "config/collections/c.yaml", "/hub/collections", nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + link(t, dir, tc.rel, tc.target) + items, skipped, err := scanHubItems(dir) + require.NoError(t, err) + if tc.want == nil { + require.Empty(t, items) + require.Equal(t, 1, skipped) + return + } + require.Equal(t, []HubItem{*tc.want}, items) + require.Zero(t, skipped) + }) + } +} + +func TestInstalledHubItemsIgnoresNonLinksAndSorts(t *testing.T) { + t.Parallel() + dir := t.TempDir() + const hub = "/etc/crowdsec/hub/" + link(t, dir, "config/scenarios/z.yaml", hub+"scenarios/zeta/z.yaml") + link(t, dir, "config/collections/b.yaml", hub+"collections/crowdsecurity/b.yaml") + link(t, dir, "config/collections/a.yaml", hub+"collections/Azlaroc/a.yaml") + link(t, dir, "config/parsers/s02-enrich/p.yaml", hub+"parsers/s02-enrich/crowdsecurity/p.yaml") + // Duplicate through the root layout, a real file, a directory link and an unrelated tree. + link(t, dir, "collections/b.yaml", hub+"collections/crowdsecurity/b.yaml") + require.NoError(t, os.WriteFile(filepath.Join(dir, "config", "collections", "mine.yaml"), []byte("x"), 0o600)) + link(t, dir, "config/contexts/sub", hub+"contexts/crowdsecurity") + link(t, dir, "config/other/o.yaml", hub+"collections/o/o.yaml") + + items, err := InstalledHubItems(dir) + require.NoError(t, err) + require.Equal(t, []HubItem{ + {"collections", "Azlaroc/a"}, + {"collections", "crowdsecurity/b"}, + {"parsers", "crowdsecurity/p"}, + {"scenarios", "zeta/z"}, + }, items) +} + +func TestInstalledHubItemsEmptyOrMissingTree(t *testing.T) { + t.Parallel() + items, err := InstalledHubItems(t.TempDir()) + require.NoError(t, err) + require.Empty(t, items) + items, err = InstalledHubItems(filepath.Join(t.TempDir(), "absent")) + require.NoError(t, err) + require.Empty(t, items) +} + +func TestInstalledHubItemsCap(t *testing.T) { + t.Parallel() + dir := t.TempDir() + for i := 0; i < MaxHubItems+25; i++ { + link(t, dir, fmt.Sprintf("config/collections/c%04d.yaml", i), fmt.Sprintf("/etc/crowdsec/hub/collections/o/c%04d.yaml", i)) + } + items, err := InstalledHubItems(dir) + require.NoError(t, err) + require.Len(t, items, MaxHubItems) + require.Equal(t, HubItem{"collections", "o/c0000"}, items[0]) + require.Equal(t, HubItem{"collections", fmt.Sprintf("o/c%04d", MaxHubItems-1)}, items[MaxHubItems-1]) +} + +func TestInstalledHubItemsScanBound(t *testing.T) { + t.Parallel() + dir := t.TempDir() + d := filepath.Join(dir, "config", "contexts") + require.NoError(t, os.MkdirAll(d, 0o750)) + for i := 0; i < maxHubScanEntries+5; i++ { + require.NoError(t, os.WriteFile(filepath.Join(d, fmt.Sprintf("f%05d", i)), nil, 0o600)) + } + items, err := InstalledHubItems(dir) + require.NoError(t, err) + require.Empty(t, items) +} + +func TestInstalledHubItemsUnreadableDirectories(t *testing.T) { + t.Parallel() + skipIfRoot(t) + dir := t.TempDir() + link(t, dir, "config/collections/ok.yaml", "/hub/collections/o/ok.yaml") + sub := filepath.Join(dir, "config", "parsers", "s01-parse") + require.NoError(t, os.MkdirAll(sub, 0o750)) + require.NoError(t, os.Chmod(sub, 0o000)) //nolint:gosec // G302: test makes a directory unreadable + t.Cleanup(func() { _ = os.Chmod(sub, 0o700) }) //nolint:gosec // G302: restore so TempDir cleanup works + + items, err := InstalledHubItems(dir) + require.NoError(t, err, "an unreadable stage directory is skipped") + require.Equal(t, []HubItem{{"collections", "o/ok"}}, items) + + root := filepath.Join(dir, "config", "scenarios") + require.NoError(t, os.MkdirAll(root, 0o750)) + require.NoError(t, os.Chmod(root, 0o000)) //nolint:gosec // G302: test makes a directory unreadable + t.Cleanup(func() { _ = os.Chmod(root, 0o700) }) //nolint:gosec // G302: restore so TempDir cleanup works + _, err = InstalledHubItems(dir) + require.Error(t, err, "an unreadable type directory fails the scan") +} diff --git a/backend/internal/crowdsec/hubmanifest.go b/backend/internal/crowdsec/hubmanifest.go new file mode 100644 index 000000000..36cf28ca7 --- /dev/null +++ b/backend/internal/crowdsec/hubmanifest.go @@ -0,0 +1,123 @@ +package crowdsec + +import ( + "archive/tar" + "encoding/json" + "fmt" + "path/filepath" + "slices" + "time" + + "github.com/Wikid82/charon/backend/internal/logger" +) + +// manifestVersion is the only manifest format this version reads and writes. +const manifestVersion = 1 + +// maxManifestBytes bounds the manifest an import reads; a full manifest is about 260 KB. +const maxManifestBytes int64 = 1 << 20 + +// hubManifest is the content of the ManifestEntryName archive entry. +type hubManifest struct { + Version int `json:"version"` + Items []HubItem `json:"items"` +} + +// ArchiveHubItems is the usable hub item manifest of an uploaded archive. +type ArchiveHubItems struct { + // Items are the valid, de-duplicated manifest items, sorted by type then name. + Items []HubItem + // SkippedInvalid counts manifest items that were dropped because they failed validation. + SkippedInvalid int +} + +// HubItemsReport is the hub_items part of an import response. +type HubItemsReport struct { + InExport int `json:"in_export"` + SkippedInvalid int `json:"skipped_invalid"` + Missing []HubItem `json:"missing"` +} + +// IsManifestPath reports whether rel (relative to the archive or data directory root) names the +// reserved manifest entry. Only the exact root path matches: the same name below a directory is an +// ordinary file. +func IsManifestPath(rel string) bool { + return filepath.ToSlash(filepath.Clean(rel)) == ManifestEntryName +} + +// parseHubManifest reads a manifest strictly. Invalid JSON, an unknown version or more than MaxHubItems +// entries yield nil (the whole manifest is ignored); an individually invalid item is dropped and counted. +func parseHubManifest(data []byte) *ArchiveHubItems { + var raw struct { + Version int `json:"version"` + Items []json.RawMessage `json:"items"` + } + if err := json.Unmarshal(data, &raw); err != nil || raw.Version != manifestVersion || len(raw.Items) > MaxHubItems { + return nil + } + out := &ArchiveHubItems{Items: make([]HubItem, 0, len(raw.Items))} + seen := map[HubItem]struct{}{} + for _, msg := range raw.Items { + var item HubItem + if err := json.Unmarshal(msg, &item); err != nil || !item.Valid() { + out.SkippedInvalid++ + continue + } + if _, dup := seen[item]; dup { + continue + } + seen[item] = struct{}{} + out.Items = append(out.Items, item) + } + slices.SortFunc(out.Items, compareHubItems) + return out +} + +// MissingHubItems returns the manifest items that are not in installed, sorted. It is never nil. +func (a *ArchiveHubItems) MissingHubItems(installed []HubItem) []HubItem { + have := make(map[HubItem]struct{}, len(installed)) + for _, item := range installed { + have[item] = struct{}{} + } + missing := []HubItem{} + for _, item := range a.Items { + if _, ok := have[item]; !ok { + missing = append(missing, item) + } + } + return missing +} + +// Report builds the import response field against the items installed on this host. +func (a *ArchiveHubItems) Report(installed []HubItem) HubItemsReport { + return HubItemsReport{InExport: len(a.Items), SkippedInvalid: a.SkippedInvalid, Missing: a.MissingHubItems(installed)} +} + +// writeHubManifest appends the manifest entry for the items installed below dataDir and returns the +// bytes it added. Nothing is written when no item is installed, so exports of such hosts are unchanged. +// A failed scan is logged and the manifest is omitted: it only helps a later import. +func writeHubManifest(tw *tar.Writer, dataDir string, total, maxBytes int64) (int64, error) { + items, err := InstalledHubItems(dataDir) + if err != nil { + logger.Log().WithError(err).Warn("hub item list omitted from the export") + return 0, nil + } + if len(items) == 0 { + return 0, nil + } + body, err := json.Marshal(hubManifest{Version: manifestVersion, Items: items}) + if err != nil { + return 0, fmt.Errorf("encode hub item manifest: %w", err) + } + if total+int64(len(body)) > maxBytes { + return 0, ErrExportTooLarge + } + hdr := &tar.Header{Name: ManifestEntryName, Size: int64(len(body)), Mode: 0o600, ModTime: time.Now()} + if err = tw.WriteHeader(hdr); err != nil { + return 0, fmt.Errorf("write tar header: %w", err) + } + if _, err = tw.Write(body); err != nil { + return 0, fmt.Errorf("write hub item manifest: %w", err) + } + return int64(len(body)), nil +} diff --git a/backend/internal/crowdsec/hubmanifest_test.go b/backend/internal/crowdsec/hubmanifest_test.go new file mode 100644 index 000000000..f81eb1826 --- /dev/null +++ b/backend/internal/crowdsec/hubmanifest_test.go @@ -0,0 +1,370 @@ +package crowdsec + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsManifestPath(t *testing.T) { + t.Parallel() + for _, rel := range []string{"charon-hub-items.json", "./charon-hub-items.json", "a/../charon-hub-items.json"} { + require.True(t, IsManifestPath(rel), rel) + } + for _, rel := range []string{"", ".", "config/charon-hub-items.json", "Charon-Hub-Items.json", "charon-hub-items.json.bak", "x/charon-hub-items.json"} { + require.False(t, IsManifestPath(rel), rel) + } +} + +func TestParseHubManifest(t *testing.T) { + t.Parallel() + many := make([]string, MaxHubItems+1) + for i := range many { + many[i] = fmt.Sprintf(`{"type":"collections","name":"o/c%d"}`, i) + } + atCap := strings.Join(many[:MaxHubItems], ",") + cases := []struct { + name string + body string + ignored bool + wantItems []HubItem + wantSkipped int + }{ + {"sorted and deduplicated", `{"version":1,"items":[{"type":"scenarios","name":"b/b"},{"type":"collections","name":"z/z"},{"type":"collections","name":"Azlaroc/sftpgo"},{"type":"collections","name":"z/z"}]}`, false, + []HubItem{{"collections", "Azlaroc/sftpgo"}, {"collections", "z/z"}, {"scenarios", "b/b"}}, 0}, + {"mixed case survives", `{"version":1,"items":[{"type":"scenarios","name":"crowdsecurity/CVE-2022-26134"}]}`, false, []HubItem{{"scenarios", "crowdsecurity/CVE-2022-26134"}}, 0}, + {"empty list is usable", `{"version":1,"items":[]}`, false, []HubItem{}, 0}, + {"no items key is usable", `{"version":1}`, false, []HubItem{}, 0}, + {"invalid items dropped and counted", `{"version":1,"items":[{"type":"collections","name":"ok/ok"},{"type":"appsec-rules","name":"a/b"},{"type":"collections","name":"--help"},{"type":"collections","name":"a/../b"},{"type":5,"name":"a/b"},"str",null,{"type":"parsers","name":"a/b\n"}]}`, false, + []HubItem{{"collections", "ok/ok"}}, 7}, + {"exactly the cap", `{"version":1,"items":[` + atCap + `]}`, false, nil, 0}, + {"over the cap", `{"version":1,"items":[` + strings.Join(many, ",") + `]}`, true, nil, 0}, + {"unknown version", `{"version":2,"items":[{"type":"collections","name":"a/b"}]}`, true, nil, 0}, + {"missing version", `{"items":[]}`, true, nil, 0}, + {"malformed", `{"version":1,"items":[`, true, nil, 0}, + {"trailing data", `{"version":1,"items":[]} {}`, true, nil, 0}, + {"items not a list", `{"version":1,"items":{}}`, true, nil, 0}, + {"not an object", `[]`, true, nil, 0}, + {"empty", ``, true, nil, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := parseHubManifest([]byte(tc.body)) + if tc.ignored { + require.Nil(t, got) + return + } + require.NotNil(t, got) + require.Equal(t, tc.wantSkipped, got.SkippedInvalid) + if tc.wantItems != nil { + require.Equal(t, tc.wantItems, got.Items) + } else { + require.Len(t, got.Items, MaxHubItems) + } + }) + } +} + +func TestMissingHubItemsAndReport(t *testing.T) { + t.Parallel() + m := &ArchiveHubItems{Items: []HubItem{{"collections", "a/a"}, {"collections", "b/b"}, {"parsers", "a/a"}}, SkippedInvalid: 2} + installed := []HubItem{{"collections", "b/b"}, {"scenarios", "extra/only-here"}, {"scenarios", "a/a"}} + require.Equal(t, []HubItem{{"collections", "a/a"}, {"parsers", "a/a"}}, m.MissingHubItems(installed)) + + rep := m.Report(m.Items) + require.Equal(t, HubItemsReport{InExport: 3, SkippedInvalid: 2, Missing: []HubItem{}}, rep) + raw, err := json.Marshal(rep) + require.NoError(t, err) + require.JSONEq(t, `{"in_export":3,"skipped_invalid":2,"missing":[]}`, string(raw)) +} + +// realLayoutTree builds a data dir like a real install: hub links of every type plus ordinary files. +func realLayoutTree(t *testing.T) string { + t.Helper() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"config/config.yaml": "a: 1\n", "config/collections/mine.yaml": "user file"}) + const hub = "/etc/crowdsec/hub/" + link(t, dir, "config/collections/sftpgo.yaml", hub+"collections/Azlaroc/sftpgo.yaml") + link(t, dir, "config/collections/caddy.yaml", hub+"collections/crowdsecurity/caddy.yaml") + link(t, dir, "config/scenarios/CVE-2022-26134.yaml", hub+"scenarios/crowdsecurity/CVE-2022-26134.yaml") + link(t, dir, "config/parsers/s01-parse/caddy-logs.yaml", hub+"parsers/s01-parse/crowdsecurity/caddy-logs.yaml") + link(t, dir, "config/postoverflows/s01-whitelist/rdns.yaml", hub+"postoverflows/s01-whitelist/crowdsecurity/rdns.yaml") + link(t, dir, "config/contexts/http_base.yaml", hub+"contexts/crowdsecurity/http_base.yaml") + link(t, dir, "config/collections/elsewhere.yaml", "/home/me/elsewhere.yaml") + return dir +} + +var realLayoutItems = []HubItem{ + {"collections", "Azlaroc/sftpgo"}, + {"collections", "crowdsecurity/caddy"}, + {"contexts", "crowdsecurity/http_base"}, + {"parsers", "crowdsecurity/caddy-logs"}, + {"postoverflows", "crowdsecurity/rdns"}, + {"scenarios", "crowdsecurity/CVE-2022-26134"}, +} + +func exportToFile(t *testing.T, dir string) (archivePath string, raw []byte) { + t.Helper() + var buf bytes.Buffer + _, err := WriteExportArchive(context.Background(), dir, &buf) + require.NoError(t, err) + archivePath = filepath.Join(t.TempDir(), "export.tar.gz") + require.NoError(t, os.WriteFile(archivePath, buf.Bytes(), 0o600)) + return archivePath, buf.Bytes() +} + +func TestExportManifestRoundTrip(t *testing.T) { + t.Parallel() + dir := realLayoutTree(t) + archivePath, raw := exportToFile(t, dir) + + require.Equal(t, []string{"charon-hub-items.json", "config/collections/mine.yaml", "config/config.yaml"}, exportNames(t, raw)) + + body := readTarEntry(t, raw, ManifestEntryName) + var m hubManifest + require.NoError(t, json.Unmarshal(body, &m)) + require.Equal(t, 1, m.Version) + require.Equal(t, realLayoutItems, m.Items) + require.JSONEq(t, `{"version":1,"items":[`+ + `{"type":"collections","name":"Azlaroc/sftpgo"},{"type":"collections","name":"crowdsecurity/caddy"},`+ + `{"type":"contexts","name":"crowdsecurity/http_base"},{"type":"parsers","name":"crowdsecurity/caddy-logs"},`+ + `{"type":"postoverflows","name":"crowdsecurity/rdns"},{"type":"scenarios","name":"crowdsecurity/CVE-2022-26134"}]}`, string(body)) + + info, err := InspectArchive(archivePath) + require.NoError(t, err) + require.NotNil(t, info.HubItems) + require.Equal(t, realLayoutItems, info.HubItems.Items) + require.Zero(t, info.HubItems.SkippedInvalid) + require.NotContains(t, info.Files, ManifestEntryName, "the manifest is not an inspected file") + + // Exports are deterministic in content. + _, raw2 := exportToFile(t, dir) + require.Equal(t, body, readTarEntry(t, raw2, ManifestEntryName)) +} + +func TestExportWithoutInstalledItemsHasNoManifest(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"config/config.yaml": "a: 1\n"}) + link(t, dir, "config/collections/x.yaml", "/somewhere/else.yaml") + archivePath, raw := exportToFile(t, dir) + require.Equal(t, []string{"config/config.yaml"}, exportNames(t, raw)) + info, err := InspectArchive(archivePath) + require.NoError(t, err) + require.Nil(t, info.HubItems) +} + +func TestExportNeverEmitsLiveFileWithManifestName(t *testing.T) { + t.Parallel() + t.Run("without items", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"config/config.yaml": "a: 1\n", ManifestEntryName: `{"version":1,"items":[{"type":"collections","name":"evil/evil"}]}`}) + _, raw := exportToFile(t, dir) + require.Equal(t, []string{"config/config.yaml"}, exportNames(t, raw)) + }) + t.Run("with items the generated one wins", func(t *testing.T) { + t.Parallel() + dir := realLayoutTree(t) + writeExportTree(t, dir, map[string]string{ManifestEntryName: `{"version":1,"items":[{"type":"collections","name":"evil/evil"}]}`}) + _, raw := exportToFile(t, dir) + names := exportNames(t, raw) + require.Equal(t, 1, countOf(names, ManifestEntryName)) + var m hubManifest + require.NoError(t, json.Unmarshal(readTarEntry(t, raw, ManifestEntryName), &m)) + require.Equal(t, realLayoutItems, m.Items) + }) + t.Run("nested file is ordinary", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{"config/config.yaml": "a: 1\n", "config/" + ManifestEntryName: "ordinary"}) + _, raw := exportToFile(t, dir) + require.Equal(t, []string{"config/charon-hub-items.json", "config/config.yaml"}, exportNames(t, raw)) + }) +} + +func TestExportManifestFailureModes(t *testing.T) { + t.Parallel() + t.Run("size cap counts the manifest", func(t *testing.T) { + t.Parallel() + dir := realLayoutTree(t) + _, err := writeExportArchive(context.Background(), dir, io.Discard, 20, exportHooks{}) + require.ErrorIs(t, err, ErrExportTooLarge) + }) + t.Run("manifest fits when only it is left", func(t *testing.T) { + t.Parallel() + dir := realLayoutTree(t) + total, err := writeExportArchive(context.Background(), dir, io.Discard, MaxImportUncompressedBytes, exportHooks{}) + require.NoError(t, err) + _, err = writeExportArchive(context.Background(), dir, io.Discard, total-1, exportHooks{}) + require.ErrorIs(t, err, ErrExportTooLarge) + _, err = writeExportArchive(context.Background(), dir, io.Discard, total, exportHooks{}) + require.NoError(t, err) + }) + t.Run("unreadable type directory omits the manifest", func(t *testing.T) { + t.Parallel() + skipIfRoot(t) + dir := realLayoutTree(t) + bad := filepath.Join(dir, "config", "scenarios") + require.NoError(t, os.Chmod(bad, 0o000)) //nolint:gosec // G302: test makes a directory unreadable + t.Cleanup(func() { _ = os.Chmod(bad, 0o700) }) //nolint:gosec // G302: restore so TempDir cleanup works + _, raw := exportToFile(t, dir) + require.NotContains(t, exportNames(t, raw), ManifestEntryName) + }) + t.Run("tar write failure", func(t *testing.T) { + t.Parallel() + dir := realLayoutTree(t) + _, err := WriteExportArchive(context.Background(), dir, failingWriter{err: errors.New("disk full")}) + require.Error(t, err) + }) +} + +func TestWriteHubManifestWriterErrors(t *testing.T) { + t.Parallel() + dir := realLayoutTree(t) + tw := tar.NewWriter(io.Discard) + require.NoError(t, tw.Close()) + _, err := writeHubManifest(tw, dir, 0, MaxImportUncompressedBytes) + require.Error(t, err, "a closed tar writer rejects the header") +} + +func countOf(names []string, want string) int { + n := 0 + for _, name := range names { + if name == want { + n++ + } + } + return n +} + +func readTarEntry(t *testing.T, gz []byte, name string) []byte { + t.Helper() + gr, err := gzip.NewReader(bytes.NewReader(gz)) + require.NoError(t, err) + tr := tar.NewReader(gr) + for { + hdr, nextErr := tr.Next() + require.NoError(t, nextErr, "entry %s not found", name) + if hdr.Name == name { + b, readErr := io.ReadAll(tr) + require.NoError(t, readErr) + return b + } + } +} + +// manifestArchive writes the given raw entries into a tar.gz and returns its path. +func manifestArchive(t *testing.T, build func(add func(h *tar.Header, body string))) string { + t.Helper() + raw := tarGzRaw(t, func(tw *tar.Writer) { + build(func(h *tar.Header, body string) { + h.Size = int64(len(body)) + if h.Typeflag == 0 { + h.Typeflag = tar.TypeReg + } + if h.Mode == 0 { + h.Mode = 0o600 + } + require.NoError(t, tw.WriteHeader(h)) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + }) + require.NoError(t, tw.Close()) + }) + p := filepath.Join(t.TempDir(), "m.tar.gz") + require.NoError(t, os.WriteFile(p, raw, 0o600)) + return p +} + +const goodManifest = `{"version":1,"items":[{"type":"collections","name":"a/a"}]}` + +func TestInspectArchiveManifest(t *testing.T) { + t.Parallel() + good := []HubItem{{"collections", "a/a"}} + cases := []struct { + name string + build func(add func(h *tar.Header, body string)) + want []HubItem // nil means ignored + }{ + {"root", func(add func(*tar.Header, string)) { add(&tar.Header{Name: ManifestEntryName}, goodManifest) }, good}, + {"dot slash spelling", func(add func(*tar.Header, string)) { add(&tar.Header{Name: "./" + ManifestEntryName}, goodManifest) }, good}, + {"dot dot spelling", func(add func(*tar.Header, string)) { add(&tar.Header{Name: "a/../" + ManifestEntryName}, goodManifest) }, good}, + {"nested is not the manifest", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: "config/" + ManifestEntryName}, goodManifest) + }, nil}, + {"directory typeflag", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName, Typeflag: tar.TypeDir}, "") + }, nil}, + {"symlink typeflag", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName, Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"}, "") + }, nil}, + {"non-regular entry does not count as a duplicate", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName, Typeflag: tar.TypeSymlink, Linkname: "x"}, "") + add(&tar.Header{Name: ManifestEntryName}, goodManifest) + }, good}, + {"duplicate exact", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName}, goodManifest) + add(&tar.Header{Name: ManifestEntryName}, goodManifest) + }, nil}, + {"duplicate through spellings", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName}, goodManifest) + add(&tar.Header{Name: "./" + ManifestEntryName}, `{"version":1,"items":[]}`) + }, nil}, + {"duplicate with nested is fine", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName}, goodManifest) + add(&tar.Header{Name: "x/" + ManifestEntryName}, `garbage`) + }, good}, + {"malformed", func(add func(*tar.Header, string)) { add(&tar.Header{Name: ManifestEntryName}, `{"version":`) }, nil}, + {"unknown version", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName}, `{"version":9,"items":[]}`) + }, nil}, + {"over the byte cap", func(add func(*tar.Header, string)) { + add(&tar.Header{Name: ManifestEntryName}, `{"version":1,"items":[],"pad":"`+strings.Repeat("x", int(maxManifestBytes))+`"}`) + }, nil}, + {"escaping name", func(add func(*tar.Header, string)) { add(&tar.Header{Name: "../" + ManifestEntryName}, goodManifest) }, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + info, err := InspectArchive(manifestArchive(t, tc.build)) + require.NoError(t, err) + require.False(t, info.Incomplete, "a manifest problem never makes the notification inspection incomplete") + if tc.want == nil { + require.Nil(t, info.HubItems) + return + } + require.NotNil(t, info.HubItems) + require.Equal(t, tc.want, info.HubItems.Items) + }) + } +} + +func TestInspectArchiveManifestSkippedInvalidAndLegacy(t *testing.T) { + t.Parallel() + info, err := InspectArchive(manifestArchive(t, func(add func(*tar.Header, string)) { + add(&tar.Header{Name: "config/config.yaml"}, "a: 1\n") + add(&tar.Header{Name: ManifestEntryName}, `{"version":1,"items":[{"type":"collections","name":"ok/ok"},{"type":"collections","name":"bad name"},{"type":"x","name":"a/b"}]}`) + })) + require.NoError(t, err) + require.Equal(t, []HubItem{{"collections", "ok/ok"}}, info.HubItems.Items) + require.Equal(t, 2, info.HubItems.SkippedInvalid) + + legacy, err := InspectArchive(manifestArchive(t, func(add func(*tar.Header, string)) { + add(&tar.Header{Name: "config/config.yaml"}, "a: 1\n") + })) + require.NoError(t, err) + require.Nil(t, legacy.HubItems) +} diff --git a/backend/internal/crowdsec/notifications_check.go b/backend/internal/crowdsec/notifications_check.go new file mode 100644 index 000000000..52249b2fa --- /dev/null +++ b/backend/internal/crowdsec/notifications_check.go @@ -0,0 +1,240 @@ +package crowdsec + +import ( + "bytes" + "fmt" + "path" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/Wikid82/charon/backend/internal/logger" + "github.com/Wikid82/charon/backend/internal/util" +) + +const ( + // maxListedNotifications bounds how many undefined names an error message spells out. + maxListedNotifications = 10 + // maxListedNameLen bounds one listed name; the archive is untrusted. + maxListedNameLen = 64 + + defaultConfigDir = "/etc/crowdsec" + defaultProfilesPath = "/etc/crowdsec/profiles.yaml" + defaultNotificationDir = "/etc/crowdsec/notifications" +) + +// UndefinedNotificationsError reports profiles that name notifications no file in the archive defines. +type UndefinedNotificationsError struct { + // Names are the referenced-but-undefined notification names, sorted. + Names []string +} + +func (e *UndefinedNotificationsError) Error() string { + listed := e.Names + more := 0 + if len(listed) > maxListedNotifications { + more = len(listed) - maxListedNotifications + listed = listed[:maxListedNotifications] + } + safe := make([]string, len(listed)) + for i, n := range listed { + n = util.SanitizeForLog(n) + if r := []rune(n); len(r) > maxListedNameLen { + n = string(r[:maxListedNameLen]) + "..." + } + safe[i] = n + } + msg := "profiles.yaml names notifications that have no matching file: " + strings.Join(safe, ", ") + if more > 0 { + msg += fmt.Sprintf(", and %d more", more) + } + return msg +} + +// VerifyInspectedNotifications returns an *UndefinedNotificationsError when the inspected archive's +// profiles name notifications that none of its notification files defines. Cases the archive cannot +// settle (see CheckProfileNotifications) pass. +func VerifyInspectedNotifications(info ArchiveInfo) error { + if missing := CheckProfileNotifications(info); len(missing) > 0 { + return &UndefinedNotificationsError{Names: missing} + } + return nil +} + +// CheckProfileNotifications returns the sorted notification names that profiles.yaml references and no +// notification file in the archive defines. CrowdSec keys notification configs by the name: field of +// every YAML document in the lowercase .yaml and .yml files directly inside notifications/, and refuses +// to start when a profile names one that is missing. Only the archive is considered: an import removes +// the live profiles and notification files before extracting. +// +// The check passes (returns nil) instead of guessing when the archive cannot settle the question: +// incomplete inspection, no profiles file, a config that redirects the profiles file or notification +// directory, a profiles or notification file that does not parse, or a notifications value that is not +// a list of names. +func CheckProfileNotifications(info ArchiveInfo) []string { + if info.Incomplete { + return skipNotificationCheck("an inspected entry exceeded a bound") + } + prefix := "" + if _, ok := info.Files["config/config.yaml"]; ok { + prefix = "config/" + } + profiles, ok := info.Files[prefix+"profiles.yaml"] + if !ok { + return skipNotificationCheck("no profiles file in the archive") + } + if configRedirectsPaths(info.Files[prefix+"config.yaml"]) { + return skipNotificationCheck("config redirects the profiles or notification paths") + } + referenced, ok := referencedNotifications(profiles) + if !ok || len(referenced) == 0 { + return skipNotificationCheck("profiles reference no notifications or cannot be read") + } + defined, ok := definedNotifications(info.Files, prefix+"notifications") + if !ok { + return skipNotificationCheck("a notification file cannot be read") + } + var missing []string + for name := range referenced { + if !defined[name] { + missing = append(missing, name) + } + } + sort.Strings(missing) + return missing +} + +func skipNotificationCheck(reason string) []string { + logger.Log().WithField("reason", reason).Debug("crowdsec import: notification reference check skipped") + return nil +} + +// configRedirectsPaths reports whether config.yaml (empty when absent) moves the config directory, the +// profiles file or the notification directory away from the defaults, or cannot be read: the archive +// layout then does not say where CrowdSec looks. +func configRedirectsPaths(config []byte) bool { + if len(config) == 0 { + return false + } + docs, err := ParseYAMLDocuments(bytes.NewReader(config), true) + if err != nil { + return true + } + if len(docs) == 0 { + return false + } + root := documentMapping(docs[0]) + for _, p := range []struct { + def string + keys []string + }{ + {defaultConfigDir, []string{"config_paths", "config_dir"}}, + {defaultNotificationDir, []string{"config_paths", "notification_dir"}}, + {defaultProfilesPath, []string{"api", "server", "profiles_path"}}, + } { + node, found := lookupPath(root, p.keys) + if !found { + continue + } + if node.Kind != yaml.ScalarNode || node.Tag != "!!str" || path.Clean(node.Value) != p.def { + return true + } + } + return false +} + +// lookupPath follows nested mapping keys. It reports false when a key is absent; a non-mapping on the +// way is returned as the found node so the caller treats it as unexpected. +func lookupPath(m *yaml.Node, keys []string) (*yaml.Node, bool) { + cur := m + for _, k := range keys { + if cur == nil || cur.Kind != yaml.MappingNode { + return cur, cur != nil + } + cur = mappingValue(cur, k) + if cur == nil { + return nil, false + } + } + return cur, true +} + +// referencedNotifications collects the names listed under notifications: in every profile document. +// It reports false when the file does not parse or a value is not a list of non-empty strings. +func referencedNotifications(profiles []byte) (map[string]bool, bool) { + docs, err := ParseYAMLDocuments(bytes.NewReader(profiles), false) + if err != nil { + return nil, false + } + names := map[string]bool{} + for _, doc := range docs { + v := mappingValue(documentMapping(doc), "notifications") + if v == nil || (v.Kind == yaml.ScalarNode && v.Tag == "!!null") { + continue + } + if v.Kind != yaml.SequenceNode { + return nil, false + } + for _, item := range v.Content { + item = resolveAlias(item) + if item.Kind != yaml.ScalarNode || item.Tag != "!!str" || item.Value == "" { + return nil, false + } + names[item.Value] = true + } + } + return names, true +} + +// definedNotifications collects name: from every document of every notification file CrowdSec loads +// from dir. It reports false when any such file does not parse. +func definedNotifications(files map[string][]byte, dir string) (map[string]bool, bool) { + defined := map[string]bool{} + for rel, data := range files { + if path.Dir(rel) != dir { + continue + } + docs, err := ParseYAMLDocuments(bytes.NewReader(data), false) + if err != nil { + return nil, false + } + for _, doc := range docs { + if n := mappingValue(documentMapping(doc), "name"); n != nil && n.Kind == yaml.ScalarNode && n.Tag == "!!str" { + defined[n.Value] = true + } + } + } + return defined, true +} + +func resolveAlias(n *yaml.Node) *yaml.Node { + for n != nil && n.Kind == yaml.AliasNode { + n = n.Alias + } + return n +} + +// documentMapping returns the top-level mapping of a document node, or nil. +func documentMapping(doc *yaml.Node) *yaml.Node { + if doc == nil || doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 { + return nil + } + if m := resolveAlias(doc.Content[0]); m.Kind == yaml.MappingNode { + return m + } + return nil +} + +// mappingValue returns the value node for key in a mapping (nil when absent or m is not a mapping). +func mappingValue(m *yaml.Node, key string) *yaml.Node { + if m == nil || m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return resolveAlias(m.Content[i+1]) + } + } + return nil +} diff --git a/backend/internal/crowdsec/notifications_check_test.go b/backend/internal/crowdsec/notifications_check_test.go new file mode 100644 index 000000000..62d3aaf3e --- /dev/null +++ b/backend/internal/crowdsec/notifications_check_test.go @@ -0,0 +1,274 @@ +package crowdsec + +import ( + "archive/tar" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + plainConfig = "api:\n server:\n listen_uri: 127.0.0.1:8080\n" + httpNotif = "type: http\nname: %s\nurl: https://example.invalid/\nmethod: POST\n" +) + +// profilesNaming returns a profiles stream with one document per entry; an entry is the raw text of +// the notifications value ("" omits the key). +func profilesNaming(values ...string) string { + docs := make([]string, len(values)) + for i, v := range values { + docs[i] = fmt.Sprintf("name: p%d\nfilters:\n - Alert.Remediation == true\ndecisions:\n - type: ban\n duration: 4h\n%s", i, v) + } + return strings.Join(docs, "---\n") +} + +func list(names ...string) string { + return "notifications:\n - " + strings.Join(names, "\n - ") + "\n" +} + +func notif(name string) string { return fmt.Sprintf(httpNotif, name) } + +func writeArchive(t *testing.T, files map[string]string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "cfg.tar.gz") + require.NoError(t, os.WriteFile(p, makeTarGz(t, files), 0o600)) + return p +} + +// TestCheckProfileNotificationsMatrix covers the behaviour verified against CrowdSec v1.8.1 with +// `crowdsec -t`: matching by the name field, every document of a file, .yaml and .yml only, +// no subdirectories, and the guards that skip the check instead of refusing. +func TestCheckProfileNotificationsMatrix(t *testing.T) { + t.Parallel() + big := strings.Repeat("# padding\n", 110_000) + notif("big") + cases := []struct { + name string + files map[string]string + want []string + }{ + {"defined by name field", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x")), "config/notifications/x.yaml": notif("x")}, nil}, + {"root layout", map[string]string{"config.yaml": plainConfig, "profiles.yaml": profilesNaming(list("x")), "notifications/x.yaml": notif("x")}, nil}, + {"root layout missing", map[string]string{"config.yaml": plainConfig, "profiles.yaml": profilesNaming(list("x"))}, []string{"x"}}, + {"config layout ignores root files", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x")), "notifications/x.yaml": notif("x")}, []string{"x"}}, + {"profiles in root with config layout is not used", map[string]string{"config/config.yaml": plainConfig, "profiles.yaml": profilesNaming(list("x"))}, nil}, + {"missing sorted", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("zeta", "alpha", "mid"))}, []string{"alpha", "mid", "zeta"}}, + {"file name does not match", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("a")), "config/notifications/a.yaml": notif("b")}, []string{"a"}}, + {"name field satisfies other name", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("b")), "config/notifications/a.yaml": notif("b")}, nil}, + {"multi document file defines both", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("m1", "m2")), "config/notifications/m.yaml": notif("m1") + "---\n" + notif("m2")}, nil}, + {"multi document file second name only", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("m2", "m3")), "config/notifications/m.yaml": notif("m1") + "---\n" + notif("m2")}, []string{"m3"}}, + {"yml loaded", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("y")), "config/notifications/y.yml": notif("y")}, nil}, + {"upper case extension not loaded", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("y")), "config/notifications/y.YAML": notif("y")}, []string{"y"}}, + {"other extension not loaded", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("y")), "config/notifications/y.yaml.bak": notif("y"), "config/notifications/y.txt": notif("y")}, []string{"y"}}, + {"extension only file not loaded", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("y")), "config/notifications/.yaml": notif("y")}, []string{"y"}}, + {"subdirectory not loaded", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("s")), "config/notifications/sub/s.yaml": notif("s")}, []string{"s"}}, + {"duplicate names across files", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("d")), "config/notifications/a.yaml": notif("d"), "config/notifications/b.yaml": notif("d")}, nil}, + {"duplicate reference counted once", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x", "x"), list("x"))}, []string{"x"}}, + {"second profile document", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("", list("zz"))}, []string{"zz"}}, + {"empty list", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("notifications: []\n")}, nil}, + {"null value", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("notifications:\n")}, nil}, + {"no notifications key", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("")}, nil}, + {"non-list value skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("notifications: x\n")}, nil}, + {"non-string item skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("notifications:\n - 12\n")}, nil}, + {"nested item skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("notifications:\n - [a]\n")}, nil}, + {"empty name skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("notifications:\n - \"\"\n")}, nil}, + {"no profiles file", map[string]string{"config/config.yaml": plainConfig, "config/notifications/x.yaml": notif("x")}, nil}, + {"malformed profiles skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": "name: [unclosed\n"}, nil}, + {"malformed notification skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x")), "config/notifications/bad.yaml": "name: x\n\ttype: [\n"}, nil}, + {"empty notification file", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x")), "config/notifications/e.yaml": ""}, []string{"x"}}, + {"notification without name", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x")), "config/notifications/n.yaml": "type: http\n"}, []string{"x"}}, + {"alias in profiles", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming("x-list: &l [a]\nnotifications: *l\n"), "config/notifications/a.yaml": notif("a")}, nil}, + {"redirected profiles path skips", map[string]string{"config/config.yaml": "api:\n server:\n profiles_path: /elsewhere/profiles.yaml\n", "config/profiles.yaml": profilesNaming(list("x"))}, nil}, + {"redirected notification dir skips", map[string]string{"config/config.yaml": "config_paths:\n notification_dir: /elsewhere/\n", "config/profiles.yaml": profilesNaming(list("x"))}, nil}, + {"redirected config dir skips", map[string]string{"config/config.yaml": "config_paths:\n config_dir: /elsewhere/\n", "config/profiles.yaml": profilesNaming(list("x"))}, nil}, + {"non-scalar path skips", map[string]string{"config/config.yaml": "config_paths:\n notification_dir: [a]\n", "config/profiles.yaml": profilesNaming(list("x"))}, nil}, + {"default paths still checked", map[string]string{"config/config.yaml": "config_paths:\n config_dir: /etc/crowdsec/\n notification_dir: /etc/crowdsec/notifications/\napi:\n server:\n profiles_path: /etc/crowdsec/profiles.yaml\n", "config/profiles.yaml": profilesNaming(list("x"))}, []string{"x"}}, + {"scalar section does not crash", map[string]string{"config/config.yaml": "config_paths: none\n", "config/profiles.yaml": profilesNaming(list("x"))}, nil}, + {"unparsable config skips", map[string]string{"config/config.yaml": "api: [unclosed\n", "config/profiles.yaml": profilesNaming(list("x"))}, nil}, + {"empty config", map[string]string{"config/config.yaml": "", "config/profiles.yaml": profilesNaming(list("x"))}, []string{"x"}}, + {"oversized notification skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("x")), "config/notifications/big.yaml": big}, nil}, + {"oversized profiles skips", map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": big + profilesNaming(list("x"))}, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + info, err := InspectArchive(writeArchive(t, tc.files)) + require.NoError(t, err) + require.Equal(t, tc.want, CheckProfileNotifications(info)) + }) + } +} + +func verifyArchiveNotifications(t *testing.T, path string) error { + t.Helper() + info, err := InspectArchive(path) + require.NoError(t, err) + return VerifyInspectedNotifications(info) +} + +func TestVerifyInspectedNotifications(t *testing.T) { + t.Parallel() + files := map[string]string{"config/config.yaml": plainConfig, "config/profiles.yaml": profilesNaming(list("b", "a"))} + err := verifyArchiveNotifications(t, writeArchive(t, files)) + var undefined *UndefinedNotificationsError + require.ErrorAs(t, err, &undefined) + require.Equal(t, []string{"a", "b"}, undefined.Names) + require.EqualError(t, err, "profiles.yaml names notifications that have no matching file: a, b") + + files["config/notifications/n.yaml"] = notif("a") + "---\n" + notif("b") + require.NoError(t, verifyArchiveNotifications(t, writeArchive(t, files))) +} + +func TestUndefinedNotificationsErrorMessage(t *testing.T) { + t.Parallel() + names := make([]string, 13) + for i := range names { + names[i] = fmt.Sprintf("n%02d", i) + } + require.Equal(t, + "profiles.yaml names notifications that have no matching file: n00, n01, n02, n03, n04, n05, n06, n07, n08, n09, and 3 more", + (&UndefinedNotificationsError{Names: names}).Error()) + require.Equal(t, + "profiles.yaml names notifications that have no matching file: n00, n01, n02, n03, n04, n05, n06, n07, n08, n09", + (&UndefinedNotificationsError{Names: names[:10]}).Error()) + + hostile := (&UndefinedNotificationsError{Names: []string{"a\nb\x00c", strings.Repeat("x", 100)}}).Error() + require.NotContains(t, hostile, "\n") + require.NotContains(t, hostile, "\x00") + require.Contains(t, hostile, strings.Repeat("x", 64)+"...") + require.NotContains(t, hostile, strings.Repeat("x", 65)) +} + +func TestInspectArchiveReadsOnlyNamedEntries(t *testing.T) { + t.Parallel() + raw := tarGzRaw(t, func(tw *tar.Writer) { + add := func(h *tar.Header, body string) { + h.Size = int64(len(body)) + if h.Typeflag == 0 { + h.Typeflag = tar.TypeReg + } + require.NoError(t, tw.WriteHeader(h)) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + add(&tar.Header{Name: "./config/config.yaml", Mode: 0o600}, "a: 1\n") + add(&tar.Header{Name: "x/../config/profiles.yaml", Mode: 0o600}, "first\n") + add(&tar.Header{Name: "config/profiles.yaml", Mode: 0o600}, "second\n") + add(&tar.Header{Name: "config/notifications/n.yml", Mode: 0o600}, "n\n") + add(&tar.Header{Name: "config/other.yaml", Mode: 0o600}, "other\n") + add(&tar.Header{Name: "config/notifications/link.yaml", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"}, "") + add(&tar.Header{Name: "../config/profiles.yaml", Mode: 0o600}, "escape\n") + add(&tar.Header{Name: ManifestEntryName, Mode: 0o600}, "{}") + add(&tar.Header{Name: "config/notifications/", Typeflag: tar.TypeDir, Mode: 0o700}, "") + }) + p := filepath.Join(t.TempDir(), "cfg.tar.gz") + require.NoError(t, os.WriteFile(p, raw, 0o600)) + info, err := InspectArchive(p) + require.NoError(t, err) + require.False(t, info.Incomplete) + require.Equal(t, map[string][]byte{ + "config/config.yaml": []byte("a: 1\n"), + "config/profiles.yaml": []byte("second\n"), + "config/notifications/n.yml": []byte("n\n"), + }, info.Files) +} + +func TestInspectArchiveBounds(t *testing.T) { + t.Parallel() + t.Run("too many notification files", func(t *testing.T) { + t.Parallel() + files := map[string]string{} + for i := 0; i <= maxInspectEntries; i++ { + files[fmt.Sprintf("notifications/n%03d.yaml", i)] = "name: x\n" + } + info, err := InspectArchive(writeArchive(t, files)) + require.NoError(t, err) + require.True(t, info.Incomplete) + require.Len(t, info.Files, maxInspectEntries) + }) + t.Run("total bytes", func(t *testing.T) { + t.Parallel() + files := map[string]string{} + body := strings.Repeat("#", int(maxInspectEntryBytes)-1) + for i := 0; i < 9; i++ { + files[fmt.Sprintf("notifications/n%d.yaml", i)] = body + } + info, err := InspectArchive(writeArchive(t, files)) + require.NoError(t, err) + require.True(t, info.Incomplete) + require.Len(t, info.Files, 8) + }) + t.Run("entry larger than its header claims", func(t *testing.T) { + t.Parallel() + info, err := InspectArchive(writeArchive(t, map[string]string{"profiles.yaml": strings.Repeat("#", int(maxInspectEntryBytes)+1)})) + require.NoError(t, err) + require.True(t, info.Incomplete) + require.Empty(t, info.Files) + }) +} + +func TestInspectArchiveErrors(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, err := InspectArchive(filepath.Join(dir, "absent")) + require.ErrorContains(t, err, "open archive") + require.NotContains(t, err.Error(), dir) + + notGzip := filepath.Join(dir, "plain") + require.NoError(t, os.WriteFile(notGzip, []byte("plain text"), 0o600)) + _, err = InspectArchive(notGzip) + require.ErrorContains(t, err, "gzip reader") + + raw := makeTarGz(t, map[string]string{"profiles.yaml": "name: p\n"}) + truncated := filepath.Join(dir, "cut.tar.gz") + require.NoError(t, os.WriteFile(truncated, raw[:len(raw)-12], 0o600)) + _, err = InspectArchive(truncated) + require.Error(t, err) +} + +func TestArchiveEntryRel(t *testing.T) { + t.Parallel() + cases := []struct { + in, want string + ok bool + }{ + {"config.yaml", "config.yaml", true}, + {"./config.yaml", "config.yaml", true}, + {"a/../config.yaml", "config.yaml", true}, + {"config//notifications/x.yaml", "config/notifications/x.yaml", true}, + {"dir/", "dir", true}, + {"/abs.yaml", "abs.yaml", true}, + {"../x", "", false}, + {"a/../../x", "", false}, + {".", "", false}, + {"", "", false}, + } + for _, tc := range cases { + got, ok := ArchiveEntryRel(tc.in) + require.Equal(t, tc.ok, ok, tc.in) + require.Equal(t, tc.want, got, tc.in) + } +} + +func TestParseYAMLDocuments(t *testing.T) { + t.Parallel() + docs, err := ParseYAMLDocuments(strings.NewReader(""), false) + require.NoError(t, err) + require.Empty(t, docs) + + docs, err = ParseYAMLDocuments(strings.NewReader("a: 1\n---\nb: 2\n"), false) + require.NoError(t, err) + require.Len(t, docs, 2) + + docs, err = ParseYAMLDocuments(strings.NewReader("a: 1\n---\nb: [\n"), true) + require.NoError(t, err) + require.Len(t, docs, 1) + + _, err = ParseYAMLDocuments(strings.NewReader("a: 1\n---\nb: [\n"), false) + require.ErrorContains(t, err, "decode yaml document") + _, err = ParseYAMLDocuments(strings.NewReader("a: 1\na: 2\n"), false) + require.ErrorContains(t, err, "decode yaml value") +} diff --git a/backend/internal/crowdsec/presets.go b/backend/internal/crowdsec/presets.go index e2634eac6..24db436c3 100644 --- a/backend/internal/crowdsec/presets.go +++ b/backend/internal/crowdsec/presets.go @@ -3,7 +3,6 @@ package crowdsec import ( "errors" "fmt" - "regexp" ) // ErrInvalidPresetDefinition is returned when a curated preset definition fails validation. @@ -24,7 +23,6 @@ var ( "scenarios": {}, "postoverflows": {}, } - presetItemNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*/[a-z0-9][a-z0-9_.-]*$`) ) // Preset represents a curated CrowdSec preset offered by Charon. @@ -51,7 +49,7 @@ func (p Preset) Validate() error { if _, ok := allowedPresetItemTypes[item.Type]; !ok { return fmt.Errorf("%w: unsupported item type %q", ErrInvalidPresetDefinition, item.Type) } - if !presetItemNamePattern.MatchString(item.Name) { + if !ValidPresetItemName(item.Name) { return fmt.Errorf("%w: invalid item name %q", ErrInvalidPresetDefinition, item.Name) } } diff --git a/backend/internal/crowdsec/yaml_documents.go b/backend/internal/crowdsec/yaml_documents.go new file mode 100644 index 000000000..ae0db3038 --- /dev/null +++ b/backend/internal/crowdsec/yaml_documents.go @@ -0,0 +1,36 @@ +package crowdsec + +import ( + "errors" + "fmt" + "io" + + "gopkg.in/yaml.v3" +) + +// ParseYAMLDocuments decodes the documents of a YAML stream, each first into a yaml.Node and then +// into a generic value so duplicate mapping keys and alias abuse are rejected. It returns the document +// nodes in order (none for an empty stream). With firstOnly set, later documents are never read; +// otherwise every document must parse. +func ParseYAMLDocuments(r io.Reader, firstOnly bool) ([]*yaml.Node, error) { + dec := yaml.NewDecoder(r) + var docs []*yaml.Node + for { + node := new(yaml.Node) + err := dec.Decode(node) + if errors.Is(err, io.EOF) { + return docs, nil + } + if err != nil { + return nil, fmt.Errorf("decode yaml document: %w", err) + } + var doc any + if err := node.Decode(&doc); err != nil { + return nil, fmt.Errorf("decode yaml value: %w", err) + } + docs = append(docs, node) + if firstOnly { + return docs, nil + } + } +} diff --git a/docs/features/crowdsec.md b/docs/features/crowdsec.md index 1010e9f40..b504a6299 100644 --- a/docs/features/crowdsec.md +++ b/docs/features/crowdsec.md @@ -67,6 +67,9 @@ Hub presets are applied on the server too, with the same safety net: before anyt - The config file editor only accepts common text and settings files (`.yaml`, `.yml`, `.json`, `.txt`, `.conf`) up to 1 MiB, and validates file type, size and YAML. - The config file editor does not list CrowdSec's account and connection credential files. They stay safely on disk; an operator can still reach them from the container shell if needed. - An exported CrowdSec configuration is meant for moving settings, not for sharing. CrowdSec's account and connection details are not included, but other saved settings (for example notification and database settings) still can be, so treat the file as private. When you import one, your running system keeps its own. Its installed hub items and runtime state stay as they are, so an import cannot remove or change them. An exported configuration no longer contains the downloaded hub data; CrowdSec downloads it again when needed. CrowdSec re-creates its local connection details on its own. If you move to a new host and use community blocklists or the CrowdSec Console, re-enable console enrollment there. +- An exported configuration also lists which hub items (collections, parsers, scenarios and so on) were installed on the system it came from. After an import, if some of those items are not installed on this system, the CrowdSec page offers an **Install missing hub items** button. It downloads and installs them from the official CrowdSec hub. This needs an internet connection and can take several minutes. If the hub cannot be reached, nothing is installed and you can simply try again later. If some items were not reached in time, an **Install remaining** button lets you retry just those. Items that are already installed are never removed or changed by an import. +- Before changing anything, an import checks that every notification name used by your alert profiles exists in the imported configuration. If any are missing, the import is refused and the message lists the missing names. +- The export and import messages are translated into your language. - The CrowdSec import accepts only `.tar.gz` archives and checks that the main configuration file is valid YAML before applying anything. If it isn't, Charon tells you what's wrong, for example `config.yaml is not valid YAML` or `only .tar.gz archives are supported`. - Backups are shown by name and are stored next to the CrowdSec data folder (for example `/app/data/`). Full snapshots are named `crowdsec.backup.`, single-file saves `crowdsec.filebackup.`. Saved acquisition settings are backed up as `acquis.yaml.backup.` right beside the acquisition file instead. To recover by hand, stop CrowdSec, copy the backup you want back into place, and start it again. diff --git a/docs/reports/qa_report.md b/docs/reports/qa_report.md index 9e77ac973..93dc6af7a 100644 --- a/docs/reports/qa_report.md +++ b/docs/reports/qa_report.md @@ -1,48 +1,83 @@ -# QA and Security Report: chore/repeatable-backend-tests +# QA and Security Report: feat/crowdsec-hub-items-restore -Date: 2026-10-10 -Scope: full diff `origin/development...HEAD` (HEAD 5d7b2f73, 5 commits, 35 files, tests/CI/config only) against `SECURITY.md`, `CLAUDE.md` (Definition of Done) and PR 1 of the local batch-3 plan (issue #1548). The branch adds `testutil.MemoryDSN`/`OpenMemoryDB`, moves the CrowdSec and handler tests onto them so `-count=N` and `-shuffle` work in one process, adds a guard test that forbids hand-written shared-cache DSNs in those two packages, and adds a weekly workflow that runs the repeat check. +Date: 2026-10-11 +Scope: full diff `origin/development...HEAD` (HEAD 96260b17, 7 commits, 47 files) against `SECURITY.md`, `CLAUDE.md` (Definition of Done) and the approved batch-3 plan (with the Supervisor amendments). The branch adds a hub item list to the CrowdSec export, an import-time check of notification references, a new admin endpoint `POST /api/v1/admin/crowdsec/hub-items/restore` that installs missing hub items, the matching frontend panel, translated export/import messages, E2E specs and docs. ## Verdict: PASS (no blocking findings) -No critical, high or medium findings. Four low or informational observations (F1-F4) are listed; none blocks merge. No application code changed (only `_test.go` files, `testutil/dsn.go`, `testdb.go` (a test helper), one workflow and `codecov.yml`), no model or migration file is in the diff, and no security-sensitive finding was found. +No critical, high or medium findings. Four low or informational observations (F1-F4) are listed; none blocks merge. No model, migration or GORM query file is in the diff, and no security-sensitive finding was found. One E2E test failed once on the first gate run because the local E2E container's 256 MB data tmpfs filled up (environment, see F1); the same test passes on a clean tmpfs. ## Definition of Done evidence | Check | Command | Result | |---|---|---| | Build and vet | `cd backend && go build ./... && go vet ./...` | pass, no output | -| golangci-lint, backend | `golangci-lint run --config .golangci.yml ./...` | 0 issues | +| golangci-lint, backend | `golangci-lint run --config .golangci.yml ./...` (v2.14.0) | 0 issues | | golangci-lint, agent | `cd agent && golangci-lint run --config ../backend/.golangci.yml ./...` | 0 issues | -| Lefthook | `lefthook run pre-commit --all-files` | all 17 hooks pass, including actionlint, semgrep (367 rules, 1924 files, 0 findings), frontend type-check and lint. Working tree stayed clean (only the three untracked plan files). | +| Frontend | `cd frontend && npm run type-check && npm run build` | pass | +| i18n keys | `vitest run src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts` | 6 passed; the `crowdsecConfig` key set (185 keys, including the new `hubItems` group) is identical in en, de, es, fr and zh | +| Lefthook | `lefthook run pre-commit --all-files` | all 17 hooks pass (795 s), including actionlint, shellcheck, semgrep (367 rules, 1946 files, 0 findings), frontend type-check and lint; the working tree stayed clean (only the three untracked plan files) | | govulncheck | skill `security-scan-go-vuln` | 0 affected; 1 module-level advisory in a required module that Charon code does not call | | Trivy | `trivy fs --scanners vuln,misconfig --severity HIGH,CRITICAL` on go.mod/go.sum (backend, agent) and the frontend package files, copied under `$HOME` (the snap trivy cannot read `/projects`) and removed afterwards | no HIGH/CRITICAL | -| CodeQL | `lefthook run codeql` (CLI 2.27.2, already current) | Go and JS scans completed; JS 0 results; findings gate "0 blocking"; "All CodeQL checks passed"; parity check passed | -| GORM scan | not run | no `backend/internal/models/**`, migration or GORM query files in the diff (0 matches) | -| E2E | not run | no application code changed; the E2E image does not need a rebuild | -| Coverage (given, not rerun) | prior runs | backend 91.3% stmt / 90.3% line, frontend 91.49% lines (no frontend change), patch coverage 90.0% (the 2 uncovered lines are in the codecov-ignored `dsn.go`) | -| codecov.yml | `curl -s -m 30 --data-binary @codecov.yml https://codecov.io/validate` | `Valid!`; only `backend/internal/testutil/dsn.go` was added to `ignore` (1 line); `scripts/ci/check-codecov-trigger-parity.sh` passes | +| CodeQL | `lefthook run codeql` (CLI 2.27.2, current) | Go: 4 results, all 4 suppressed pre-existing entries in `auth_handler.go`, `remote_server_handler.go`, `uptime_service.go` (none in a file of this diff); JS: 0 results; "0 blocking"; "All CodeQL checks passed"; parity check passed. None of the earlier log-injection findings in the restore code remain. | +| GORM scan | not run | no `backend/internal/models/**`, migration or GORM query file in the diff (0 matches) | +| Coverage (given, not rerun) | prior runs | backend 91.4% stmt / 90.4% line, frontend 91.52% lines, patch coverage 94.8% | +| Patch report artifacts | `test-results/local-patch-report.{md,json}` | produced by the earlier run; the directory is cleared by Playwright's output folder, so the files were gone after the E2E gate and were not regenerated (instruction: do not rerun coverage scripts) | -## Repeatability proof (run during this audit) +## Race and repeat runs | Run | Result | |---|---| -| `go test -race -count=2 -shuffle=on -timeout=30m ./internal/crowdsec/...` | ok, 213 s | -| `go test -race -count=2 -timeout=40m -run 'TestAuthHandler\|TestSettingsHandler\|TestCertificateHandler_Export_PasswordGuard\|TestDeleteCertificate' ./internal/api/handlers/` | ok, 23 s | -| `go test -race -count=2 -run 'TestConsoleEnroll\|TestHeartbeatPoller_\|TestCertificateHandler_Export_PasswordGuard\|TestAuthHandler_GetAccessibleHosts_PermittedHosts' ./internal/crowdsec/ ./internal/api/handlers/` (the tests named on #1548) | ok, ok | -| `go test -count=1 ./internal/testutil/` | ok | -| Guard red/green: temporary `internal/crowdsec/zz_guard_probe_test.go` containing a shared-cache literal | guard failed and named `internal/crowdsec/zz_guard_probe_test.go:3`; probe file removed | -| Full handlers repetition | not rerun (14+ min); recorded in commit 1.3 as green: `-count=2` 829 s, `-count=3` 1268 s, `-shuffle=on -count=2` 840 s | - -## Check-by-check results - -1. **The DSN sweep did not change test semantics.** The 24 handler test files in the diff change only DSN construction: every non-DSN changed line was filtered out (the only other changes are `fmt` import removals and the `testutil` import additions). No `t.Skip`/`Skipf`/`Skip(` was added, no `func Test*` or `func Benchmark*` was removed, and no assertion changed. `git diff --numstat` shows every handler file is a 1:1 or 2:1 line swap. Cases checked by hand: the previously process-wide `file::memory:?cache=shared` sites (db_health tests, `pr_coverage_test.go`, `permission_helpers_test.go`, `system_permissions_handler_test.go`, `security_handler_settings_test.go` which had a hard-coded test name) now get one unique name per test, which is stored in a variable and reused where the old code shared it (for example `db_health_handler_test.go` passes the same DSN to `database.Connect` and to `NewDBHealthHandler`, so `CheckIntegrityDedicated` still opens the same named shared-cache database). No function calls `MemoryDSN` twice where the old code used one string twice (scripted check over all changed files: none). The unrelated-connection design of the dedicated-corruption test is kept (separate `MemoryDSN` for the unrelated handler connection, an on-disk file for the corrupt one). Tests that once relied on a database leaking from an earlier test would now fail, and none does. The benchmark file now uses `MemoryDSN(b, ...)`, whose per-call counter keeps every benchmark iteration set unique. -2. **`testutil.MemoryDSN` / `OpenMemoryDB`.** Uniqueness comes from a process-wide `atomic.Uint64` counter combined with the sanitized test name, so it holds across `-count`, `-shuffle`, parallel tests and subtests (tests: same-name subtests, two calls in one test). Sanitization replaces every run of non-`[A-Za-z0-9_]` characters (slashes, spaces, unicode, punctuation) with one underscore, so the name is always a valid SQLite file-URI path; a very long name only makes a long in-memory name (no filesystem is touched). Params are appended with `&` after the fixed `?mode=memory&cache=shared`, so the `?`/`&` ordering is right. It takes `testing.TB`, with a benchmark test. `OpenMemoryDB` closes the pool in `t.Cleanup` (tested: ping fails after the subtest). `handlers.OpenTestDB` keeps `_journal_mode=WAL` and `_busy_timeout=5000`, the silent logger and the close-on-cleanup; only the name suffix changed (counter instead of time plus random), so its 231 callers are unaffected. See F2 for a nit. -3. **Guard test.** It finds the module root by walking up from its own source file via `runtime.Caller` to the first `go.mod`, and fails loudly if a guarded directory is missing. It walks guarded packages recursively over `*_test.go` only and reports `path:line`; its own reporting is covered by a unit test and the red/green probe above. The `"cache=" + "shared"` split keeps the marker out of the guard's own file (the guard lives in `testutil`, which is not a guarded package anyway). The `#nosec G304 G122` comment is adequate: the path comes from a walk of a fixed in-repo directory in a test, with no external input. See F3 for the false-negative scope. -4. **Workflow `.github/workflows/backend-repeatability.yml`.** `actionlint` clean. `actions/checkout@3d3c42e5...` (v7) and `actions/setup-go@b7ad1dad...` (v7.0.0) match the SHAs and versions used in `quality-checks.yml` and the rest of the repo. Permissions: `contents: read` only; `persist-credentials: false`; no secrets; triggers are `schedule` and `workflow_dispatch` only (no `pull_request`, so nothing runs from forks). `concurrency` group per workflow and ref with `cancel-in-progress: false`. Cron `0 5 * * 4` (Thursday 05:00 UTC) collides with no other cron in the repo (checked all `cron:` lines). The run step has no interpolated untrusted input: the seed is `GITHUB_RUN_ID % 2147483646 + 1` (always 1..2147483646, so a valid `-shuffle` value), echoed to the log and the step summary so a failure is replayable, and `TMPDIR` is the runner temp directory. Timeouts: `go test -timeout=60m` inside a 75-minute job; the measured work is about 4 min (CrowdSec) plus about 14 min (handlers) with `-count=2` under `-race`, so there is wide headroom. It is not a required check. The plan suggested `-timeout=30m` and `timeout-minutes: 40`; the larger values are sensible given measured times (see F1 for a note on default-branch-only scheduling). -5. **Plan alignment.** Commit order and subjects match plan commits 1.1-1.5. The plan's "fixes for non-DSN shared state" in 1.3 turned out to be unneeded: the DSN sweep alone made every named test repeatable, which the runs above confirm. -6. **Commits.** `git log origin/development..HEAD --format=%s`: `test: add a unique in-memory database helper for backend tests`, `test: make the CrowdSec tests repeatable in one process`, `test: make the handler tests repeatable in one process`, `test: guard against shared database names in the repeatable packages`, `ci: run the backend repeatability check weekly`. All generic, none uses the `(security)` scope, no session ID or link. -7. **Security-sensitive items.** None. Nothing in the diff touches production code paths, secrets, tokens or CI permissions beyond read-only. +| `go test -race -count=2 -run 'HubItems\|Restore\|Manifest\|Inspect\|Notification\|Export\|Import\|Keep\|Rollback' ./internal/crowdsec/... ./internal/api/handlers/...` | ok (crowdsec 5.7 s, handlers 64.6 s) | +| Full package, `go test -race -count=1 -timeout=40m ./internal/api/handlers/` | ok, 386.6 s, no failure. The earlier unrecorded one-off failure in this package was not reproduced; no test name is available to chase. | + +## E2E gate (single project, once, on a freshly rebuilt `charon-e2e`) + +`npx playwright test tests/security/crowdsec-hub-items.spec.ts tests/security/crowdsec-import.spec.ts tests/security/crowdsec-config.spec.ts tests/security/crowdsec-diagnostics.spec.ts tests/security/crowdsec-file-editor.spec.ts tests/security-enforcement/zzz-security-ui/crowdsec-import.spec.ts --project=security-tests` + +Result: 149 passed, 1 failed, no fixme remaining. The failure was `crowdsec-import.spec.ts:132 Rejected Archives > should reject an archive larger than 50 MB`, failing in its `beforeEach` baseline import with 500. Container log: `no space left on device` on the backup write; `/app/data` (256 MB tmpfs) was at 91%, holding about 155 MB of retained backups (each about 39 MB, because the data dir holds about 73 MB of CrowdSec data) and the data dir itself. After removing the backups and `crowdsec.backup.*` leftovers inside the throwaway container, the whole "Rejected Archives" group (5 tests, including that one) passed. `changelog.json` was restored with `git checkout`. See F1. + +## Security review + +### 1. Restore endpoint + +Verified live against the rebuilt `charon-e2e` container (validation paths only; no install was run, the hub was unreachable there): + +| Probe | Result | +|---|---| +| No credentials | 401 | +| Authenticated user with role `user` (created for the probe) | 403 `Forbidden` | +| Admin, each invalid item: unknown type, `a/b..c`, leading `-` on owner and on name, NUL, trailing newline, embedded newline, Cyrillic look-alike character, `=`, space, two slashes, 129-character name, empty list, malformed JSON | 400 with the fixed messages `invalid hub item`, `no hub items given`, `invalid request` | +| 1,001 items | 400 `invalid hub item` | +| Body over 1 MiB | 413 `request too large` | +| Unknown JSON field alongside a valid item | ignored (valid item proceeds) | + +- Route sits in `managementAdmin` (`RequireRole(admin)`), the same group as the sibling CrowdSec routes; a route test pins the 401. +- Argv is built as the literal `cscli`, then ``, `install`, ``, through the command executor (no shell). Name pattern `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}$` plus a `..` ban and a five-value type allowlist; Go RE2 `$` matches only at the end of text, so a trailing newline cannot slip through (probe above confirms). No validated string starts with `-` or contains `=`, whitespace or a second slash, so cobra cannot read it as a flag or extra argument; names beginning with a digit or containing dots are plain positional arguments. cscli v1.8.1 in the image was confirmed to be the version in use. +- Response bodies carry only fixed reason codes (`not_found`, `install_failed`, `verify_failed`, `not_attempted`) and fixed messages; cscli output and paths go to the log only. All log fields in the new code (`type`, `name`, `error`, `output`) pass through `hubLogDetail` (`sanitizeForLog` plus a 300-byte cap); the remaining log lines use fixed strings or filesystem errors that contain no request data. CodeQL confirms no log-injection result. +- Concurrency: a compare-and-swap single-flight flag (never waited on) returns 409 to a second call; each item runs under `dataMu` for one cscli section with a context of min(remaining overall time, 120 s); the overall budget is 5 minutes; the hub update has 60 s; the reload uses a detached 10 s context so an install that succeeded just before a disconnect is still reloaded. +- Lock order `restore.running -> dataMu -> HubService.mu`: all seven `dataMu` users (import, export, file write, preset apply, and the three restore sections) take it non-nested and never call another `dataMu` taker while holding it; `installedHubSet` is the only restore helper that locks and is called without the lock held; the reload runs after every lock is released. No re-entrancy or inversion was found, and the full race run passed. +- Resource use: an admin can start at most one restore at a time (409 otherwise), at most 1,000 items per call, at most 5 minutes per call, and other `dataMu` users wait for at most one 120 s section. The 1 MiB body cap is applied before the flag is taken; slow body reads are a general server property (see F4). + +### 2. Manifest path + +- `ArchiveEntryRel` is now the single normalization used by both the extractor and `InspectArchive`, so `./charon-hub-items.json`, `a/../charon-hub-items.json` and `charon-hub-items.json` are the same entry for both; a nested `x/charon-hub-items.json` is an ordinary file. The extractor skips the manifest path regardless of the entry type, so it is never written to disk (E2E "never writes the manifest to disk" and the `./` spelling spec pass). +- Bounds: manifest at most 1 MiB, a duplicate entry ignores the whole manifest, malformed JSON, unknown version and more than 1,000 items ignore it, an individually invalid item is dropped and counted (`skipped_invalid`, names not echoed). `InspectArchive` keeps at most 1 MiB per entry, 8 MiB in total and 256 files, and sets `Incomplete` instead of failing. +- Export side: the manifest is derived from link targets only; the target is cleaned and parsed as `hub//[stage/]/.yaml`, and the name is re-validated with the same function the restore uses. The test table covers relative links, links elsewhere, wrong type, missing stage, too deep, odd extension, invalid name, overlong owner and a dangling target. A file with the reserved name inside the data dir is skipped on export. + +### 3. Notification check + +- The check runs after the archive validator and before the lock, so a refusal changes nothing (E2E and handler tests assert the live files are untouched). It skips instead of failing when the inspection is incomplete, there is no profiles file, the config redirects the paths, a profile or notification file does not parse, or a notifications value is not a list of names. +- The 422 message lists at most 10 names, each sanitized and cut at 64 characters, plus "and N more". +- YAML alias abuse: a temporary probe fed alias-expansion documents of 5 to 40 nested levels (up to 2.1 KB) through `CheckProfileNotifications` in both profiles.yaml and a notification file. Each finished in 1-2 ms with under 400 KB allocated in total (the parser rejects the excessive aliasing before expansion, and the check then skips). The probe file was removed; the working tree is clean. + +### 4. Frontend panel + +- No `dangerouslySetInnerHTML` in the panel or the page; item type and name are rendered as React text; error messages are fixed i18n strings chosen from the HTTP status (502, 409, other); the result area is a `role="status"` `aria-live="polite"` region and the section carries `aria-busy`; the 330 s timeout applies only to the restore call; the panel is passive (no banner, no toast; only the existing import success toast). The import failure toast shows the server message (including the 422 notification text) through the existing response interceptor. + +### 5. Commits and docs + +- Subjects: `docs: describe the CrowdSec hub item list and the import check`, `feat: offer to install missing hub items after a CrowdSec import`, `feat: add an action to install missing CrowdSec hub items`, `feat: record the installed hub items in a CrowdSec export`, `fix: validate notification references in a CrowdSec import`, `fix: translate the CrowdSec export and import messages`, `test(e2e): add fixme specs for the CrowdSec import checks and hub item restore`. All generic; none uses the `(security)` scope; no session ID or link in any message. The docs and `ARCHITECTURE.md` additions describe behavior only, with no file names, process IDs or secret details. ## Findings @@ -50,12 +85,18 @@ No critical, high or medium findings. ### Low -- **F1: low (CI behavior).** `.github/workflows/backend-repeatability.yml`: GitHub runs `schedule` triggers only from the default branch, so after merge into `development` the weekly job first exists on `main` after the next promotion, and until then only `workflow_dispatch` on the branch works. Not a defect; mention in the PR body, and run one manual dispatch once the file is on a branch GitHub knows about. +- **F1: low (test environment, not product).** `tests/security/crowdsec-import.spec.ts:132` (and any import spec late in a full gate run): the local E2E container's `/app/data` is a 256 MB tmpfs, and each import and manual backup writes a roughly 39 MB zip (the data dir holds about 73 MB of CrowdSec data). The gate filled the tmpfs once during the CrowdSec config/import specs and one `beforeEach` import returned 500 (`no space left on device`). Evidence: container log lines `checksum writer: write /app/data/backups/...: no space left on device` and `df` showing 91% used with 4 retained backups. Suggested approach: raise the tmpfs size in the E2E compose file, or have the import/config specs delete the backups they create. Already known behavior ("fills up if the specs run twice"), but it now happens on a single run, so CI on a fresh container should be watched (CI uses its own container; a failure there would look the same). ### Informational -- **F2: info (nit).** `backend/internal/testutil/dsn.go` `MemoryDSN`: a test name that sanitizes to an empty string cannot happen (names always start with `Test`/`Benchmark`/`Example`/`Fuzz`), so no change is needed; if the helper is ever used from `TestMain` or an unnamed context, `t.Name()` would be empty and the DSN `file:_N` would still be unique and valid. -- **F3: info (guard scope).** `backend/internal/testutil/dsn_guard_test.go`: the guard only matches the literal `cache=shared` on a single source line, so a DSN assembled across lines or with a different spelling (for example `file::memory:` without `cache=shared`, or the marker built by concatenation like the guard's own constant) is not detected, and only `internal/crowdsec` and `internal/api/handlers` are covered (192 other `cache=shared` test lines exist in other packages that are not yet clean). Acceptable as a cheap tripwire. Optional: add other packages to `dsnGuardPackages` as they are cleaned up. -- **F4: info (follow-up, already planned).** Packages outside the two guarded ones still use name-derived or fixed shared-cache DSNs; extending the repeatability check to them is future work, not part of this PR. +- **F2: info.** `backend/internal/api/handlers/crowdsec_hub_items.go` `hubItemNotFound`: classifies `not_found` by matching "can't find" or "not found" (case-insensitive) in cscli output. It is best effort by design; a different cscli wording only turns `not_found` into `install_failed`, with the same safe response. Nothing to change unless cscli is upgraded; the wording was verified against the real hub in the manual run. +- **F3: info.** `backend/internal/api/handlers/crowdsec_hub_items.go` `installHubItem` / `hubUpdate`: `dataMu.Lock()` is not context-aware, so a restore whose client has left still holds the single-flight flag until it gets the lock and notices the cancelled context (at most one other `dataMu` section, itself bounded). No action needed; mention only for operators who see a 409 right after a cancelled restore. +- **F4: info.** The restore body read (1 MiB cap) has no per-request read deadline of its own; slow-client behavior is whatever the HTTP server configuration provides, as for every other admin endpoint. Admin-only, so not a new exposure. + +## Known limitations (not re-opened) + +Flat archives (#1559), the image ships no notification plugins (#1551 comment), profiles file hazard (#1554), restore edge cases (#1556). + +## Manual evidence (given, on throwaway containers at HEAD bcda295d, not repeated here) -Out-of-scope findings to file: none that need an issue beyond F3/F4 (low priority, optional). +Real-hub restore of 2 real items and 1 bogus (200, bogus reported as `not_found`); export manifest with 61 items; cross-host import reporting the missing items; restore on the second host, then a second restore reporting everything as already installed; a parallel restore refused with 409; import and export waiting during a 34 s ten-collection restore (export about 10.9 s, import about 6.5 s); a real 422 on a missing notification and 200 once defined; an offline restart that is non-fatal. diff --git a/docs/troubleshooting/crowdsec.md b/docs/troubleshooting/crowdsec.md index 5eb44dd40..6b77ca9c0 100644 --- a/docs/troubleshooting/crowdsec.md +++ b/docs/troubleshooting/crowdsec.md @@ -193,6 +193,8 @@ charon-local-machine 127.0.0.1 password v1.x.x - Hub unreachable (503): retry once, then Charon falls back to cached Hub data if available; otherwise stay on curated/offline presets until connectivity returns. - Hub returns HTML/redirect: set HUB_BASE_URL to the JSON endpoint above or install cscli so the index is fetched locally. - Bad preset slug (400): the slug must match Hub naming; correct the slug before retrying. +- **"Install missing hub items" says the hub could not be reached**: nothing was installed and nothing was changed. Check that the container has internet access and can resolve names (DNS), then click the button again. Items that were not reached in time can be retried with **Install remaining**. +- **Import refused because a profile names notifications that have no matching file**: your imported alert profiles refer to notification names that are not in the archive. Add the missing notification files to the configuration, or remove those names from the profile, then import again. Nothing was changed by the refused import. - Apply failed: review the apply response and restore from the backup that was taken automatically, then retry after fixing the underlying issue. - Apply not supported (501): use curated/offline presets; Hub apply will be re-enabled when supported in your environment. - **Security Engine Offline**: If your dashboard says "Offline", it means CrowdSec LAPI is not running. diff --git a/frontend/src/api/__tests__/crowdsec.test.ts b/frontend/src/api/__tests__/crowdsec.test.ts index 70de8ba49..847c4a5a7 100644 --- a/frontend/src/api/__tests__/crowdsec.test.ts +++ b/frontend/src/api/__tests__/crowdsec.test.ts @@ -46,6 +46,27 @@ describe('crowdsec API', () => { }) }) + describe('restoreHubItems', () => { + it('posts the items with a 330 s timeout override and returns the result', async () => { + const result = { installed: [], already_installed: [], failed: [], remaining: 0, skipped_invalid: 0, reloaded: false } + vi.mocked(client.post).mockResolvedValue({ data: result }) + const items = [{ type: 'collections', name: 'crowdsecurity/caddy' }] + + await expect(crowdsec.restoreHubItems(items)).resolves.toEqual(result) + + expect(client.post).toHaveBeenCalledWith( + '/admin/crowdsec/hub-items/restore', + { items }, + { timeout: 330_000 }, + ) + }) + + it('propagates request errors', async () => { + vi.mocked(client.post).mockRejectedValue(new Error('boom')) + await expect(crowdsec.restoreHubItems([])).rejects.toThrow('boom') + }) + }) + describe('importCrowdsecConfig', () => { it('should call POST /admin/crowdsec/import with FormData', async () => { const mockFile = new File(['content'], 'config.tar.gz', { type: 'application/gzip' }) diff --git a/frontend/src/api/crowdsec.ts b/frontend/src/api/crowdsec.ts index 77b1b425e..f8b47ef71 100644 --- a/frontend/src/api/crowdsec.ts +++ b/frontend/src/api/crowdsec.ts @@ -47,21 +47,77 @@ export async function statusCrowdsec(): Promise { return resp.data } +/** A CrowdSec hub item identified by its type and name. */ +export interface HubItem { + type: string + name: string +} + +/** Why a hub item was not installed by a restore. */ +export type HubItemFailureReason = 'not_found' | 'install_failed' | 'verify_failed' | 'not_attempted' + +/** A hub item the restore could not install. */ +export interface HubItemFailure extends HubItem { + reason: HubItemFailureReason +} + +/** Hub item summary returned by an import of an export that lists hub items. */ +export interface ImportHubItems { + in_export: number + skipped_invalid: number + missing: HubItem[] +} + +/** Result of a CrowdSec configuration import. */ +export interface ImportCrowdsecResult { + status?: string + backup?: string + hub_items?: ImportHubItems +} + +/** Result of a hub item restore. */ +export interface RestoreHubItemsResult { + installed: HubItem[] + already_installed: HubItem[] + failed: HubItemFailure[] + remaining: number + skipped_invalid: number + reloaded: boolean +} + +/** A restore downloads items one by one; the server budget is 5 minutes. */ +const RESTORE_HUB_ITEMS_TIMEOUT_MS = 330_000 + /** * Imports a CrowdSec configuration file. * @param file - The configuration file to import * @returns Promise resolving to import result * @throws {AxiosError} If import fails or file is invalid */ -export async function importCrowdsecConfig(file: File) { +export async function importCrowdsecConfig(file: File): Promise { const fd = new FormData() fd.append('file', file) - const resp = await client.post('/admin/crowdsec/import', fd, { + const resp = await client.post('/admin/crowdsec/import', fd, { headers: { 'Content-Type': 'multipart/form-data' }, }) return resp.data } +/** + * Installs the given hub items from the official CrowdSec hub. + * @param items - The hub items to install + * @returns Promise resolving to the per-item outcome + * @throws {AxiosError} 409 if a restore is running, 502 if the hub is unreachable + */ +export async function restoreHubItems(items: HubItem[]): Promise { + const resp = await client.post( + '/admin/crowdsec/hub-items/restore', + { items }, + { timeout: RESTORE_HUB_ITEMS_TIMEOUT_MS }, + ) + return resp.data +} + /** * Exports the current CrowdSec configuration. * @returns Promise resolving to configuration blob for download @@ -183,4 +239,4 @@ export const deleteWhitelist = async (uuid: string): Promise => { await client.delete(`/admin/crowdsec/whitelist/${uuid}`) } -export default { startCrowdsec, stopCrowdsec, statusCrowdsec, importCrowdsecConfig, exportCrowdsecConfig, listCrowdsecFiles, readCrowdsecFile, writeCrowdsecFile, listCrowdsecDecisions, banIP, unbanIP, getCrowdsecKeyStatus, listWhitelists, addWhitelist, deleteWhitelist } +export default { startCrowdsec, stopCrowdsec, statusCrowdsec, importCrowdsecConfig, restoreHubItems, exportCrowdsecConfig, listCrowdsecFiles, readCrowdsecFile, writeCrowdsecFile, listCrowdsecDecisions, banIP, unbanIP, getCrowdsecKeyStatus, listWhitelists, addWhitelist, deleteWhitelist } diff --git a/frontend/src/components/crowdsec/HubItemsRestorePanel.tsx b/frontend/src/components/crowdsec/HubItemsRestorePanel.tsx new file mode 100644 index 000000000..13b26e4a0 --- /dev/null +++ b/frontend/src/components/crowdsec/HubItemsRestorePanel.tsx @@ -0,0 +1,129 @@ +import { isAxiosError } from 'axios' +import { Loader2 } from 'lucide-react' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' + +import type { HubItem, ImportHubItems, RestoreHubItemsResult } from '../../api/crowdsec' +import { useRestoreHubItems } from '../../hooks/useCrowdsecHubItems' +import { Button } from '../ui/Button' + +interface HubItemsRestorePanelProps { + hubItems: ImportHubItems +} + +const toItem = ({ type, name }: HubItem): HubItem => ({ type, name }) + +function ItemList({ items, label }: { items: HubItem[]; label: string }) { + if (items.length === 0) return null + return ( +
+
{label}
+
    + {items.map((item) => ( +
  • + {item.type} + {item.name} +
    • + ))} +
+
+ ) +} + +/** + * Passive follow-up to an import: lists the hub items the export had that this + * host lacks and lets the user install them (or the remainder of a long run). + */ +export default function HubItemsRestorePanel({ hubItems }: HubItemsRestorePanelProps) { + const { t } = useTranslation() + const restore = useRestoreHubItems() + // Kept across runs so a failed "Install remaining" does not hide the earlier result. + const [result, setResult] = useState(null) + const run = (items: HubItem[]) => restore.mutate(items, { onSuccess: setResult }) + + const status = restore.error && isAxiosError(restore.error) ? restore.error.response?.status : undefined + let errorMessage: string | null = null + if (restore.isError) { + if (status === 502) errorMessage = t('crowdsecConfig.hubItems.errors.hubUnreachable') + else if (status === 409) errorMessage = t('crowdsecConfig.hubItems.errors.alreadyRunning') + else errorMessage = t('crowdsecConfig.hubItems.errors.generic') + } + + const notAttempted = (result?.failed ?? []).filter((f) => f.reason === 'not_attempted').map(toItem) + const failed = result?.failed ?? [] + + return ( +
+
+

{t('crowdsecConfig.hubItems.title')}

+ {!result &&

{t('crowdsecConfig.hubItems.description')}

} + {hubItems.skipped_invalid > 0 && ( +

{t('crowdsecConfig.hubItems.skippedInvalid', { count: hubItems.skipped_invalid })}

+ )} +
+ + {!result && } + +
+ {restore.isPending && ( +

+

+ )} + {errorMessage &&

{errorMessage}

} + {result && ( + <> +

+ {t('crowdsecConfig.hubItems.summary', { + installed: result.installed.length, + alreadyInstalled: result.already_installed.length, + failed: failed.length, + })} +

+ {result.remaining > 0 && ( +

{t('crowdsecConfig.hubItems.remaining', { count: result.remaining })}

+ )} + + )} +
+ + {result && ( +
+ + + {failed.length > 0 && ( +
+
{t('crowdsecConfig.hubItems.failedLabel')}
+
    + {failed.map((item) => ( +
  • + {item.type} + {item.name} + {t(`crowdsecConfig.hubItems.reasons.${item.reason}`, { defaultValue: t('crowdsecConfig.hubItems.reasons.install_failed') })} +
    • + ))} +
+
+ )} +
+ )} + + {!result && ( + + )} + {result && result.remaining > 0 && notAttempted.length > 0 && ( + + )} +
+ ) +} diff --git a/frontend/src/components/crowdsec/__tests__/HubItemsRestorePanel.test.tsx b/frontend/src/components/crowdsec/__tests__/HubItemsRestorePanel.test.tsx new file mode 100644 index 000000000..fc47afc7b --- /dev/null +++ b/frontend/src/components/crowdsec/__tests__/HubItemsRestorePanel.test.tsx @@ -0,0 +1,171 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { AxiosError } from 'axios' +import { describe, it, expect, vi, beforeEach } from 'vitest' + +import * as api from '../../../api/crowdsec' +import HubItemsRestorePanel from '../HubItemsRestorePanel' + +vi.mock('../../../api/crowdsec') + +const MISSING: api.HubItem[] = [ + { type: 'collections', name: 'crowdsecurity/caddy' }, + { type: 'parsers', name: 'crowdsecurity/sshd-logs' }, + { type: 'scenarios', name: 'crowdsecurity/ssh-bf' }, +] + +const result = (over: Partial = {}): api.RestoreHubItemsResult => ({ + installed: [], + already_installed: [], + failed: [], + remaining: 0, + skipped_invalid: 0, + reloaded: false, + ...over, +}) + +const httpError = (status: number) => + new AxiosError('failed', undefined, undefined, undefined, { + status, + statusText: String(status), + headers: {}, + config: {}, + data: { error: 'x' }, + } as never) + +const renderPanel = (skipped = 0, missing = MISSING) => + render( + + + , + ) + +const installButton = () => screen.getByRole('button', { name: 'Install missing hub items' }) + +describe('HubItemsRestorePanel', () => { + beforeEach(() => vi.clearAllMocks()) + + it('lists the missing items with an enabled install button', () => { + renderPanel() + for (const item of MISSING) { + expect(screen.getByText(item.name, { exact: true })).toBeInTheDocument() + } + expect(installButton()).toBeEnabled() + expect(screen.queryByText(/could not be used/i)).not.toBeInTheDocument() + }) + + it('shows a neutral note when entries were skipped', () => { + renderPanel(2) + expect(screen.getByText('2 entries in the export could not be used.')).toBeInTheDocument() + }) + + it('disables the button and announces progress while installing', async () => { + let release: (value: api.RestoreHubItemsResult) => void = () => undefined + vi.mocked(api.restoreHubItems).mockReturnValue(new Promise((resolve) => { release = resolve })) + renderPanel() + + await userEvent.click(installButton()) + + expect(installButton()).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Installing hub items') + expect(screen.getByTestId('hub-items-panel')).toHaveAttribute('aria-busy', 'true') + expect(api.restoreHubItems).toHaveBeenCalledWith(MISSING) + + release(result({ installed: MISSING })) + await waitFor(() => expect(screen.queryByRole('button', { name: 'Install missing hub items' })).not.toBeInTheDocument()) + }) + + it('reports a full success without further buttons', async () => { + vi.mocked(api.restoreHubItems).mockResolvedValue(result({ installed: MISSING.slice(0, 2), already_installed: [MISSING[2]] })) + renderPanel() + + await userEvent.click(installButton()) + + expect(await screen.findByText('2 installed, 1 already installed, 0 failed.')).toBeInTheDocument() + expect(screen.getByText(MISSING[0].name, { exact: true })).toBeInTheDocument() + expect(screen.getByText(MISSING[2].name, { exact: true })).toBeInTheDocument() + expect(screen.queryByRole('button')).not.toBeInTheDocument() + }) + + it('shows translated failure reasons and resends only not_attempted entries', async () => { + vi.mocked(api.restoreHubItems) + .mockResolvedValueOnce( + result({ + installed: [MISSING[0]], + failed: [ + { ...MISSING[1], reason: 'install_failed' }, + { ...MISSING[2], reason: 'not_attempted' }, + ], + remaining: 1, + }), + ) + .mockResolvedValueOnce(result({ installed: [MISSING[2]] })) + renderPanel() + + await userEvent.click(installButton()) + + expect(await screen.findByText('Installation failed')).toBeInTheDocument() + expect(screen.getByText('Not attempted yet')).toBeInTheDocument() + expect(screen.getByText('1 items were not attempted before the time limit.')).toBeInTheDocument() + expect(screen.getByText(MISSING[1].name, { exact: true })).toBeVisible() + expect(screen.queryByRole('button', { name: 'Install missing hub items' })).not.toBeInTheDocument() + + await userEvent.click(screen.getByRole('button', { name: 'Install remaining' })) + + await waitFor(() => expect(api.restoreHubItems).toHaveBeenCalledTimes(2)) + expect(vi.mocked(api.restoreHubItems).mock.calls[1][0]).toEqual([MISSING[2]]) + await waitFor(() => expect(screen.queryByRole('button', { name: 'Install remaining' })).not.toBeInTheDocument()) + }) + + it('keeps the earlier result when a retry fails', async () => { + vi.mocked(api.restoreHubItems) + .mockResolvedValueOnce(result({ failed: [{ ...MISSING[2], reason: 'not_attempted' }], remaining: 1 })) + .mockRejectedValueOnce(httpError(502)) + renderPanel() + + await userEvent.click(installButton()) + await userEvent.click(await screen.findByRole('button', { name: 'Install remaining' })) + + expect(await screen.findByText(/hub could not be reached/i)).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Install remaining' })).toBeEnabled() + expect(screen.getByText('Not attempted yet')).toBeInTheDocument() + }) + + it('does not offer a retry when remaining entries are reported without not_attempted rows', async () => { + vi.mocked(api.restoreHubItems).mockResolvedValue( + result({ failed: [{ ...MISSING[1], reason: 'verify_failed' }, { ...MISSING[2], reason: 'not_found' }], remaining: 0 }), + ) + renderPanel() + + await userEvent.click(installButton()) + + expect(await screen.findByText('Installed but failed verification')).toBeInTheDocument() + expect(screen.getByText('Not found in the hub')).toBeInTheDocument() + expect(screen.queryByRole('button')).not.toBeInTheDocument() + }) + + it.each([ + [502, /hub could not be reached.*nothing was installed/i], + [409, /restore is already running/i], + [500, /could not be installed/i], + ])('shows a clear message for HTTP %i and keeps the button enabled', async (status, message) => { + vi.mocked(api.restoreHubItems).mockRejectedValue(httpError(status)) + renderPanel() + + await userEvent.click(installButton()) + + expect(await screen.findByText(message)).toBeInTheDocument() + expect(installButton()).toBeEnabled() + expect(screen.getByText(MISSING[0].name, { exact: true })).toBeInTheDocument() + }) + + it('shows the generic message for a non-HTTP error', async () => { + vi.mocked(api.restoreHubItems).mockRejectedValue(new Error('network down')) + renderPanel() + + await userEvent.click(installButton()) + + expect(await screen.findByText(/could not be installed/i)).toBeInTheDocument() + }) +}) diff --git a/frontend/src/hooks/__tests__/useCrowdsecHubItems.test.tsx b/frontend/src/hooks/__tests__/useCrowdsecHubItems.test.tsx new file mode 100644 index 000000000..9a377e79c --- /dev/null +++ b/frontend/src/hooks/__tests__/useCrowdsecHubItems.test.tsx @@ -0,0 +1,47 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { renderHook, waitFor, act } from '@testing-library/react' +import type { ReactNode } from 'react' +import { describe, it, expect, vi, beforeEach } from 'vitest' + +import * as api from '../../api/crowdsec' +import { useRestoreHubItems } from '../useCrowdsecHubItems' + +vi.mock('../../api/crowdsec') + +const result = { installed: [], already_installed: [], failed: [], remaining: 0, skipped_invalid: 0, reloaded: false } + +const setup = () => { + const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } }) + const invalidate = vi.spyOn(queryClient, 'invalidateQueries') + const wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ) + return { invalidate, ...renderHook(() => useRestoreHubItems(), { wrapper }) } +} + +describe('useRestoreHubItems', () => { + beforeEach(() => vi.clearAllMocks()) + + it('calls the API and refreshes the file list on success', async () => { + vi.mocked(api.restoreHubItems).mockResolvedValue(result) + const { result: hook, invalidate } = setup() + const items = [{ type: 'collections', name: 'crowdsecurity/caddy' }] + + await act(async () => { + await hook.current.mutateAsync(items) + }) + + expect(api.restoreHubItems).toHaveBeenCalledWith(items) + expect(invalidate).toHaveBeenCalledWith({ queryKey: ['crowdsec-files'] }) + }) + + it('exposes the error without invalidating', async () => { + vi.mocked(api.restoreHubItems).mockRejectedValue(new Error('boom')) + const { result: hook, invalidate } = setup() + + act(() => hook.current.mutate([])) + + await waitFor(() => expect(hook.current.isError).toBe(true)) + expect(invalidate).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/hooks/useCrowdsecHubItems.ts b/frontend/src/hooks/useCrowdsecHubItems.ts new file mode 100644 index 000000000..2dd1f3ef8 --- /dev/null +++ b/frontend/src/hooks/useCrowdsecHubItems.ts @@ -0,0 +1,17 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' + +import { restoreHubItems, type HubItem } from '../api/crowdsec' + +/** + * Installs missing hub items. Errors are surfaced inline by the caller + * (no toast: the restore is an optional follow-up to an import). + */ +export const useRestoreHubItems = () => { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (items: HubItem[]) => restoreHubItems(items), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['crowdsec-files'] }) + }, + }) +} diff --git a/frontend/src/locales/de/translation.json b/frontend/src/locales/de/translation.json index f7af1c5e6..19efa84f6 100644 --- a/frontend/src/locales/de/translation.json +++ b/frontend/src/locales/de/translation.json @@ -1156,6 +1156,36 @@ } }, "crowdsecConfig": { + "exportSuccess": "CrowdSec-Konfiguration exportiert", + "exportFailed": "Export der CrowdSec-Konfiguration fehlgeschlagen", + "hubItems": { + "title": "Fehlende Hub-Elemente installieren", + "description": "Dieser Export verwendete Hub-Elemente, die auf diesem Host nicht installiert sind. Sie werden vom offiziellen CrowdSec-Hub heruntergeladen.", + "missingLabel": "Fehlende Elemente", + "install": "Fehlende Hub-Elemente installieren", + "installRemaining": "Verbleibende installieren", + "installing": "Hub-Elemente werden installiert. Das kann einige Minuten dauern.", + "summary": "{{installed}} installiert, {{alreadyInstalled}} bereits installiert, {{failed}} fehlgeschlagen.", + "remaining": "{{count}} Elemente wurden vor Ablauf des Zeitlimits nicht bearbeitet.", + "installedLabel": "Installiert", + "alreadyInstalledLabel": "Bereits installiert", + "failedLabel": "Nicht installiert", + "skippedInvalid": "{{count}} Einträge im Export konnten nicht verwendet werden.", + "reasons": { + "not_found": "Im Hub nicht gefunden", + "install_failed": "Installation fehlgeschlagen", + "verify_failed": "Installiert, aber Überprüfung fehlgeschlagen", + "not_attempted": "Noch nicht versucht" + }, + "errors": { + "hubUnreachable": "Der CrowdSec-Hub war nicht erreichbar. Es wurde nichts installiert. Prüfen Sie die Internetverbindung und versuchen Sie es erneut.", + "alreadyRunning": "Eine Wiederherstellung läuft bereits. Versuchen Sie es erneut, wenn sie abgeschlossen ist.", + "generic": "Die Hub-Elemente konnten nicht installiert werden. Versuchen Sie es erneut." + } + }, + "importSuccess": "CrowdSec-Konfiguration importiert (Sicherung erstellt)", + "importFailed": "Import fehlgeschlagen", + "exportPrompt": "Name für das CrowdSec-Exportarchiv", "title": "CrowdSec-Konfiguration", "checkNow": "Jetzt prüfen", "loading": "Lade CrowdSec-Konfiguration...", diff --git a/frontend/src/locales/en/translation.json b/frontend/src/locales/en/translation.json index 930974029..ca4cabb00 100644 --- a/frontend/src/locales/en/translation.json +++ b/frontend/src/locales/en/translation.json @@ -1477,6 +1477,36 @@ } }, "crowdsecConfig": { + "exportSuccess": "CrowdSec configuration exported", + "exportFailed": "Failed to export CrowdSec configuration", + "hubItems": { + "title": "Install missing hub items", + "description": "This export used hub items that are not installed on this host. They are downloaded from the official CrowdSec hub.", + "missingLabel": "Missing items", + "install": "Install missing hub items", + "installRemaining": "Install remaining", + "installing": "Installing hub items. This can take several minutes.", + "summary": "{{installed}} installed, {{alreadyInstalled}} already installed, {{failed}} failed.", + "remaining": "{{count}} items were not attempted before the time limit.", + "installedLabel": "Installed", + "alreadyInstalledLabel": "Already installed", + "failedLabel": "Not installed", + "skippedInvalid": "{{count}} entries in the export could not be used.", + "reasons": { + "not_found": "Not found in the hub", + "install_failed": "Installation failed", + "verify_failed": "Installed but failed verification", + "not_attempted": "Not attempted yet" + }, + "errors": { + "hubUnreachable": "The CrowdSec hub could not be reached. Nothing was installed. Check the internet connection and try again.", + "alreadyRunning": "A restore is already running. Try again when it has finished.", + "generic": "The hub items could not be installed. Try again." + } + }, + "importSuccess": "CrowdSec config imported (backup created)", + "importFailed": "Failed to import", + "exportPrompt": "Name your CrowdSec export archive", "title": "CrowdSec Configuration", "checkNow": "Check Now", "loading": "Loading CrowdSec configuration...", diff --git a/frontend/src/locales/es/translation.json b/frontend/src/locales/es/translation.json index 3c1d875f8..cf73d3d45 100644 --- a/frontend/src/locales/es/translation.json +++ b/frontend/src/locales/es/translation.json @@ -1158,6 +1158,36 @@ } }, "crowdsecConfig": { + "exportSuccess": "Configuración de CrowdSec exportada", + "exportFailed": "No se pudo exportar la configuración de CrowdSec", + "hubItems": { + "title": "Instalar elementos del hub que faltan", + "description": "Esta exportación usaba elementos del hub que no están instalados en este host. Se descargan del hub oficial de CrowdSec.", + "missingLabel": "Elementos que faltan", + "install": "Instalar elementos del hub que faltan", + "installRemaining": "Instalar los restantes", + "installing": "Instalando elementos del hub. Esto puede tardar varios minutos.", + "summary": "{{installed}} instalados, {{alreadyInstalled}} ya instalados, {{failed}} con error.", + "remaining": "{{count}} elementos no se intentaron antes del límite de tiempo.", + "installedLabel": "Instalados", + "alreadyInstalledLabel": "Ya instalados", + "failedLabel": "No instalados", + "skippedInvalid": "{{count}} entradas de la exportación no se pudieron usar.", + "reasons": { + "not_found": "No encontrado en el hub", + "install_failed": "Falló la instalación", + "verify_failed": "Instalado, pero falló la verificación", + "not_attempted": "Aún no intentado" + }, + "errors": { + "hubUnreachable": "No se pudo acceder al hub de CrowdSec. No se instaló nada. Compruebe la conexión a Internet e inténtelo de nuevo.", + "alreadyRunning": "Ya hay una restauración en curso. Inténtelo de nuevo cuando termine.", + "generic": "No se pudieron instalar los elementos del hub. Inténtelo de nuevo." + } + }, + "importSuccess": "Configuración de CrowdSec importada (copia de seguridad creada)", + "importFailed": "Error al importar", + "exportPrompt": "Nombre del archivo de exportación de CrowdSec", "title": "Configuración de CrowdSec", "checkNow": "Comprobar ahora", "loading": "Cargando configuración de CrowdSec...", diff --git a/frontend/src/locales/fr/translation.json b/frontend/src/locales/fr/translation.json index 228b7734c..558592087 100644 --- a/frontend/src/locales/fr/translation.json +++ b/frontend/src/locales/fr/translation.json @@ -1158,6 +1158,36 @@ } }, "crowdsecConfig": { + "exportSuccess": "Configuration CrowdSec exportée", + "exportFailed": "Échec de l'exportation de la configuration CrowdSec", + "hubItems": { + "title": "Installer les éléments du hub manquants", + "description": "Cet export utilisait des éléments du hub qui ne sont pas installés sur cet hôte. Ils sont téléchargés depuis le hub officiel de CrowdSec.", + "missingLabel": "Éléments manquants", + "install": "Installer les éléments du hub manquants", + "installRemaining": "Installer les restants", + "installing": "Installation des éléments du hub. Cela peut prendre plusieurs minutes.", + "summary": "{{installed}} installés, {{alreadyInstalled}} déjà installés, {{failed}} en échec.", + "remaining": "{{count}} éléments n'ont pas été traités avant la limite de temps.", + "installedLabel": "Installés", + "alreadyInstalledLabel": "Déjà installés", + "failedLabel": "Non installés", + "skippedInvalid": "{{count}} entrées de l'export n'ont pas pu être utilisées.", + "reasons": { + "not_found": "Introuvable dans le hub", + "install_failed": "Échec de l'installation", + "verify_failed": "Installé, mais la vérification a échoué", + "not_attempted": "Pas encore tenté" + }, + "errors": { + "hubUnreachable": "Le hub CrowdSec est injoignable. Rien n'a été installé. Vérifiez la connexion Internet et réessayez.", + "alreadyRunning": "Une restauration est déjà en cours. Réessayez lorsqu'elle sera terminée.", + "generic": "Les éléments du hub n'ont pas pu être installés. Réessayez." + } + }, + "importSuccess": "Configuration CrowdSec importée (sauvegarde créée)", + "importFailed": "Échec de l'importation", + "exportPrompt": "Nom de l'archive d'exportation CrowdSec", "title": "Configuration CrowdSec", "checkNow": "Vérifier maintenant", "loading": "Chargement de la configuration CrowdSec...", diff --git a/frontend/src/locales/zh/translation.json b/frontend/src/locales/zh/translation.json index 7811968ab..5b46c2bf9 100644 --- a/frontend/src/locales/zh/translation.json +++ b/frontend/src/locales/zh/translation.json @@ -1157,6 +1157,36 @@ } }, "crowdsecConfig": { + "exportSuccess": "CrowdSec 配置已导出", + "exportFailed": "导出 CrowdSec 配置失败", + "hubItems": { + "title": "安装缺失的 Hub 项目", + "description": "此导出使用了本主机上未安装的 Hub 项目。它们将从 CrowdSec 官方 Hub 下载。", + "missingLabel": "缺失的项目", + "install": "安装缺失的 Hub 项目", + "installRemaining": "安装剩余项目", + "installing": "正在安装 Hub 项目，可能需要几分钟。", + "summary": "已安装 {{installed}} 个，已存在 {{alreadyInstalled}} 个，失败 {{failed}} 个。", + "remaining": "有 {{count}} 个项目在时间限制内未处理。", + "installedLabel": "已安装", + "alreadyInstalledLabel": "已存在", + "failedLabel": "未安装", + "skippedInvalid": "导出中有 {{count}} 个条目无法使用。", + "reasons": { + "not_found": "在 Hub 中未找到", + "install_failed": "安装失败", + "verify_failed": "已安装，但验证失败", + "not_attempted": "尚未尝试" + }, + "errors": { + "hubUnreachable": "无法连接到 CrowdSec Hub，未安装任何内容。请检查网络连接后重试。", + "alreadyRunning": "已有恢复任务正在运行，请在其完成后重试。", + "generic": "无法安装 Hub 项目，请重试。" + } + }, + "importSuccess": "CrowdSec 配置已导入（已创建备份）", + "importFailed": "导入失败", + "exportPrompt": "为 CrowdSec 导出文件命名", "title": "CrowdSec 配置", "checkNow": "立即检查", "loading": "正在加载 CrowdSec 配置...", diff --git a/frontend/src/pages/CrowdSecConfig.tsx b/frontend/src/pages/CrowdSecConfig.tsx index ea359871b..bbf04619d 100644 --- a/frontend/src/pages/CrowdSecConfig.tsx +++ b/frontend/src/pages/CrowdSecConfig.tsx @@ -6,12 +6,13 @@ import { useTranslation } from 'react-i18next' import { useNavigate, Link } from 'react-router' import { createBackup } from '../api/backups' -import { exportCrowdsecConfig, importCrowdsecConfig, listCrowdsecFiles, readCrowdsecFile, writeCrowdsecFile, listCrowdsecDecisions, banIP, unbanIP, type CrowdSecDecision, type CrowdSecWhitelistEntry, statusCrowdsec, type CrowdSecStatus, startCrowdsec } from '../api/crowdsec' +import { exportCrowdsecConfig, importCrowdsecConfig, listCrowdsecFiles, readCrowdsecFile, writeCrowdsecFile, listCrowdsecDecisions, banIP, unbanIP, type CrowdSecDecision, type CrowdSecWhitelistEntry, statusCrowdsec, type CrowdSecStatus, startCrowdsec, type ImportHubItems } from '../api/crowdsec' import { getFeatureFlags } from '../api/featureFlags' import { listCrowdsecPresets, pullCrowdsecPreset, applyCrowdsecPreset, getCrowdsecPresetCache } from '../api/presets' import { getSecurityStatus } from '../api/security' import { getMyIP } from '../api/system' import { CrowdSecBouncerKeyDisplay } from '../components/CrowdSecBouncerKeyDisplay' +import HubItemsRestorePanel from '../components/crowdsec/HubItemsRestorePanel' import { ConfigReloadOverlay } from '../components/LoadingStates' import { Button } from '../components/ui/Button' import { Card } from '../components/ui/Card' @@ -42,6 +43,8 @@ export default function CrowdSecConfig() { const [searchQuery, setSearchQuery] = useState('') const [sortBy, setSortBy] = useState<'alpha' | 'type' | 'source'>('alpha') const [file, setFile] = useState(null) + // Hub items the last import reported as missing; the key restarts the panel per import. + const [importHubItems, setImportHubItems] = useState<{ key: number; items: ImportHubItems } | null>(null) const [selectedPath, setSelectedPath] = useState(null) const [fileContent, setFileContent] = useState(null) const [selectedPresetSlug, setSelectedPresetSlug] = useState('') @@ -119,12 +122,14 @@ export default function CrowdSecConfig() { mutationFn: async (file: File) => { return await importCrowdsecConfig(file) }, - onSuccess: () => { - toast.success('CrowdSec config imported (backup created)') + onSuccess: (result) => { + toast.success(t('crowdsecConfig.importSuccess')) + const items = result.hub_items + setImportHubItems(items && items.missing.length > 0 ? { key: Date.now(), items } : null) queryClient.invalidateQueries({ queryKey: ['security-status'] }) }, onError: (err: unknown) => { - toast.error(err instanceof Error ? err.message : 'Failed to import') + toast.error(err instanceof Error ? err.message : t('crowdsecConfig.importFailed')) } }) @@ -419,15 +424,15 @@ export default function CrowdSecConfig() { const handleExport = async () => { const defaultName = buildCrowdsecExportFilename() - const filename = promptCrowdsecFilename(defaultName) + const filename = promptCrowdsecFilename(t, defaultName) if (!filename) return try { const blob = await exportCrowdsecConfig() downloadCrowdsecExport(blob, filename) - toast.success('CrowdSec configuration exported') + toast.success(t('crowdsecConfig.exportSuccess')) } catch { - toast.error('Failed to export CrowdSec configuration') + toast.error(t('crowdsecConfig.exportFailed')) } } @@ -974,11 +979,15 @@ export default function CrowdSecConfig() {

{t('crowdsecConfig.packages.description')}

setFile(e.target.files?.[0] ?? null)} + onChange={(e) => { + setFile(e.target.files?.[0] ?? null) + setImportHubItems(null) + }} data-testid="import-file" accept=".tar.gz" aria-label={t('crowdsecConfig.packages.selectFile')} /> + {importHubItems && } diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx index 1eedbc17f..3d39e5cfc 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx +++ b/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx @@ -110,7 +110,7 @@ describe('CrowdSecConfig coverage', () => { vi.mocked(crowdsecApi.banIP).mockResolvedValue(undefined) vi.mocked(crowdsecApi.unbanIP).mockResolvedValue(undefined) vi.mocked(crowdsecApi.exportCrowdsecConfig).mockResolvedValue(new Blob(['data'])) - vi.mocked(crowdsecApi.importCrowdsecConfig).mockResolvedValue(undefined) + vi.mocked(crowdsecApi.importCrowdsecConfig).mockResolvedValue({}) vi.mocked(presetsApi.listCrowdsecPresets).mockResolvedValue({ presets: [ { @@ -220,6 +220,7 @@ describe('CrowdSecConfig coverage', () => { await renderPage() await userEvent.click(screen.getByRole('button', { name: 'Export' })) await waitFor(() => expect(crowdsecApi.exportCrowdsecConfig).toHaveBeenCalled()) + expect(exportUtils.promptCrowdsecFilename).toHaveBeenCalledWith(expect.any(Function), expect.any(String)) expect(exportUtils.downloadCrowdsecExport).toHaveBeenCalled() expect(toast.success).toHaveBeenCalledWith('CrowdSec configuration exported') diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx new file mode 100644 index 000000000..0f99141c7 --- /dev/null +++ b/frontend/src/pages/__tests__/CrowdSecConfig.hubItems.test.tsx @@ -0,0 +1,75 @@ +import { screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { describe, it, expect, vi, beforeEach } from 'vitest' + +import * as backupsApi from '../../api/backups' +import * as crowdsecApi from '../../api/crowdsec' +import * as featureFlagsApi from '../../api/featureFlags' +import * as presetsApi from '../../api/presets' +import * as securityApi from '../../api/security' +import { renderWithQueryClient } from '../../test-utils/renderWithQueryClient' +import CrowdSecConfig from '../CrowdSecConfig' + +vi.mock('../../api/security') +vi.mock('../../api/crowdsec') +vi.mock('../../api/presets') +vi.mock('../../api/backups') +vi.mock('../../api/featureFlags') +vi.mock('../../components/CrowdSecBouncerKeyDisplay', () => ({ CrowdSecBouncerKeyDisplay: () => null })) +vi.mock('../../utils/toast', () => ({ toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() } })) + +const MISSING = [{ type: 'collections', name: 'crowdsecurity/caddy' }] + +const importWith = async (response: crowdsecApi.ImportCrowdsecResult) => { + vi.mocked(crowdsecApi.importCrowdsecConfig).mockResolvedValue(response) + renderWithQueryClient() + await screen.findByText('CrowdSec Configuration') + await userEvent.upload(screen.getByTestId('import-file'), new File(['data'], 'cfg.tar.gz')) + await userEvent.click(screen.getByTestId('import-btn')) + await waitFor(() => expect(crowdsecApi.importCrowdsecConfig).toHaveBeenCalled()) +} + +describe('CrowdSecConfig hub items panel', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(securityApi.getSecurityStatus).mockResolvedValue({ + cerberus: { enabled: true }, + crowdsec: { enabled: true, mode: 'local', api_url: '' }, + waf: { enabled: true, mode: 'enabled' }, + rate_limit: { enabled: true }, + acl: { enabled: true }, + }) + vi.mocked(crowdsecApi.statusCrowdsec).mockResolvedValue({ running: true, pid: 1, lapi_ready: true }) + vi.mocked(crowdsecApi.listCrowdsecFiles).mockResolvedValue({ files: [] }) + vi.mocked(crowdsecApi.listCrowdsecDecisions).mockResolvedValue({ decisions: [] }) + vi.mocked(presetsApi.listCrowdsecPresets).mockResolvedValue({ presets: [] }) + vi.mocked(backupsApi.createBackup).mockResolvedValue({ job_id: 'j', type: 'create', status: 'pending' }) + vi.mocked(featureFlagsApi.getFeatureFlags).mockResolvedValue({ 'feature.crowdsec.console_enrollment': false }) + }) + + it('offers the install action after an import that reports missing items', async () => { + await importWith({ status: 'imported', hub_items: { in_export: 2, skipped_invalid: 0, missing: MISSING } }) + + expect(await screen.findByRole('button', { name: 'Install missing hub items' })).toBeEnabled() + expect(screen.getByText('crowdsecurity/caddy', { exact: true })).toBeInTheDocument() + }) + + it.each([ + ['no hub_items field', { status: 'imported' }], + ['nothing missing', { status: 'imported', hub_items: { in_export: 2, skipped_invalid: 0, missing: [] } }], + ])('shows no panel with %s', async (_label, response) => { + await importWith(response) + + await waitFor(() => expect(crowdsecApi.importCrowdsecConfig).toHaveBeenCalled()) + expect(screen.queryByTestId('hub-items-panel')).not.toBeInTheDocument() + }) + + it('clears the panel when another file is selected', async () => { + await importWith({ status: 'imported', hub_items: { in_export: 2, skipped_invalid: 0, missing: MISSING } }) + await screen.findByTestId('hub-items-panel') + + await userEvent.upload(screen.getByTestId('import-file'), new File(['other'], 'other.tar.gz')) + + expect(screen.queryByTestId('hub-items-panel')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts b/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts index 9484188cf..c759d80fc 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts +++ b/frontend/src/pages/__tests__/CrowdSecConfig.i18nKeys.test.ts @@ -5,6 +5,8 @@ type Json = { [key: string]: Json | string } const LOCALES = ['de', 'en', 'es', 'fr', 'zh'] as const const sourceModules = { ...import.meta.glob('../CrowdSecConfig.tsx', { query: '?raw', import: 'default', eager: true }), + ...import.meta.glob('../../utils/crowdsecExport.ts', { query: '?raw', import: 'default', eager: true }), + ...import.meta.glob('../../components/crowdsec/HubItemsRestorePanel.tsx', { query: '?raw', import: 'default', eager: true }), ...import.meta.glob('../../components/crowdsec/DashboardTimeRangeSelector.tsx', { query: '?raw', import: 'default', eager: true }), } as Record const localeModules = import.meta.glob('../../locales/*/translation.json', { import: 'default', eager: true }) as Record diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.whitelist.test.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.whitelist.test.tsx index 98e7a105e..f0c6dd063 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.whitelist.test.tsx +++ b/frontend/src/pages/__tests__/CrowdSecConfig.whitelist.test.tsx @@ -117,7 +117,7 @@ describe('CrowdSecConfig – whitelist tab', () => { vi.mocked(crowdsecApi.banIP).mockResolvedValue(undefined) vi.mocked(crowdsecApi.unbanIP).mockResolvedValue(undefined) vi.mocked(crowdsecApi.exportCrowdsecConfig).mockResolvedValue(new Blob(['data'])) - vi.mocked(crowdsecApi.importCrowdsecConfig).mockResolvedValue(undefined) + vi.mocked(crowdsecApi.importCrowdsecConfig).mockResolvedValue({}) vi.mocked(crowdsecApi.listWhitelists).mockResolvedValue([]) vi.mocked(crowdsecApi.addWhitelist).mockResolvedValue({ uuid: 'uuid-new', diff --git a/frontend/src/pages/__tests__/ImportCrowdSec.spec.tsx b/frontend/src/pages/__tests__/ImportCrowdSec.spec.tsx index 1611ac015..a59b743a7 100644 --- a/frontend/src/pages/__tests__/ImportCrowdSec.spec.tsx +++ b/frontend/src/pages/__tests__/ImportCrowdSec.spec.tsx @@ -31,7 +31,7 @@ describe('ImportCrowdSec page', () => { it('creates a backup then imports crowdsec', async () => { const file = new File(['fake'], 'crowdsec.tar.gz', { type: 'application/gzip' }) vi.mocked(backups.createBackup).mockResolvedValue({ job_id: 'job-1', type: 'create', status: 'pending' }) - vi.mocked(api.importCrowdsecConfig).mockResolvedValue({ success: true }) + vi.mocked(api.importCrowdsecConfig).mockResolvedValue({ status: 'imported' }) renderWithProviders() const fileInput = document.querySelector('input[type="file"]') diff --git a/frontend/src/utils/__tests__/crowdsecExport.test.ts b/frontend/src/utils/__tests__/crowdsecExport.test.ts index ddb4f70cc..b31b1ee4e 100644 --- a/frontend/src/utils/__tests__/crowdsecExport.test.ts +++ b/frontend/src/utils/__tests__/crowdsecExport.test.ts @@ -1,3 +1,4 @@ +import type { TFunction } from 'i18next' import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' import { @@ -6,6 +7,8 @@ import { downloadCrowdsecExport, } from '../crowdsecExport' +const t = vi.fn((key: string) => `translated:${key}`) as unknown as TFunction + describe('crowdsecExport', () => { describe('buildCrowdsecExportFilename', () => { it('should generate filename with ISO timestamp', () => { @@ -54,94 +57,101 @@ describe('crowdsecExport', () => { vi.unstubAllGlobals() }) + it('should ask the translator for the prompt text', () => { + vi.mocked(window.prompt).mockReturnValue('x') + promptCrowdsecFilename(t, 'default.tar.gz') + expect(t).toHaveBeenCalledWith('crowdsecConfig.exportPrompt') + expect(window.prompt).toHaveBeenCalledWith('translated:crowdsecConfig.exportPrompt', 'default.tar.gz') + }) + it('should return null when user cancels', () => { vi.mocked(window.prompt).mockReturnValue(null) - const result = promptCrowdsecFilename('default.tar.gz') + const result = promptCrowdsecFilename(t, 'default.tar.gz') expect(result).toBeNull() }) it('should return default filename when user provides empty string', () => { vi.mocked(window.prompt).mockReturnValue(' ') - const result = promptCrowdsecFilename('default.tar.gz') + const result = promptCrowdsecFilename(t, 'default.tar.gz') expect(result).toBe('default.tar.gz') }) it('should sanitize user input by replacing slashes', () => { vi.mocked(window.prompt).mockReturnValue('backup/prod/config') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup-prod-config.tar.gz') }) it('should replace spaces with hyphens', () => { vi.mocked(window.prompt).mockReturnValue('crowdsec backup 2025') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('crowdsec-backup-2025.tar.gz') }) it('should append .tar.gz if missing', () => { vi.mocked(window.prompt).mockReturnValue('my-backup') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('my-backup.tar.gz') }) it('should not double-append .tar.gz', () => { vi.mocked(window.prompt).mockReturnValue('my-backup.tar.gz') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('my-backup.tar.gz') }) it('should handle case-insensitive .tar.gz extension', () => { vi.mocked(window.prompt).mockReturnValue('my-backup.TAR.GZ') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) // Implementation checks lowercase, so uppercase extension gets .tar.gz appended expect(result).toBe('my-backup.TAR.GZ') }) it('should trim whitespace from user input', () => { vi.mocked(window.prompt).mockReturnValue(' my-backup ') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('my-backup.tar.gz') }) it('should use generated default when no default provided', () => { vi.mocked(window.prompt).mockReturnValue('') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toMatch(/^crowdsec-export-.*\.tar\.gz$/) }) it('should handle backslashes (Windows paths)', () => { vi.mocked(window.prompt).mockReturnValue('backup\\windows\\path') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup-windows-path.tar.gz') }) it('should handle multiple consecutive slashes', () => { vi.mocked(window.prompt).mockReturnValue('backup///prod') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup-prod.tar.gz') }) it('should handle multiple consecutive spaces', () => { vi.mocked(window.prompt).mockReturnValue('backup prod') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup-prod.tar.gz') }) it('should handle mixed slashes and spaces', () => { vi.mocked(window.prompt).mockReturnValue('backup / prod \\ test') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup---prod---test.tar.gz') }) it('should handle special characters', () => { vi.mocked(window.prompt).mockReturnValue('backup@prod#2025') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup@prod#2025.tar.gz') }) it('should handle undefined return from prompt', () => { vi.mocked(window.prompt).mockReturnValue(undefined as unknown as string | null) - const result = promptCrowdsecFilename('default.tar.gz') + const result = promptCrowdsecFilename(t, 'default.tar.gz') expect(result).toBeNull() }) }) @@ -303,49 +313,49 @@ describe('crowdsecExport', () => { it('should sanitize directory traversal attempts', () => { vi.mocked(window.prompt).mockReturnValue('../../etc/passwd') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('..-..-etc-passwd.tar.gz') expect(result).not.toContain('../') }) it('should handle absolute paths', () => { vi.mocked(window.prompt).mockReturnValue('/etc/crowdsec/backup') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('-etc-crowdsec-backup.tar.gz') expect(result).not.toMatch(/^\//) }) it('should handle Windows absolute paths', () => { vi.mocked(window.prompt).mockReturnValue('C:\\Windows\\System32\\backup') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) // Backslashes are replaced with hyphens, but case is preserved expect(result).toBe('C:-Windows-System32-backup.tar.gz') }) it('should handle null bytes', () => { vi.mocked(window.prompt).mockReturnValue('backup\0malicious') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toContain('backup') // Null bytes should be handled by sanitization }) it('should handle mixed attack vectors', () => { vi.mocked(window.prompt).mockReturnValue('../../../etc/passwd\0.tar.gz') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).not.toContain('../') expect(result!.endsWith('.tar.gz')).toBe(true) }) it('should handle URL-encoded path traversal', () => { vi.mocked(window.prompt).mockReturnValue('%2e%2e%2f%2e%2e%2f') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toContain('%2e') expect(result!.endsWith('.tar.gz')).toBe(true) }) it('should not allow overwriting system files via filename', () => { vi.mocked(window.prompt).mockReturnValue('/etc/passwd') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) // After sanitization, leading slash should be replaced expect(result).toBe('-etc-passwd.tar.gz') }) @@ -363,45 +373,45 @@ describe('crowdsecExport', () => { it('should handle very long filenames', () => { const longName = 'a'.repeat(300) vi.mocked(window.prompt).mockReturnValue(longName) - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toContain('a'.repeat(100)) // Should still work expect(result!.endsWith('.tar.gz')).toBe(true) }) it('should handle unicode characters', () => { vi.mocked(window.prompt).mockReturnValue('backup-日本語-2025') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup-日本語-2025.tar.gz') }) it('should handle emoji characters', () => { vi.mocked(window.prompt).mockReturnValue('backup-🔒-secure') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup-🔒-secure.tar.gz') }) it('should handle only special characters', () => { vi.mocked(window.prompt).mockReturnValue('///\\\\\\') - const result = promptCrowdsecFilename('default.tar.gz') + const result = promptCrowdsecFilename(t, 'default.tar.gz') // Slashes become hyphens, resulting in single hyphen after sanitization expect(result).toBe('-.tar.gz') }) it('should handle filename with only spaces', () => { vi.mocked(window.prompt).mockReturnValue(' ') - const result = promptCrowdsecFilename('default.tar.gz') + const result = promptCrowdsecFilename(t, 'default.tar.gz') expect(result).toBe('default.tar.gz') }) it('should handle filename with .tar.gz in the middle', () => { vi.mocked(window.prompt).mockReturnValue('backup.tar.gz.old') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) expect(result).toBe('backup.tar.gz.old.tar.gz') }) it('should handle case variations of .tar.gz', () => { vi.mocked(window.prompt).mockReturnValue('backup.Tar.Gz') - const result = promptCrowdsecFilename() + const result = promptCrowdsecFilename(t) // Implementation uses lowercase check, so mixed case doesn't match expect(result).toBe('backup.Tar.Gz') }) @@ -450,7 +460,7 @@ describe('crowdsecExport', () => { // 2. User provides custom filename promptSpy.mockReturnValue('my-backup') - const customFilename = promptCrowdsecFilename(defaultFilename) + const customFilename = promptCrowdsecFilename(t, defaultFilename) expect(customFilename).toBe('my-backup.tar.gz') // 3. Download with custom filename @@ -464,7 +474,7 @@ describe('crowdsecExport', () => { it('should handle user cancellation', () => { const defaultFilename = buildCrowdsecExportFilename() promptSpy.mockReturnValue(null) - const result = promptCrowdsecFilename(defaultFilename) + const result = promptCrowdsecFilename(t, defaultFilename) expect(result).toBeNull() // Download should not be triggered when user cancels @@ -473,7 +483,7 @@ describe('crowdsecExport', () => { it('should use default filename when user provides empty input', () => { const defaultFilename = buildCrowdsecExportFilename() promptSpy.mockReturnValue('') - const result = promptCrowdsecFilename(defaultFilename) + const result = promptCrowdsecFilename(t, defaultFilename) expect(result).toBe(defaultFilename) }) diff --git a/frontend/src/utils/crowdsecExport.ts b/frontend/src/utils/crowdsecExport.ts index b07fcfdf4..8ded5ebbd 100644 --- a/frontend/src/utils/crowdsecExport.ts +++ b/frontend/src/utils/crowdsecExport.ts @@ -1,10 +1,12 @@ +import type { TFunction } from 'i18next' + export const buildCrowdsecExportFilename = (): string => { const timestamp = new Date().toISOString().replace(/:/g, '-') return `crowdsec-export-${timestamp}.tar.gz` } -export const promptCrowdsecFilename = (defaultName = buildCrowdsecExportFilename()): string | null => { - const input = window.prompt('Name your CrowdSec export archive', defaultName) +export const promptCrowdsecFilename = (t: TFunction, defaultName = buildCrowdsecExportFilename()): string | null => { + const input = window.prompt(t('crowdsecConfig.exportPrompt'), defaultName) if (input === null || typeof input === 'undefined') return null const trimmed = typeof input === 'string' ? input.trim() : '' const candidate = trimmed || defaultName diff --git a/tests/security/crowdsec-hub-items.spec.ts b/tests/security/crowdsec-hub-items.spec.ts new file mode 100644 index 000000000..3f08eaa84 --- /dev/null +++ b/tests/security/crowdsec-hub-items.spec.ts @@ -0,0 +1,480 @@ +/** + * CrowdSec hub item restore E2E tests (#1545) + * + * Installing items cannot be tested end to end (hub reachability of the E2E container is + * environment-dependent). These cases cover what is deterministic: + * - Restore endpoint validation, answered by the real backend before anything runs + * - The import-result panel, driven by `page.route` mocks of the import and restore endpoints + * + * The restore-endpoint cases run against the real backend. The import-result panel cases mock + * both endpoints, so they never reach the real hub or install anything. + */ + +import { promises as fs } from 'fs'; +import type { APIRequestContext, Page, Route, TestInfo } from '@playwright/test'; +import { request as playwrightRequest } from '@playwright/test'; +import { test, expect, loginUser } from '../fixtures/auth-fixtures'; +import { createTarGz } from '../utils/archive-helpers'; +import { waitForLoadingComplete } from '../utils/wait-helpers'; + +const ADMIN = '/api/v1/admin/crowdsec'; +const RESTORE = `${ADMIN}/hub-items/restore`; +const MAX_ITEMS = 1000; +const INVALID_ITEM_MESSAGE = 'invalid hub item'; +const NO_ITEMS_MESSAGE = 'no hub items given'; +const INVALID_REQUEST_MESSAGE = 'invalid request'; +const TOO_LARGE_MESSAGE = 'request too large'; +const ALREADY_RUNNING_MESSAGE = 'a restore is already running'; +const HUB_UNREACHABLE_MESSAGE = 'the CrowdSec hub could not be reached; nothing was installed'; + +/** Anchored matchers: a loose glob would also match sibling endpoints. */ +const IMPORT_ROUTE = /\/api\/v1\/admin\/crowdsec\/import(?:\?.*)?$/; +const RESTORE_ROUTE = /\/api\/v1\/admin\/crowdsec\/hub-items\/restore(?:\?.*)?$/; + +interface HubItem { + type: string; + name: string; +} + +async function restore(request: APIRequestContext, data: unknown) { + return request.post(RESTORE, { data }); +} + +test.describe('CrowdSec Hub Item Restore @security', () => { + test.describe('Restore Validation', () => { + const INVALID_NAMES: Array<{ title: string; name: string }> = [ + { title: 'a flag-like name', name: '--help' }, + { title: 'a leading dash', name: '-x/y' }, + { title: 'a path traversal', name: '../x' }, + { title: 'an embedded path traversal', name: 'a/../b' }, + { title: 'shell metacharacters', name: 'a;b/c' }, + { title: 'a name with a space', name: 'a/b c' }, + { title: 'a trailing newline', name: 'crowdsecurity/sshd\n' }, + { title: 'a second slash', name: 'a/b/c' }, + { title: 'an empty name', name: '' }, + { title: 'an empty owner', name: '/sshd' }, + { title: 'a non-ASCII name', name: 'crowdsecurity/sshé' }, + { title: 'an over-long owner', name: `${'a'.repeat(65)}/sshd` }, + { title: 'an over-long item name', name: `crowdsecurity/${'a'.repeat(129)}` }, + ]; + for (const { title, name } of INVALID_NAMES) { + test(`rejects ${title} with 400 and runs nothing`, async ({ request }) => { + const response = await restore(request, { items: [{ type: 'collections', name }] }); + + expect(response.status()).toBe(400); + expect((await response.json()).error).toBe(INVALID_ITEM_MESSAGE); + }); + } + + test('rejects an unknown item type with 400', async ({ request }) => { + const response = await restore(request, { items: [{ type: 'bouncers', name: 'crowdsecurity/sshd' }] }); + + expect(response.status()).toBe(400); + expect((await response.json()).error).toBe(INVALID_ITEM_MESSAGE); + }); + + test('rejects one invalid item among valid ones and installs none of them', async ({ request }) => { + const response = await restore(request, { + items: [ + { type: 'collections', name: 'crowdsecurity/sshd' }, + { type: 'collections', name: '--help' }, + ], + }); + + expect(response.status()).toBe(400); + expect((await response.json()).error).toBe(INVALID_ITEM_MESSAGE); + }); + + test('rejects more than 1,000 items with 400', async ({ request }) => { + const items = Array.from({ length: MAX_ITEMS + 1 }, (_, i) => ({ type: 'collections', name: `owner/item-${i}` })); + + const response = await restore(request, { items }); + + expect(response.status()).toBe(400); + expect((await response.json()).error).toBe(INVALID_ITEM_MESSAGE); + }); + + test('rejects an empty item list with 400', async ({ request }) => { + const response = await restore(request, { items: [] }); + + expect(response.status()).toBe(400); + expect((await response.json()).error).toBe(NO_ITEMS_MESSAGE); + }); + + test('rejects malformed JSON with 400', async ({ request }) => { + const response = await request.post(RESTORE, { + headers: { 'Content-Type': 'application/json' }, + data: '{"items": [', + }); + + expect(response.status()).toBe(400); + expect((await response.json()).error).toBe(INVALID_REQUEST_MESSAGE); + }); + + test('rejects a body over 1 MiB with 413', async ({ request }) => { + const response = await request.post(RESTORE, { + headers: { 'Content-Type': 'application/json' }, + data: JSON.stringify({ items: [], padding: 'x'.repeat(1024 * 1024 + 1) }), + }); + + expect(response.status()).toBe(413); + expect((await response.json()).error).toBe(TOO_LARGE_MESSAGE); + }); + + // Valid requests, the 200 success shape and the 409 "already running" answer are covered by the + // Go handler tests with a fake executor, not here: whether the E2E container can reach the + // CrowdSec hub depends on its environment, and a successful restore really installs items, + // which would change the installed-item set the import manifest specs depend on. + + test('requires authentication', async ({ baseURL }) => { + // An explicit empty storage state: a bare newContext() inherits the project's signed-in one. + const anonymous = await playwrightRequest.newContext({ + baseURL, + storageState: { cookies: [], origins: [] }, + }); + try { + const response = await anonymous.post(RESTORE, { + data: { items: [{ type: 'collections', name: 'crowdsecurity/sshd' }] }, + }); + + expect(response.status()).toBe(401); + } finally { + await anonymous.dispose(); + } + }); + }); + + test.describe('Import Result Panel', () => { + const MISSING: HubItem[] = [ + { type: 'collections', name: 'Azlaroc/sftpgo' }, + { type: 'parsers', name: 'crowdsecurity/sshd-logs' }, + { type: 'scenarios', name: 'crowdsecurity/ssh-bf' }, + ]; + + test.beforeEach(async ({ page, adminUser }) => { + await loginUser(page, adminUser); + await waitForLoadingComplete(page); + }); + + /** Mocks the import answer, then performs an import through the page. */ + async function importArchive( + page: Page, + testInfo: TestInfo, + hubItems: { in_export: number; skipped_invalid: number; missing: HubItem[] } | null, + ) { + await page.route(IMPORT_ROUTE, (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + json: { status: 'imported', backup: 'backup-1', ...(hubItems ? { hub_items: hubItems } : {}) }, + }), + ); + await page.goto('/security/crowdsec'); + await waitForLoadingComplete(page); + + const archive = testInfo.outputPath('import.tar.gz'); + await createTarGz({ 'config.yaml': 'api:\n server:\n listen_uri: 0.0.0.0:8080\n' }, archive); + await page.getByTestId('import-file').setInputFiles({ + name: 'import.tar.gz', + mimeType: 'application/gzip', + buffer: await fs.readFile(archive), + }); + await page.getByRole('button', { name: /^import$/i }).click(); + } + + /** Mocks the restore endpoint and records every request body. */ + async function mockRestore(page: Page, respond: (route: Route, call: number) => Promise) { + const bodies: Array<{ items: HubItem[] }> = []; + await page.route(RESTORE_ROUTE, async (route) => { + bodies.push(route.request().postDataJSON() as { items: HubItem[] }); + await respond(route, bodies.length); + }); + return bodies; + } + + const INSTALL_BUTTON = { name: 'Install missing hub items' }; + const ALL_INSTALLED = { + installed: MISSING, + already_installed: [], + failed: [], + remaining: 0, + skipped_invalid: 0, + reloaded: true, + }; + const HUB_UNREACHABLE_TEXT = + 'The CrowdSec hub could not be reached. Nothing was installed. Check the internet connection and try again.'; + const ALREADY_RUNNING_TEXT = 'A restore is already running. Try again when it has finished.'; + const GENERIC_ERROR_TEXT = 'The hub items could not be installed. Try again.'; + + test('offers to install the missing items after an import that reports them', async ({ page }, testInfo) => { + await importArchive(page, testInfo, { in_export: 5, skipped_invalid: 0, missing: MISSING }); + + const panel = page.getByTestId('hub-items-panel'); + await expect(panel).toBeVisible(); + await expect(panel.getByRole('heading', { name: 'Install missing hub items' })).toBeVisible(); + await expect(panel.getByRole('button', INSTALL_BUTTON)).toBeEnabled(); + for (const item of MISSING) { + await expect(panel.getByText(item.name, { exact: true })).toBeVisible(); + } + // Accessibility: the panel is a labelled region, idle (not busy), and the button is keyboard focusable. + await expect(panel).toHaveAttribute('aria-busy', 'false'); + await expect(panel).toHaveAccessibleName('Install missing hub items'); + await expect(panel.getByRole('status')).toBeAttached(); + await panel.getByRole('button', INSTALL_BUTTON).focus(); + await expect(panel.getByRole('button', INSTALL_BUTTON)).toBeFocused(); + }); + + test('shows no panel when the import reports no hub items', async ({ page }, testInfo) => { + await importArchive(page, testInfo, null); + + await expect(page.getByTestId('import-file')).toBeAttached(); + await expect(page.getByTestId('hub-items-panel')).toHaveCount(0); + await expect(page.getByRole('button', INSTALL_BUTTON)).toHaveCount(0); + }); + + test('shows no panel when nothing is missing', async ({ page }, testInfo) => { + await importArchive(page, testInfo, { in_export: 2, skipped_invalid: 0, missing: [] }); + + await expect(page.getByTestId('hub-items-panel')).toHaveCount(0); + await expect(page.getByRole('button', INSTALL_BUTTON)).toHaveCount(0); + }); + + test('sends the missing items and reports a full success', async ({ page }, testInfo) => { + const bodies = await mockRestore(page, (route) => + route.fulfill({ status: 200, contentType: 'application/json', json: ALL_INSTALLED }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('status')).toHaveText('3 installed, 0 already installed, 0 failed.'); + expect(bodies).toHaveLength(1); + expect(bodies[0]).toEqual({ items: MISSING }); + await expect(panel.getByRole('heading', { name: 'Installed', exact: true })).toBeVisible(); + await expect(panel.getByRole('heading', { name: 'Already installed', exact: true })).toHaveCount(0); + await expect(panel.getByRole('heading', { name: 'Not installed', exact: true })).toHaveCount(0); + for (const item of MISSING) { + await expect(panel.getByText(item.name, { exact: true })).toHaveCount(1); + } + await expect(panel.getByRole('button', INSTALL_BUTTON)).toHaveCount(0); + await expect(panel.getByRole('button', { name: 'Install remaining' })).toHaveCount(0); + await expect(panel.getByText('items were not attempted before the time limit.', { exact: false })).toHaveCount(0); + }); + + test('disables the button and announces progress while the install runs', async ({ page }, testInfo) => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + const bodies = await mockRestore(page, async (route) => { + await gate; + await route.fulfill({ status: 200, contentType: 'application/json', json: ALL_INSTALLED }); + }); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('button', INSTALL_BUTTON)).toBeDisabled(); + await expect(panel).toHaveAttribute('aria-busy', 'true'); + await expect(panel.getByRole('status')).toHaveText('Installing hub items. This can take several minutes.'); + for (const item of MISSING) { + await expect(panel.getByText(item.name, { exact: true })).toBeVisible(); + } + release(); + await expect(panel.getByRole('status')).toHaveText('3 installed, 0 already installed, 0 failed.'); + await expect(panel).toHaveAttribute('aria-busy', 'false'); + await expect(panel.getByRole('button', INSTALL_BUTTON)).toHaveCount(0); + expect(bodies).toHaveLength(1); + }); + + test('reports installed, already installed and failed items with a reason for each', async ({ page }, testInfo) => { + const failed = [ + { ...MISSING[0], reason: 'not_found' }, + { ...MISSING[1], reason: 'install_failed' }, + { ...MISSING[2], reason: 'verify_failed' }, + { type: 'postoverflows', name: 'crowdsecurity/rdns', reason: 'not_attempted' }, + ]; + await mockRestore(page, (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + json: { + installed: [{ type: 'parsers', name: 'crowdsecurity/nginx-logs' }], + already_installed: [{ type: 'scenarios', name: 'crowdsecurity/http-probing' }], + failed, + remaining: 1, + skipped_invalid: 0, + reloaded: true, + }, + }), + ); + await importArchive(page, testInfo, { in_export: 7, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('status')).toContainText('1 installed, 1 already installed, 4 failed.'); + await expect(panel.getByRole('status')).toContainText('1 items were not attempted before the time limit.'); + await expect(panel.getByRole('heading', { name: 'Installed', exact: true })).toBeVisible(); + await expect(panel.getByRole('heading', { name: 'Already installed', exact: true })).toBeVisible(); + await expect(panel.getByRole('heading', { name: 'Not installed', exact: true })).toBeVisible(); + await expect(panel.getByText('crowdsecurity/nginx-logs', { exact: true })).toBeVisible(); + await expect(panel.getByText('crowdsecurity/http-probing', { exact: true })).toBeVisible(); + + const expectedRows: Array<{ item: HubItem; reason: string }> = [ + { item: failed[0], reason: 'Not found in the hub' }, + { item: failed[1], reason: 'Installation failed' }, + { item: failed[2], reason: 'Installed but failed verification' }, + { item: failed[3], reason: 'Not attempted yet' }, + ]; + for (const { item, reason } of expectedRows) { + const row = panel.getByRole('listitem').filter({ has: page.getByText(item.name, { exact: true }) }); + await expect(row).toHaveCount(1); + await expect(row.getByText(item.type, { exact: true })).toBeVisible(); + await expect(row.getByText(reason, { exact: true })).toBeVisible(); + } + await expect(panel.getByRole('button', INSTALL_BUTTON)).toHaveCount(0); + }); + + test('resends only the not_attempted entries from "Install remaining"', async ({ page }, testInfo) => { + const failed = [ + { ...MISSING[1], reason: 'install_failed' }, + { ...MISSING[2], reason: 'not_attempted' }, + ]; + const bodies = await mockRestore(page, (route, call) => + route.fulfill({ + status: 200, + contentType: 'application/json', + json: + call === 1 + ? { installed: [MISSING[0]], already_installed: [], failed, remaining: 1, skipped_invalid: 0, reloaded: true } + : { installed: [MISSING[2]], already_installed: [], failed: [], remaining: 0, skipped_invalid: 0, reloaded: true }, + }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + await expect(panel.getByRole('status')).toContainText('1 installed, 0 already installed, 2 failed.'); + await expect(panel.getByRole('status')).toContainText('1 items were not attempted before the time limit.'); + await expect(panel.getByRole('button', { name: 'Install remaining' })).toBeEnabled(); + await expect(panel.getByText(MISSING[1].name, { exact: true })).toBeVisible(); + + await panel.getByRole('button', { name: 'Install remaining' }).click(); + + await expect(panel.getByRole('status')).toContainText('1 installed, 0 already installed, 0 failed.'); + expect(bodies).toHaveLength(2); + expect(bodies[0]).toEqual({ items: MISSING }); + expect(bodies[1]).toEqual({ items: [{ type: MISSING[2].type, name: MISSING[2].name }] }); + await expect(panel.getByRole('button', { name: 'Install remaining' })).toHaveCount(0); + }); + + test('keeps the earlier result and the "Install remaining" button when the follow-up fails', async ({ page }, testInfo) => { + const failed = [{ ...MISSING[2], reason: 'not_attempted' }]; + const bodies = await mockRestore(page, (route, call) => + call === 1 + ? route.fulfill({ + status: 200, + contentType: 'application/json', + json: { installed: [MISSING[0], MISSING[1]], already_installed: [], failed, remaining: 1, skipped_invalid: 0, reloaded: true }, + }) + : route.fulfill({ status: 502, contentType: 'application/json', json: { error: HUB_UNREACHABLE_MESSAGE } }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + await panel.getByRole('button', INSTALL_BUTTON).click(); + await expect(panel.getByRole('status')).toContainText('2 installed, 0 already installed, 1 failed.'); + + await panel.getByRole('button', { name: 'Install remaining' }).click(); + + await expect(panel.getByRole('status')).toContainText(HUB_UNREACHABLE_TEXT); + expect(bodies).toHaveLength(2); + await expect(panel.getByRole('status')).toContainText('2 installed, 0 already installed, 1 failed.'); + await expect(panel.getByText(MISSING[2].name, { exact: true })).toBeVisible(); + await expect(panel.getByRole('button', { name: 'Install remaining' })).toBeEnabled(); + }); + + test('shows a plain note when some manifest entries could not be used', async ({ page }, testInfo) => { + await mockRestore(page, (route) => + route.fulfill({ status: 200, contentType: 'application/json', json: { ...ALL_INSTALLED, skipped_invalid: 2 } }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 2, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await expect(panel.getByText('2 entries in the export could not be used.', { exact: true })).toBeVisible(); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('status')).toContainText('3 installed, 0 already installed, 0 failed.'); + await expect(panel.getByText('2 entries in the export could not be used.', { exact: true })).toBeVisible(); + }); + + test('keeps the page usable and the button available when the hub is unreachable (502)', async ({ page }, testInfo) => { + const bodies = await mockRestore(page, (route) => + route.fulfill({ status: 502, contentType: 'application/json', json: { error: HUB_UNREACHABLE_MESSAGE } }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('status')).toHaveText(HUB_UNREACHABLE_TEXT); + expect(bodies).toHaveLength(1); + await expect(panel.getByRole('button', INSTALL_BUTTON)).toBeEnabled(); + for (const item of MISSING) { + await expect(panel.getByText(item.name, { exact: true })).toBeVisible(); + } + await expect(page.getByRole('button', { name: 'Export', exact: true })).toBeEnabled(); + }); + + test('keeps the button available when a restore is already running (409)', async ({ page }, testInfo) => { + const bodies = await mockRestore(page, (route) => + route.fulfill({ status: 409, contentType: 'application/json', json: { error: ALREADY_RUNNING_MESSAGE } }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('status')).toHaveText(ALREADY_RUNNING_TEXT); + expect(bodies).toHaveLength(1); + await expect(panel.getByRole('button', INSTALL_BUTTON)).toBeEnabled(); + for (const item of MISSING) { + await expect(panel.getByText(item.name, { exact: true })).toBeVisible(); + } + }); + + test('shows a generic error and keeps the button available on any other failure', async ({ page }, testInfo) => { + const bodies = await mockRestore(page, (route) => + route.fulfill({ status: 500, contentType: 'application/json', json: { error: 'internal error' } }), + ); + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + const panel = page.getByTestId('hub-items-panel'); + + await panel.getByRole('button', INSTALL_BUTTON).click(); + + await expect(panel.getByRole('status')).toHaveText(GENERIC_ERROR_TEXT); + expect(bodies).toHaveLength(1); + await expect(panel.getByRole('button', INSTALL_BUTTON)).toBeEnabled(); + for (const item of MISSING) { + await expect(panel.getByText(item.name, { exact: true })).toBeVisible(); + } + }); + + test('clears the panel when another file is selected', async ({ page }, testInfo) => { + await importArchive(page, testInfo, { in_export: 3, skipped_invalid: 0, missing: MISSING }); + await expect(page.getByTestId('hub-items-panel')).toBeVisible(); + + await page.getByTestId('import-file').setInputFiles({ + name: 'other.tar.gz', + mimeType: 'application/gzip', + buffer: Buffer.from('other'), + }); + + await expect(page.getByTestId('hub-items-panel')).toHaveCount(0); + await expect(page.getByRole('button', INSTALL_BUTTON)).toHaveCount(0); + }); + }); +}); diff --git a/tests/security/crowdsec-import.spec.ts b/tests/security/crowdsec-import.spec.ts index 0dbaea230..49e844cb8 100644 --- a/tests/security/crowdsec-import.spec.ts +++ b/tests/security/crowdsec-import.spec.ts @@ -21,7 +21,12 @@ import { createTarGz, createZip, createZipBomb, + HUB_MANIFEST_NAME, + hubManifest, listTarGzEntries, + notificationYaml, + profilesYaml, + readTarGzEntry, } from '../utils/archive-helpers'; const ADMIN = '/api/v1/admin/crowdsec'; @@ -214,6 +219,8 @@ test.describe('CrowdSec Config Import Validation', () => { const baseNames = files.map((name) => name.split('/').pop()?.toLowerCase()); expect(files).toEqual(expect.arrayContaining([expect.stringMatching(/^(config\/)?config\.yaml$/)])); + // The hub item manifest is the one extra root entry, present exactly once. + expect(files.filter((name) => name === HUB_MANIFEST_NAME)).toEqual([HUB_MANIFEST_NAME]); expect(files.filter((name) => /^(\.\/)?(data|hub_cache)\//.test(name))).toEqual([]); expect(files.filter((name) => /\.db(-wal|-shm)?$/.test(name))).toEqual([]); expect(baseNames).not.toContain('bouncer_key'); @@ -231,6 +238,25 @@ test.describe('CrowdSec Config Import Validation', () => { const entries = await listTarGzEntries(body); expect(entries).toEqual(expect.arrayContaining([expect.stringMatching(/^(config\/)?config\.yaml$/)])); + expect(entries).toContain(HUB_MANIFEST_NAME); + }); + + test('export carries a sorted version 1 hub item manifest listing the installed collection', async ({ request }) => { + const response = await request.get(`${ADMIN}/export`, { timeout: 120_000 }); + expect(response.status()).toBe(200); + + const manifestText = await readTarGzEntry(await response.body(), HUB_MANIFEST_NAME); + expect(manifestText).not.toBeNull(); + const manifest = JSON.parse(manifestText as string) as { version: number; items: Array<{ type: string; name: string }> }; + + expect(manifest.version).toBe(1); + expect(Object.keys(manifest).sort()).toEqual(['items', 'version']); + expect(manifest.items).toEqual(expect.arrayContaining([{ type: 'collections', name: 'crowdsecurity/caddy' }])); + for (const item of manifest.items) { + expect(Object.keys(item).sort()).toEqual(['name', 'type']); + } + const keys = manifest.items.map((item) => `${item.type}\u0000${item.name}`); + expect(keys).toEqual([...keys].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))); }); test('import accepts an export laid out as config/config.yaml', async ({ request }, testInfo) => { @@ -264,7 +290,12 @@ test.describe('CrowdSec Config Import Validation', () => { }); expect(response.status()).toBe(200); - expect((await response.json()).status).toBe('imported'); + const body = await response.json(); + expect(body.status).toBe('imported'); + // Everything the export lists is already installed on the same host. + expect(body.hub_items.missing).toEqual([]); + expect(body.hub_items.skipped_invalid).toBe(0); + expect(body.hub_items.in_export).toBeGreaterThanOrEqual(1); expect(await readRegistration()).toBe(registeredBefore); }); @@ -343,4 +374,303 @@ test.describe('CrowdSec Config Import Validation', () => { expect((await response.json()).error).toBe('validation failed: only .tar.gz archives are supported'); }); }); + + // An import whose profiles.yaml names a notification that no file defines would leave CrowdSec + // unable to start, so it is refused before anything changes. + test.describe('Notification References', () => { + const missingMessage = (names: string) => + `config validation failed: profiles.yaml names notifications that have no matching file: ${names}`; + + test('refuses a profiles.yaml naming a notification no file defines and changes nothing', async ({ request }, testInfo) => { + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, 'profiles.yaml': profilesYaml(['x']) }, + testInfo.outputPath('missing-notification.tar.gz'), + ); + const filesBefore = (await (await request.get(`${ADMIN}/files`)).json()).files as string[]; + + const response = await upload(request, archive, 'missing-notification.tar.gz'); + + expect(response.status()).toBe(422); + expect((await response.json()).error).toBe(missingMessage('x')); + expect(await readConfigFile(request, 'config.yaml')).toBe(BASELINE_CONFIG); + expect(await readConfigFile(request, 'acquis.yaml')).toBe(BASELINE_ACQUIS); + expect((await (await request.get(`${ADMIN}/files`)).json()).files as string[]).toEqual(filesBefore); + }); + + test('lists every missing notification name in sorted order', async ({ request }, testInfo) => { + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, 'profiles.yaml': profilesYaml(['zeta', 'alpha']) }, + testInfo.outputPath('two-missing.tar.gz'), + ); + + const response = await upload(request, archive, 'two-missing.tar.gz'); + + expect(response.status()).toBe(422); + expect((await response.json()).error).toBe(missingMessage('alpha, zeta')); + }); + + test('lists at most ten missing names and counts the rest', async ({ request }, testInfo) => { + const names = Array.from({ length: 12 }, (_, index) => `n${String(index + 1).padStart(2, '0')}`); + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, 'profiles.yaml': profilesYaml([...names].reverse()) }, + testInfo.outputPath('many-missing.tar.gz'), + ); + + const response = await upload(request, archive, 'many-missing.tar.gz'); + + expect(response.status()).toBe(422); + expect((await response.json()).error).toBe(missingMessage(`${names.slice(0, 10).join(', ')}, and 2 more`)); + }); + + test('accepts a profiles.yaml whose notification is defined by a notifications file', async ({ request }, testInfo) => { + const archive = await createTarGz( + { + 'config.yaml': VALID_CONFIG, + 'profiles.yaml': profilesYaml(['x']), + 'notifications/x.yaml': notificationYaml('x'), + }, + testInfo.outputPath('defined-notification.tar.gz'), + ); + + const response = await upload(request, archive, 'defined-notification.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).status).toBe('imported'); + expect(await readConfigFile(request, 'notifications/x.yaml')).toBe(notificationYaml('x')); + }); + + test('accepts a notification defined in a .yml file', async ({ request }, testInfo) => { + const archive = await createTarGz( + { + 'config.yaml': VALID_CONFIG, + 'profiles.yaml': profilesYaml(['x']), + 'notifications/x.yml': notificationYaml('x'), + }, + testInfo.outputPath('yml-notification.tar.gz'), + ); + + const response = await upload(request, archive, 'yml-notification.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).status).toBe('imported'); + }); + + test('matches by the name field, so a multi-document file defines every name in it', async ({ request }, testInfo) => { + const archive = await createTarGz( + { + 'config.yaml': VALID_CONFIG, + 'profiles.yaml': profilesYaml(['first', 'second']), + 'notifications/both.yaml': notificationYaml('first', 'second'), + }, + testInfo.outputPath('multi-document.tar.gz'), + ); + + const response = await upload(request, archive, 'multi-document.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).status).toBe('imported'); + }); + + test('refuses a name that only the file name matches', async ({ request }, testInfo) => { + const archive = await createTarGz( + { + 'config.yaml': VALID_CONFIG, + 'profiles.yaml': profilesYaml(['a']), + 'notifications/a.yaml': notificationYaml('b'), + }, + testInfo.outputPath('file-name-match.tar.gz'), + ); + + const response = await upload(request, archive, 'file-name-match.tar.gz'); + + expect(response.status()).toBe(422); + expect((await response.json()).error).toBe(missingMessage('a')); + }); + + test('accepts a name defined under a different file name', async ({ request }, testInfo) => { + const archive = await createTarGz( + { + 'config.yaml': VALID_CONFIG, + 'profiles.yaml': profilesYaml(['b']), + 'notifications/a.yaml': notificationYaml('b'), + }, + testInfo.outputPath('name-field-match.tar.gz'), + ); + + const response = await upload(request, archive, 'name-field-match.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).status).toBe('imported'); + }); + + // CrowdSec only loads lowercase .yaml/.yml files directly inside notifications/. + const ignoredDefinitions = [ + { title: 'a subdirectory', path: 'notifications/sub/s.yaml' }, + { title: 'a .txt file', path: 'notifications/s.txt' }, + { title: 'a .bak file', path: 'notifications/s.bak' }, + { title: 'an uppercase .YAML file', path: 'notifications/s.YAML' }, + ]; + for (const { title, path } of ignoredDefinitions) { + test(`refuses a notification defined only in ${title}`, async ({ request }, testInfo) => { + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, 'profiles.yaml': profilesYaml(['s']), [path]: notificationYaml('s') }, + testInfo.outputPath('ignored-definition.tar.gz'), + ); + const filesBefore = (await (await request.get(`${ADMIN}/files`)).json()).files as string[]; + + const response = await upload(request, archive, 'ignored-definition.tar.gz'); + + expect(response.status()).toBe(422); + expect((await response.json()).error).toBe(missingMessage('s')); + expect(await readConfigFile(request, 'config.yaml')).toBe(BASELINE_CONFIG); + expect((await (await request.get(`${ADMIN}/files`)).json()).files as string[]).toEqual(filesBefore); + }); + } + + test('accepts an archive whose profiles.yaml names nothing', async ({ request }, testInfo) => { + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, 'profiles.yaml': 'name: default_ip_remediation\nnotifications: []\non_success: break\n' }, + testInfo.outputPath('no-notifications.tar.gz'), + ); + + const response = await upload(request, archive, 'no-notifications.tar.gz'); + + expect(response.status()).toBe(200); + }); + + test('accepts an archive with no profiles.yaml', async ({ request }, testInfo) => { + const archive = await createTarGz({ 'config.yaml': VALID_CONFIG }, testInfo.outputPath('no-profiles.tar.gz')); + + const response = await upload(request, archive, 'no-profiles.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).status).toBe('imported'); + }); + }); + + // An export carries charon-hub-items.json (names and types only); an import reports which of + // the listed items the target lacks. The manifest is read, never written to disk. + test.describe('Hub Item Manifest', () => { + const ITEMS = [ + { type: 'collections', name: 'Azlaroc/sftpgo' }, + { type: 'parsers', name: 'crowdsecurity/sshd-logs' }, + { type: 'scenarios', name: 'crowdsecurity/ssh-bf' }, + ]; + + async function importWithManifest(request: APIRequestContext, manifest: string, outputPath: string) { + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, [HUB_MANIFEST_NAME]: manifest }, + outputPath, + ); + return upload(request, archive, 'with-manifest.tar.gz'); + } + + test('an archive without a manifest imports with no hub_items field', async ({ request }, testInfo) => { + const archive = await createTarGz({ 'config.yaml': VALID_CONFIG }, testInfo.outputPath('legacy.tar.gz')); + + const response = await upload(request, archive, 'legacy.tar.gz'); + + expect(response.status()).toBe(200); + expect(await response.json()).not.toHaveProperty('hub_items'); + }); + + test('reports the manifest items the target lacks as hub_items.missing', async ({ request }, testInfo) => { + const response = await importWithManifest(request, hubManifest(ITEMS), testInfo.outputPath('manifest.tar.gz')); + + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.status).toBe('imported'); + expect(body.hub_items).toEqual({ in_export: 3, skipped_invalid: 0, missing: ITEMS }); + }); + + test('never writes the manifest to disk or lets it shape a later export', async ({ request }, testInfo) => { + const response = await importWithManifest(request, hubManifest(ITEMS), testInfo.outputPath('manifest-disk.tar.gz')); + expect(response.status()).toBe(200); + + const files = (await (await request.get(`${ADMIN}/files`)).json()).files as string[]; + expect(files).not.toContain(HUB_MANIFEST_NAME); + + const exported = await request.get(`${ADMIN}/export`, { timeout: 120_000 }); + expect(exported.status()).toBe(200); + // The export regenerates its own manifest from the installed links, not from the upload. + const manifest = JSON.parse((await readTarGzEntry(await exported.body(), HUB_MANIFEST_NAME)) as string); + expect(manifest.items).not.toContainEqual(ITEMS[0]); + }); + + test('reads a manifest spelled ./charon-hub-items.json and still does not write it', async ({ request }, testInfo) => { + const archive = await createRawTarGz( + [ + { name: 'config.yaml', content: VALID_CONFIG }, + { name: `./${HUB_MANIFEST_NAME}`, content: hubManifest(ITEMS) }, + ], + testInfo.outputPath('dot-slash-manifest.tar.gz'), + ); + + const response = await upload(request, archive, 'dot-slash-manifest.tar.gz'); + + expect(response.status()).toBe(200); + expect((await response.json()).hub_items.missing).toEqual(ITEMS); + const files = (await (await request.get(`${ADMIN}/files`)).json()).files as string[]; + expect(files).not.toContain(HUB_MANIFEST_NAME); + }); + + test('counts individually invalid manifest items in skipped_invalid without naming them', async ({ request }, testInfo) => { + const manifest = hubManifest([ + ITEMS[0], + { type: 'collections', name: '--help' }, + { type: 'nonsense', name: 'crowdsecurity/sshd' }, + { type: 'parsers', name: 'a/../b' }, + ]); + + const response = await importWithManifest(request, manifest, testInfo.outputPath('some-invalid.tar.gz')); + + expect(response.status()).toBe(200); + expect((await response.json()).hub_items).toEqual({ in_export: 1, skipped_invalid: 3, missing: [ITEMS[0]] }); + }); + + const ignoredManifests: Array<{ title: string; manifest: string }> = [ + { title: 'is not JSON', manifest: '{not json' }, + { title: 'has an unknown version', manifest: hubManifest(ITEMS, 2) }, + { title: 'lists more than 1,000 items', manifest: hubManifest(Array.from({ length: 1001 }, (_, i) => ({ type: 'collections', name: `owner/item-${i}` }))) }, + ]; + for (const { title, manifest } of ignoredManifests) { + test(`ignores a manifest that ${title} and still imports`, async ({ request }, testInfo) => { + const response = await importWithManifest(request, manifest, testInfo.outputPath('ignored-manifest.tar.gz')); + + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.status).toBe('imported'); + expect(body).not.toHaveProperty('hub_items'); + }); + } + + test('ignores the whole manifest when the archive carries it twice', async ({ request }, testInfo) => { + const archive = await createRawTarGz( + [ + { name: 'config.yaml', content: VALID_CONFIG }, + { name: HUB_MANIFEST_NAME, content: hubManifest(ITEMS) }, + { name: HUB_MANIFEST_NAME, content: hubManifest(ITEMS.slice(0, 1)) }, + ], + testInfo.outputPath('duplicate-manifest.tar.gz'), + ); + + const response = await upload(request, archive, 'duplicate-manifest.tar.gz'); + + expect(response.status()).toBe(200); + expect(await response.json()).not.toHaveProperty('hub_items'); + }); + + test('extracts a nested file with the manifest name as an ordinary file', async ({ request }, testInfo) => { + const archive = await createTarGz( + { 'config.yaml': VALID_CONFIG, [`nested/${HUB_MANIFEST_NAME}`]: hubManifest(ITEMS) }, + testInfo.outputPath('nested-manifest.tar.gz'), + ); + + const response = await upload(request, archive, 'nested-manifest.tar.gz'); + + expect(response.status()).toBe(200); + expect(await response.json()).not.toHaveProperty('hub_items'); + expect(await readConfigFile(request, `nested/${HUB_MANIFEST_NAME}`)).toBe(hubManifest(ITEMS)); + }); + }); }); diff --git a/tests/utils/archive-helpers.ts b/tests/utils/archive-helpers.ts index 3d94a719c..a3ec3cf22 100644 --- a/tests/utils/archive-helpers.ts +++ b/tests/utils/archive-helpers.ts @@ -265,3 +265,69 @@ export async function listTarGzEntries(archive: Buffer): Promise { }); return names; } + +/** Reads the text content of one entry (matched by exact path) from a tar.gz buffer, or null when absent. */ +export async function readTarGzEntry(archive: Buffer, entryName: string): Promise { + let content: string | null = null; + await new Promise((resolve, reject) => { + const parser = new tar.Parser(); + parser.on('entry', (entry) => { + if (entry.path !== entryName) { + entry.resume(); + return; + } + const chunks: Buffer[] = []; + entry.on('data', (chunk: Buffer) => chunks.push(chunk)); + entry.on('end', () => { + content = Buffer.concat(chunks).toString('utf8'); + }); + }); + parser.on('end', resolve); + parser.on('error', reject); + parser.end(archive); + }); + return content; +} + +/** Name of the hub item manifest that a CrowdSec export carries at the archive root. */ +export const HUB_MANIFEST_NAME = 'charon-hub-items.json'; + +export interface HubItemRef { + type: string; + name: string; +} + +/** Serialises a hub item manifest. Pass `version` or raw `items` to build an unsupported or damaged one. */ +export function hubManifest(items: unknown[], version: number = 1): string { + return JSON.stringify({ version, items }); +} + +/** A minimal profiles.yaml whose single profile names the given notifications. */ +export function profilesYaml(notifications: string[]): string { + const list = notifications.map((name) => ` - ${name}`).join('\n'); + return `name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +notifications: +${list} +on_success: break +`; +} + +/** A notification plugin config document with a placeholder URL. */ +export function notificationYaml(...names: string[]): string { + return names + .map( + (name) => `type: http +name: ${name} +log_level: info +format: "" +url: https://notify.example.invalid/hook +method: POST +`, + ) + .join('---\n'); +}