From c8b34d84c3fbfb64622dcde8b0f0fcdbe1081324 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:04:12 +0200 Subject: [PATCH 01/22] feat(integrations): load and validate webhook configuration Read INTEGRATION_GITLAB_SIGNING_TOKEN, INTEGRATION_GITLAB_SECRET_TOKEN, INTEGRATION_FLUX_HMAC_KEY, INTEGRATION_WEBHOOK_TOLERANCE and INTEGRATION_ENVIRONMENTS at startup. An invalid value stops the server; secrets never appear in errors or logs. Refs #208 --- cmd/serv.go | 13 ++ internal/integrations/config.go | 226 ++++++++++++++++++++++++++ internal/integrations/config_test.go | 228 +++++++++++++++++++++++++++ internal/integrations/doc.go | 4 + internal/integrations/errors.go | 50 ++++++ internal/integrations/errors_test.go | 44 ++++++ 6 files changed, 565 insertions(+) create mode 100644 internal/integrations/config.go create mode 100644 internal/integrations/config_test.go create mode 100644 internal/integrations/doc.go create mode 100644 internal/integrations/errors.go create mode 100644 internal/integrations/errors_test.go diff --git a/cmd/serv.go b/cmd/serv.go index c2ccc7b..d9b2950 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -23,6 +23,7 @@ import ( "github.com/bananaops/tracker/internal/auth" "github.com/bananaops/tracker/internal/auth/identity" "github.com/bananaops/tracker/internal/auth/sso" + "github.com/bananaops/tracker/internal/integrations" store "github.com/bananaops/tracker/internal/stores" "github.com/bananaops/tracker/server" "github.com/go-openapi/runtime/middleware" @@ -49,6 +50,18 @@ var serv = &cobra.Command{ if authCfg.AnonymousDefaulted { slog.Warn("AUTH_ANONYMOUS_PERMISSIONS is not set: anonymous callers keep every permission except access:manage. This default becomes empty in the next major release.") } + + // Deployment integrations (GitLab and Flux webhooks). The routes are + // registered once the mux exists; an invalid configuration stops here. + integrationsCfg, err := integrations.LoadConfig(os.LookupEnv) + if err != nil { + log.Fatalf("invalid integrations configuration: %v", err) + } + for _, w := range integrationsCfg.Warnings { + slog.Warn(w) + } + slog.Info("deployment integrations", integrationsCfg.LogAttrs()...) + userStore := store.NewAuthUserStore() teamStore := store.NewAuthTeamStore() keyStore := store.NewAuthAPIKeyStore() diff --git a/internal/integrations/config.go b/internal/integrations/config.go new file mode 100644 index 0000000..359f4ae --- /dev/null +++ b/internal/integrations/config.go @@ -0,0 +1,226 @@ +package integrations + +import ( + "encoding/base64" + "fmt" + "sort" + "strings" + "time" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +// LookupEnv has the signature of os.LookupEnv. +type LookupEnv func(key string) (string, bool) + +// Environment variables read by LoadConfig. +const ( + EnvGitLabSigningToken = "INTEGRATION_GITLAB_SIGNING_TOKEN" + EnvGitLabSecretToken = "INTEGRATION_GITLAB_SECRET_TOKEN" + EnvFluxHMACKey = "INTEGRATION_FLUX_HMAC_KEY" + EnvWebhookTolerance = "INTEGRATION_WEBHOOK_TOLERANCE" + EnvEnvironments = "INTEGRATION_ENVIRONMENTS" +) + +// DefaultTolerance is the clock skew allowed between a webhook timestamp and +// the time it is received, when INTEGRATION_WEBHOOK_TOLERANCE is not set. +const DefaultTolerance = 5 * time.Minute + +// DefaultEnvironments is the mapping used when INTEGRATION_ENVIRONMENTS is not set. +const DefaultEnvironments = "production=production,staging=preproduction" + +// Config is the deployment integrations configuration read from the environment. +type Config struct { + // GitLabSigningKey is the decoded INTEGRATION_GITLAB_SIGNING_TOKEN, used to + // verify the GitLab "webhook-signature" HMAC header. Empty when unset. + GitLabSigningKey []byte + // GitLabSecretToken is the raw INTEGRATION_GITLAB_SECRET_TOKEN, compared + // against the "X-Gitlab-Token" header. Empty when unset, or when a signing + // key is also configured (the signing key then takes precedence). + GitLabSecretToken string + // FluxHMACKey is the raw INTEGRATION_FLUX_HMAC_KEY, used to verify Flux + // notification-controller HMAC signatures. + FluxHMACKey []byte + // Tolerance is the maximum accepted gap between a webhook timestamp and now. + Tolerance time.Duration + // Environments maps a lower-cased external environment name to a Tracker + // environment. + Environments map[string]eventv1.Environment + // Warnings are non-fatal configuration issues, to be logged by the caller. + Warnings []string +} + +// LoadConfig reads and validates the INTEGRATION_* variables. An invalid +// value returns an error whose text never contains the offending secret. +func LoadConfig(lookup LookupEnv) (Config, error) { + get := func(k string) string { + v, _ := lookup(k) + return strings.TrimSpace(v) + } + + cfg := Config{} + + if v := get(EnvGitLabSigningToken); v != "" { + key, err := decodeSigningToken(v) + if err != nil { + return Config{}, err + } + cfg.GitLabSigningKey = key + } + + if v := get(EnvGitLabSecretToken); v != "" { + if len(cfg.GitLabSigningKey) > 0 { + cfg.Warnings = append(cfg.Warnings, fmt.Sprintf( + "%s is ignored because %s is set", EnvGitLabSecretToken, EnvGitLabSigningToken)) + } else { + if len(v) < 16 { + return Config{}, fmt.Errorf("%s must be at least %d characters", EnvGitLabSecretToken, 16) + } + cfg.GitLabSecretToken = v + } + } + + if v := get(EnvFluxHMACKey); v != "" { + if len([]byte(v)) < 32 { + return Config{}, fmt.Errorf("%s must be at least %d bytes", EnvFluxHMACKey, 32) + } + cfg.FluxHMACKey = []byte(v) + } + + tolerance, err := parseTolerance(get(EnvWebhookTolerance)) + if err != nil { + return Config{}, err + } + cfg.Tolerance = tolerance + + environments, err := parseEnvironments(get(EnvEnvironments)) + if err != nil { + return Config{}, err + } + cfg.Environments = environments + + return cfg, nil +} + +// decodeSigningToken validates and decodes INTEGRATION_GITLAB_SIGNING_TOKEN. +// v must be non-empty; the error never repeats v. +func decodeSigningToken(v string) ([]byte, error) { + invalid := fmt.Errorf("%s must be whsec_ followed by a non empty base64 value", EnvGitLabSigningToken) + if !strings.HasPrefix(v, "whsec_") { + return nil, invalid + } + key, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(v, "whsec_")) + if err != nil || len(key) == 0 { + return nil, invalid + } + return key, nil +} + +// parseTolerance validates INTEGRATION_WEBHOOK_TOLERANCE. v is already trimmed. +func parseTolerance(v string) (time.Duration, error) { + if v == "" { + return DefaultTolerance, nil + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s must be a positive duration such as 5m, got %q", EnvWebhookTolerance, v) + } + return d, nil +} + +// parseEnvironments validates INTEGRATION_ENVIRONMENTS. v is already trimmed; +// an empty v falls back to DefaultEnvironments. +func parseEnvironments(v string) (map[string]eventv1.Environment, error) { + if v == "" { + v = DefaultEnvironments + } + + result := make(map[string]eventv1.Environment) + for _, rawEntry := range strings.Split(v, ",") { + entry := strings.TrimSpace(rawEntry) + if entry == "" { + return nil, fmt.Errorf("%s: entry must not be empty", EnvEnvironments) + } + + idx := strings.Index(entry, "=") + if idx < 0 { + return nil, fmt.Errorf("%s: entry %q must be key=value", EnvEnvironments, entry) + } + + key := strings.TrimSpace(entry[:idx]) + value := strings.TrimSpace(entry[idx+1:]) + if key == "" { + return nil, fmt.Errorf("%s: entry %q has an empty key", EnvEnvironments, entry) + } + + lowerKey := strings.ToLower(key) + if _, exists := result[lowerKey]; exists { + return nil, fmt.Errorf("%s: duplicate key %q", EnvEnvironments, lowerKey) + } + + n, ok := eventv1.Environment_value[value] + if !ok || value == "ENVIRONMENT_UNSPECIFIED" { + return nil, fmt.Errorf("%s: %q is not a known environment", EnvEnvironments, value) + } + + result[lowerKey] = eventv1.Environment(n) + } + + return result, nil +} + +// GitLabEnabled reports whether GitLab webhook verification is configured. +func (c Config) GitLabEnabled() bool { + return len(c.GitLabSigningKey) > 0 || c.GitLabSecretToken != "" +} + +// FluxEnabled reports whether Flux webhook verification is configured. +func (c Config) FluxEnabled() bool { + return len(c.FluxHMACKey) > 0 +} + +// Enabled reports whether any deployment integration is configured. +func (c Config) Enabled() bool { + return c.GitLabEnabled() || c.FluxEnabled() +} + +// LookupEnvironment resolves an external environment name, trimmed and +// compared case-insensitively, to a Tracker environment. +func (c Config) LookupEnvironment(name string) (eventv1.Environment, bool) { + k := strings.ToLower(strings.TrimSpace(name)) + if k == "" { + return 0, false + } + e, ok := c.Environments[k] + return e, ok +} + +// LogAttrs returns slog-style key/value pairs describing the configuration, +// without ever exposing a secret. +func (c Config) LogAttrs() []any { + mode := "disabled" + switch { + case len(c.GitLabSigningKey) > 0: + mode = "signing_token" + case c.GitLabSecretToken != "": + mode = "secret_token" + } + + keys := make([]string, 0, len(c.Environments)) + for k := range c.Environments { + keys = append(keys, k) + } + sort.Strings(keys) + + pairs := make([]string, 0, len(keys)) + for _, k := range keys { + pairs = append(pairs, k+"="+c.Environments[k].String()) + } + + return []any{ + "gitlab", mode, + "flux", c.FluxEnabled(), + "tolerance", c.Tolerance.String(), + "environments", strings.Join(pairs, ","), + } +} diff --git a/internal/integrations/config_test.go b/internal/integrations/config_test.go new file mode 100644 index 0000000..5c85ee8 --- /dev/null +++ b/internal/integrations/config_test.go @@ -0,0 +1,228 @@ +package integrations + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "testing" + "time" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func env(m map[string]string) LookupEnv { + return func(k string) (string, bool) { v, ok := m[k]; return v, ok } +} + +var ( + signing = "whsec_" + base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x11}, 32)) + secret16 = "0123456789abcdef" + flux32 = strings.Repeat("k", 32) +) + +func TestLoadConfig(t *testing.T) { + tests := []struct { + name string + vars map[string]string + wantErr string + check func(t *testing.T, cfg Config) + }{ + { + name: "no variables", + vars: map[string]string{}, + check: func(t *testing.T, cfg Config) { + assert.False(t, cfg.Enabled()) + assert.Equal(t, 5*time.Minute, cfg.Tolerance) + assert.Equal(t, map[string]eventv1.Environment{ + "production": eventv1.Environment_production, + "staging": eventv1.Environment_preproduction, + }, cfg.Environments) + }, + }, + { + name: "valid signing token", + vars: map[string]string{EnvGitLabSigningToken: signing}, + check: func(t *testing.T, cfg Config) { + assert.Len(t, cfg.GitLabSigningKey, 32) + assert.True(t, cfg.GitLabEnabled()) + assert.False(t, cfg.FluxEnabled()) + }, + }, + { + name: "signing token without whsec_ prefix", + vars: map[string]string{EnvGitLabSigningToken: "bad"}, + wantErr: EnvGitLabSigningToken, + }, + { + name: "signing token with invalid base64", + vars: map[string]string{EnvGitLabSigningToken: "whsec_!!!"}, + wantErr: EnvGitLabSigningToken, + }, + { + name: "signing token empty after prefix", + vars: map[string]string{EnvGitLabSigningToken: "whsec_"}, + wantErr: EnvGitLabSigningToken, + }, + { + name: "secret token too short", + vars: map[string]string{EnvGitLabSecretToken: "short"}, + wantErr: EnvGitLabSecretToken, + }, + { + name: "secret token valid", + vars: map[string]string{EnvGitLabSecretToken: secret16}, + check: func(t *testing.T, cfg Config) { + assert.Equal(t, secret16, cfg.GitLabSecretToken) + }, + }, + { + name: "secret token ignored when signing token is set", + vars: map[string]string{EnvGitLabSigningToken: signing, EnvGitLabSecretToken: "short"}, + check: func(t *testing.T, cfg Config) { + assert.Empty(t, cfg.GitLabSecretToken) + assert.Len(t, cfg.Warnings, 1) + }, + }, + { + name: "flux key too short", + vars: map[string]string{EnvFluxHMACKey: strings.Repeat("k", 31)}, + wantErr: EnvFluxHMACKey, + }, + { + name: "flux key valid", + vars: map[string]string{EnvFluxHMACKey: flux32}, + check: func(t *testing.T, cfg Config) { + assert.True(t, cfg.FluxEnabled()) + assert.Equal(t, []byte(flux32), cfg.FluxHMACKey) + }, + }, + { + name: "tolerance not a duration", + vars: map[string]string{EnvWebhookTolerance: "abc"}, + wantErr: EnvWebhookTolerance, + }, + { + name: "tolerance zero", + vars: map[string]string{EnvWebhookTolerance: "0s"}, + wantErr: EnvWebhookTolerance, + }, + { + name: "tolerance negative", + vars: map[string]string{EnvWebhookTolerance: "-1m"}, + wantErr: EnvWebhookTolerance, + }, + { + name: "tolerance valid", + vars: map[string]string{EnvWebhookTolerance: "30s"}, + check: func(t *testing.T, cfg Config) { + assert.Equal(t, 30*time.Second, cfg.Tolerance) + }, + }, + { + name: "environments entry without =", + vars: map[string]string{EnvEnvironments: "production"}, + wantErr: EnvEnvironments, + }, + { + name: "environments empty key", + vars: map[string]string{EnvEnvironments: "=production"}, + wantErr: EnvEnvironments, + }, + { + name: "environments duplicate key case insensitive", + vars: map[string]string{EnvEnvironments: "prod=production,PROD=production"}, + wantErr: EnvEnvironments, + }, + { + name: "environments unknown value", + vars: map[string]string{EnvEnvironments: "prod=live"}, + wantErr: EnvEnvironments, + }, + { + name: "environments unspecified value", + vars: map[string]string{EnvEnvironments: "prod=ENVIRONMENT_UNSPECIFIED"}, + wantErr: EnvEnvironments, + }, + { + name: "environments empty entry", + vars: map[string]string{EnvEnvironments: "prod=production,,x=UAT"}, + wantErr: EnvEnvironments, + }, + { + name: "environments trimmed and case insensitive keys, case sensitive values", + vars: map[string]string{EnvEnvironments: " Prod=production , QA=UAT "}, + check: func(t *testing.T, cfg Config) { + assert.Equal(t, map[string]eventv1.Environment{ + "prod": eventv1.Environment_production, + "qa": eventv1.Environment_UAT, + }, cfg.Environments) + }, + }, + { + name: "environments blank falls back to default", + vars: map[string]string{EnvEnvironments: " "}, + check: func(t *testing.T, cfg Config) { + assert.Equal(t, map[string]eventv1.Environment{ + "production": eventv1.Environment_production, + "staging": eventv1.Environment_preproduction, + }, cfg.Environments) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg, err := LoadConfig(env(tt.vars)) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + for _, secretValue := range []string{signing, secret16, flux32} { + assert.NotContains(t, err.Error(), secretValue) + } + return + } + require.NoError(t, err) + if tt.check != nil { + tt.check(t, cfg) + } + }) + } +} + +func TestLookupEnvironment(t *testing.T) { + cfg, err := LoadConfig(env(map[string]string{})) + require.NoError(t, err) + + e, ok := cfg.LookupEnvironment("PRODUCTION") + assert.True(t, ok) + assert.Equal(t, eventv1.Environment_production, e) + + e, ok = cfg.LookupEnvironment(" staging ") + assert.True(t, ok) + assert.Equal(t, eventv1.Environment_preproduction, e) + + _, ok = cfg.LookupEnvironment("") + assert.False(t, ok) + + _, ok = cfg.LookupEnvironment("review") + assert.False(t, ok) +} + +func TestLogAttrsHidesSecrets(t *testing.T) { + cfg, err := LoadConfig(env(map[string]string{ + EnvGitLabSigningToken: signing, + EnvGitLabSecretToken: "short", + EnvFluxHMACKey: flux32, + })) + require.NoError(t, err) + + s := fmt.Sprint(cfg.LogAttrs()...) + assert.NotContains(t, s, signing) + assert.NotContains(t, s, strings.TrimPrefix(signing, "whsec_")) + assert.NotContains(t, s, flux32) + assert.Contains(t, s, "signing_token") + assert.Contains(t, s, "production=production") +} diff --git a/internal/integrations/doc.go b/internal/integrations/doc.go new file mode 100644 index 0000000..6cc046c --- /dev/null +++ b/internal/integrations/doc.go @@ -0,0 +1,4 @@ +// Package integrations turns GitLab deployment webhooks and Flux +// notification-controller events into normalized deployment observations. +// It is pure: no I/O, no database, no global state. +package integrations diff --git a/internal/integrations/errors.go b/internal/integrations/errors.go new file mode 100644 index 0000000..c65ca6c --- /dev/null +++ b/internal/integrations/errors.go @@ -0,0 +1,50 @@ +package integrations + +import ( + "errors" + "time" +) + +// Authentication errors. Their text is safe to log: it never contains a +// secret, a signature or a header value. +var ( + ErrMissingSignature = errors.New("missing signature") + ErrInvalidSignature = errors.New("invalid signature") + ErrStaleTimestamp = errors.New("timestamp outside tolerance") +) + +// Reasons sent back with a 202 response. +const ( + ReasonUnsupportedEvent = "unsupported event" + ReasonUnsupportedObjectKind = "unsupported object_kind" + ReasonUnsupportedKind = "unsupported kind" + ReasonUnsupportedStatus = "unsupported status" + ReasonApprovalIgnored = "approval not tracked" + ReasonUnsupportedReason = "unsupported reason" + ReasonUnmappedEnvironment = "unmapped environment" + ReasonMissingRevision = "missing revision" + ReasonDuplicate = "duplicate" + ReasonStale = "stale" +) + +// IgnoredError marks a valid notification that Tracker does not record (HTTP 202). +type IgnoredError struct{ Reason string } + +func (e *IgnoredError) Error() string { return "ignored: " + e.Reason } + +// InvalidError marks a malformed or incomplete payload (HTTP 400). +type InvalidError struct{ Reason string } + +func (e *InvalidError) Error() string { return "invalid payload: " + e.Reason } + +// CheckFreshness accepts at when it is within tolerance of now, in both directions. +func CheckFreshness(at, now time.Time, tolerance time.Duration) error { + d := now.Sub(at) + if d < 0 { + d = -d + } + if d > tolerance { + return ErrStaleTimestamp + } + return nil +} diff --git a/internal/integrations/errors_test.go b/internal/integrations/errors_test.go new file mode 100644 index 0000000..adca380 --- /dev/null +++ b/internal/integrations/errors_test.go @@ -0,0 +1,44 @@ +package integrations + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCheckFreshness(t *testing.T) { + now := time.Unix(1790000000, 0) + tol := 5 * time.Minute + + tests := []struct { + name string + at time.Time + want error + }{ + {"at now", now, nil}, + {"at now minus tolerance", now.Add(-tol), nil}, + {"at now plus tolerance", now.Add(tol), nil}, + {"just past tolerance in the past", now.Add(-tol - time.Second), ErrStaleTimestamp}, + {"just past tolerance in the future", now.Add(tol + time.Second), ErrStaleTimestamp}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := CheckFreshness(tt.at, now, tol) + if tt.want == nil { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tt.want) + } + }) + } +} + +func TestErrorTypes(t *testing.T) { + ignored := &IgnoredError{Reason: "x"} + assert.Equal(t, "ignored: x", ignored.Error()) + + invalid := &InvalidError{Reason: "y"} + assert.Equal(t, "invalid payload: y", invalid.Error()) +} From 004dc943198794f1d33d701be2fd7d27e0f1b427 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:08:53 +0200 Subject: [PATCH 02/22] feat(integrations): verify GitLab webhook signatures and secret token Standard Webhooks v1 signatures (HMAC-SHA256 over id.timestamp.body) with a freshness window, and the legacy X-Gitlab-Token compared in constant time. Refs #208 --- internal/integrations/signature_gitlab.go | 88 +++++++ .../integrations/signature_gitlab_test.go | 217 ++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 internal/integrations/signature_gitlab.go create mode 100644 internal/integrations/signature_gitlab_test.go diff --git a/internal/integrations/signature_gitlab.go b/internal/integrations/signature_gitlab.go new file mode 100644 index 0000000..0985580 --- /dev/null +++ b/internal/integrations/signature_gitlab.go @@ -0,0 +1,88 @@ +package integrations + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "net/http" + "strconv" + "strings" + "time" +) + +const ( + HeaderWebhookID = "webhook-id" + HeaderWebhookTimestamp = "webhook-timestamp" + HeaderWebhookSignature = "webhook-signature" + HeaderGitLabToken = "X-Gitlab-Token" // #nosec G101 -- header name, not a credential +) + +// SignGitLab returns the "v1," signature GitLab computes for a delivery. +func SignGitLab(key []byte, id, timestamp string, body []byte) string { + return "v1," + base64.StdEncoding.EncodeToString(gitlabMAC(key, id, timestamp, body)) +} + +func gitlabMAC(key []byte, id, timestamp string, body []byte) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write([]byte(id)) + _, _ = mac.Write([]byte{'.'}) + _, _ = mac.Write([]byte(timestamp)) + _, _ = mac.Write([]byte{'.'}) + _, _ = mac.Write(body) + return mac.Sum(nil) +} + +// VerifyGitLabSignature checks a Standard Webhooks signature (signing token) +// and the freshness of webhook-timestamp. Only v1 entries are considered. +func VerifyGitLabSignature(key []byte, h http.Header, body []byte, now time.Time, tolerance time.Duration) error { + if len(key) == 0 { + return ErrInvalidSignature + } + id := strings.TrimSpace(h.Get(HeaderWebhookID)) + ts := strings.TrimSpace(h.Get(HeaderWebhookTimestamp)) + sigs := strings.TrimSpace(h.Get(HeaderWebhookSignature)) + if id == "" || ts == "" || sigs == "" { + return ErrMissingSignature + } + seconds, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return ErrInvalidSignature + } + expected := gitlabMAC(key, id, ts, body) + matched := false + for _, entry := range strings.Fields(sigs) { + version, value, ok := strings.Cut(entry, ",") + if !ok || version != "v1" { + continue + } + got, err := base64.StdEncoding.DecodeString(value) + if err != nil { + continue + } + if hmac.Equal(got, expected) { + matched = true + } + } + if !matched { + return ErrInvalidSignature + } + return CheckFreshness(time.Unix(seconds, 0), now, tolerance) +} + +// VerifyGitLabSecretToken compares X-Gitlab-Token in constant time. Both +// values are hashed first so that the comparison does not leak the length. +func VerifyGitLabSecretToken(expected, got string) error { + if expected == "" { + return ErrInvalidSignature + } + if got == "" { + return ErrMissingSignature + } + a := sha256.Sum256([]byte(expected)) + b := sha256.Sum256([]byte(got)) + if subtle.ConstantTimeCompare(a[:], b[:]) != 1 { + return ErrInvalidSignature + } + return nil +} diff --git a/internal/integrations/signature_gitlab_test.go b/internal/integrations/signature_gitlab_test.go new file mode 100644 index 0000000..aedfafe --- /dev/null +++ b/internal/integrations/signature_gitlab_test.go @@ -0,0 +1,217 @@ +package integrations + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "net/http" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func hdr(id, ts, sig string) http.Header { + h := http.Header{} + if id != "" { + h.Set(HeaderWebhookID, id) + } + if ts != "" { + h.Set(HeaderWebhookTimestamp, ts) + } + if sig != "" { + h.Set(HeaderWebhookSignature, sig) + } + return h +} + +func TestSignGitLab(t *testing.T) { + key := bytes.Repeat([]byte{0x2a}, 32) + id := "msg_2Lh9" + ts := "1790000000" + body := []byte(`{"object_kind":"deployment"}`) + + mac := hmac.New(sha256.New, key) + mac.Write([]byte(id + "." + ts + "." + string(body))) + want := "v1," + base64.StdEncoding.EncodeToString(mac.Sum(nil)) + + assert.Equal(t, want, SignGitLab(key, id, ts, body)) +} + +func TestVerifyGitLabSignature(t *testing.T) { + key := bytes.Repeat([]byte{0x2a}, 32) + id := "msg_2Lh9" + now := time.Unix(1790000000, 0) + ts := "1790000000" + body := []byte(`{"object_kind":"deployment"}`) + tol := 5 * time.Minute + + mac := hmac.New(sha256.New, key) + mac.Write([]byte(id + "." + ts + "." + string(body))) + want := "v1," + base64.StdEncoding.EncodeToString(mac.Sum(nil)) + + tests := []struct { + name string + key []byte + h http.Header + body []byte + now time.Time + tol time.Duration + wantErr error + }{ + { + name: "valid headers", + key: key, h: hdr(id, ts, want), body: body, now: now, tol: tol, + wantErr: nil, + }, + { + name: "multiple entries, one valid", + key: key, h: hdr(id, ts, "v1,AAAA "+want), body: body, now: now, tol: tol, + wantErr: nil, + }, + { + name: "version ignored", + key: key, h: hdr(id, ts, "v2,"+want[3:]), body: body, now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + { + name: "signature not base64", + key: key, h: hdr(id, ts, "v1,!!!notbase64"), body: body, now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + { + name: "body modified", + key: key, h: hdr(id, ts, want), body: []byte(`{"object_kind":"push"}`), now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + { + name: "webhook-id modified", + key: key, h: hdr("msg_other", ts, want), body: body, now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + { + name: "webhook-timestamp changed", + key: key, h: hdr(id, "1790000001", want), body: body, now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + { + name: "timestamp not a number", + key: key, h: hdr(id, "abc", want), body: body, now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + { + name: "missing webhook-id", + key: key, h: hdr("", ts, want), body: body, now: now, tol: tol, + wantErr: ErrMissingSignature, + }, + { + name: "missing webhook-timestamp", + key: key, h: hdr(id, "", want), body: body, now: now, tol: tol, + wantErr: ErrMissingSignature, + }, + { + name: "missing webhook-signature", + key: key, h: hdr(id, ts, ""), body: body, now: now, tol: tol, + wantErr: ErrMissingSignature, + }, + { + name: "nil key", + key: nil, h: hdr(id, ts, want), body: body, now: now, tol: tol, + wantErr: ErrInvalidSignature, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := VerifyGitLabSignature(tt.key, tt.h, tt.body, tt.now, tt.tol) + if tt.wantErr == nil { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tt.wantErr) + } + }) + } +} + +func TestVerifyGitLabSignatureFreshness(t *testing.T) { + key := bytes.Repeat([]byte{0x2a}, 32) + id := "msg_2Lh9" + now := time.Unix(1790000000, 0) + body := []byte(`{"object_kind":"deployment"}`) + tol := 5 * time.Minute + + sign := func(ts string) (string, string) { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(id + "." + ts + "." + string(body))) + return ts, "v1," + base64.StdEncoding.EncodeToString(mac.Sum(nil)) + } + + tests := []struct { + name string + at time.Time + wantErr error + }{ + {"now minus tolerance", now.Add(-tol), nil}, + {"now minus tolerance minus one second", now.Add(-tol - time.Second), ErrStaleTimestamp}, + {"now plus tolerance plus one second", now.Add(tol + time.Second), ErrStaleTimestamp}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts, sig := sign(strconv.FormatInt(tt.at.Unix(), 10)) + err := VerifyGitLabSignature(key, hdr(id, ts, sig), body, now, tol) + if tt.wantErr == nil { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tt.wantErr) + } + }) + } +} + +// TestVerifyGitLabSignatureReferenceVector uses the Standard Webhooks +// reference test vector (svix test suite, standard-webhooks/standard-webhooks +// repository) to check SignGitLab and VerifyGitLabSignature against an +// implementation-independent value. +func TestVerifyGitLabSignatureReferenceVector(t *testing.T) { + secret := "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" + key, err := base64.StdEncoding.DecodeString(secret[len("whsec_"):]) + assert.NoError(t, err) + + id := "msg_p5jXN8AQM9LWM0D4loKWxJek" + ts := "1614265330" + body := []byte(`{"test": 2432232314}`) + now := time.Unix(1614265330, 0) + want := "v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=" + + assert.Equal(t, want, SignGitLab(key, id, ts, body)) + assert.NoError(t, VerifyGitLabSignature(key, hdr(id, ts, want), body, now, 5*time.Minute)) +} + +func TestVerifyGitLabSecretToken(t *testing.T) { + tests := []struct { + name string + expected string + got string + wantErr error + }{ + {"match", "gitlab-secret-token-0123", "gitlab-secret-token-0123", nil}, + {"mismatch", "gitlab-secret-token-0123", "wrong", ErrInvalidSignature}, + {"got empty", "gitlab-secret-token-0123", "", ErrMissingSignature}, + {"expected empty", "", "x", ErrInvalidSignature}, + {"longer value with correct prefix", "gitlab-secret-token-0123", "gitlab-secret-token-0123x", ErrInvalidSignature}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := VerifyGitLabSecretToken(tt.expected, tt.got) + if tt.wantErr == nil { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tt.wantErr) + } + }) + } +} From dc0331947f4ab2cee1b58bff5f367810b69ad398 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:12:42 +0200 Subject: [PATCH 03/22] feat(integrations): verify Flux generic-hmac signatures Accept X-Signature with sha224, sha256, sha384 or sha512, compared with hmac.Equal over the raw body. Refs #208 --- internal/integrations/signature_flux.go | 61 +++++++++++++ internal/integrations/signature_flux_test.go | 90 ++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 internal/integrations/signature_flux.go create mode 100644 internal/integrations/signature_flux_test.go diff --git a/internal/integrations/signature_flux.go b/internal/integrations/signature_flux.go new file mode 100644 index 0000000..554a5fa --- /dev/null +++ b/internal/integrations/signature_flux.go @@ -0,0 +1,61 @@ +package integrations + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "fmt" + "hash" + "strings" +) + +const HeaderFluxSignature = "X-Signature" + +// fluxHashes are the HMAC algorithms the Flux generic-hmac provider can use. +var fluxHashes = map[string]func() hash.Hash{ + "sha224": sha256.New224, + "sha256": sha256.New, + "sha384": sha512.New384, + "sha512": sha512.New, +} + +// SignFlux returns the X-Signature value Flux sends for body. +func SignFlux(alg string, key, body []byte) (string, error) { + newHash, ok := fluxHashes[alg] + if !ok { + return "", fmt.Errorf("unsupported algorithm %q", alg) + } + mac := hmac.New(newHash, key) + _, _ = mac.Write(body) + return alg + "=" + hex.EncodeToString(mac.Sum(nil)), nil +} + +// VerifyFluxSignature checks X-Signature: = over the raw body. +func VerifyFluxSignature(key []byte, header string, body []byte) error { + if len(key) == 0 { + return ErrInvalidSignature + } + header = strings.TrimSpace(header) + if header == "" { + return ErrMissingSignature + } + alg, value, ok := strings.Cut(header, "=") + if !ok { + return ErrInvalidSignature + } + newHash, ok := fluxHashes[strings.ToLower(strings.TrimSpace(alg))] + if !ok { + return ErrInvalidSignature + } + got, err := hex.DecodeString(strings.TrimSpace(value)) + if err != nil { + return ErrInvalidSignature + } + mac := hmac.New(newHash, key) + _, _ = mac.Write(body) + if !hmac.Equal(got, mac.Sum(nil)) { + return ErrInvalidSignature + } + return nil +} diff --git a/internal/integrations/signature_flux_test.go b/internal/integrations/signature_flux_test.go new file mode 100644 index 0000000..baf514f --- /dev/null +++ b/internal/integrations/signature_flux_test.go @@ -0,0 +1,90 @@ +package integrations + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "hash" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func fluxMAC(newHash func() hash.Hash, key, body []byte) string { + mac := hmac.New(newHash, key) + mac.Write(body) + return hex.EncodeToString(mac.Sum(nil)) +} + +func TestSignFlux(t *testing.T) { + key := []byte(strings.Repeat("k", 32)) + body := []byte(`{"reason":"ReconciliationSucceeded"}`) + + tests := []struct { + alg string + newHash func() hash.Hash + }{ + {"sha224", sha256.New224}, + {"sha256", sha256.New}, + {"sha384", sha512.New384}, + {"sha512", sha512.New}, + } + + for _, tt := range tests { + t.Run(tt.alg, func(t *testing.T) { + want := tt.alg + "=" + fluxMAC(tt.newHash, key, body) + got, err := SignFlux(tt.alg, key, body) + assert.NoError(t, err) + assert.Equal(t, want, got) + }) + } + + t.Run("unsupported algorithm", func(t *testing.T) { + _, err := SignFlux("sha1", key, body) + assert.Error(t, err) + }) +} + +func TestVerifyFluxSignature(t *testing.T) { + key := []byte(strings.Repeat("k", 32)) + body := []byte(`{"reason":"ReconciliationSucceeded"}`) + otherBody := []byte(`{"reason":"ReconciliationFailed"}`) + otherKey := []byte(strings.Repeat("o", 32)) + + sha256Hex := fluxMAC(sha256.New, key, body) + + tests := []struct { + name string + key []byte + header string + body []byte + wantErr error + }{ + {"sha224", key, "sha224=" + fluxMAC(sha256.New224, key, body), body, nil}, + {"sha256", key, "sha256=" + sha256Hex, body, nil}, + {"sha384", key, "sha384=" + fluxMAC(sha512.New384, key, body), body, nil}, + {"sha512", key, "sha512=" + fluxMAC(sha512.New, key, body), body, nil}, + {"algorithm name case insensitive", key, "SHA256=" + sha256Hex, body, nil}, + {"algorithm too weak", key, "sha1=" + sha256Hex, body, ErrInvalidSignature}, + {"algorithm unknown", key, "md5=00", body, ErrInvalidSignature}, + {"hex value invalid", key, "sha256=zz", body, ErrInvalidSignature}, + {"no separator", key, "sha256", body, ErrInvalidSignature}, + {"wrong key", otherKey, "sha256=" + sha256Hex, body, ErrInvalidSignature}, + {"body modified", key, "sha256=" + sha256Hex, otherBody, ErrInvalidSignature}, + {"missing header", key, "", body, ErrMissingSignature}, + {"nil key", nil, "sha256=" + sha256Hex, body, ErrInvalidSignature}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := VerifyFluxSignature(tt.key, tt.header, tt.body) + if tt.wantErr == nil { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tt.wantErr) + } + }) + } +} From 49e7bf0e1bdbd8fbabd7a49ca33bfdf5df25d149 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:17:52 +0200 Subject: [PATCH 04/22] feat(integrations): add deployment observation and repository URL normalization Observation is the normalized form of a GitLab or Flux notification; Rank orders statuses for equal timestamps; NormalizeRepoURL makes SSH and HTTP repository URLs comparable. Refs #208 --- internal/integrations/observation.go | 67 +++++++++++++++++++++++ internal/integrations/observation_test.go | 49 +++++++++++++++++ internal/integrations/repository.go | 45 +++++++++++++++ internal/integrations/repository_test.go | 36 ++++++++++++ 4 files changed, 197 insertions(+) create mode 100644 internal/integrations/observation.go create mode 100644 internal/integrations/observation_test.go create mode 100644 internal/integrations/repository.go create mode 100644 internal/integrations/repository_test.go diff --git a/internal/integrations/observation.go b/internal/integrations/observation.go new file mode 100644 index 0000000..5828d78 --- /dev/null +++ b/internal/integrations/observation.go @@ -0,0 +1,67 @@ +package integrations + +import ( + "strings" + "time" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +// Source names, stored in attributes.source and in integration_deployments. +const ( + SourceGitLab = "gitlab" + SourceFlux = "flux" +) + +// ServiceHint carries what the payload says about the service. The server +// resolves it against the catalog. +type ServiceHint struct { + // RepositoryURLs are already normalized with NormalizeRepoURL. + RepositoryURLs []string + // Name is the fallback service name. + Name string +} + +// Observation is one deployment notification, normalized. +type Observation struct { + Key string + Source string + Status eventv1.Status + At time.Time + Terminal bool + Environment eventv1.Environment + Service ServiceHint + ShortRevision string + Message string + Owner string + User string + Comment string +} + +// Title builds "Deploy to ". +func (o Observation) Title(service string) string { + parts := []string{"Deploy"} + for _, p := range []string{service, o.ShortRevision} { + if p != "" { + parts = append(parts, p) + } + } + parts = append(parts, "to", o.Environment.String()) + return strings.Join(parts, " ") +} + +// Rank orders statuses for the same instant: 1 for start and +// waiting_approval, 2 for terminal statuses, 0 for anything else. +func Rank(s eventv1.Status) int { + switch s { + case eventv1.Status_start, eventv1.Status_waiting_approval: + return 1 + case eventv1.Status_success, eventv1.Status_failure, eventv1.Status_warning, eventv1.Status_close: + return 2 + default: + return 0 + } +} + +// IsTerminal reports whether s ends a deployment. +func IsTerminal(s eventv1.Status) bool { return Rank(s) == 2 } diff --git a/internal/integrations/observation_test.go b/internal/integrations/observation_test.go new file mode 100644 index 0000000..65016bb --- /dev/null +++ b/internal/integrations/observation_test.go @@ -0,0 +1,49 @@ +package integrations + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +func TestRank(t *testing.T) { + cases := []struct { + status eventv1.Status + want int + }{ + {eventv1.Status_start, 1}, + {eventv1.Status_waiting_approval, 1}, + {eventv1.Status_success, 2}, + {eventv1.Status_failure, 2}, + {eventv1.Status_warning, 2}, + {eventv1.Status_close, 2}, + {eventv1.Status_in_progress, 0}, + {eventv1.Status_done, 0}, + } + + for _, tt := range cases { + t.Run(tt.status.String(), func(t *testing.T) { + assert.Equal(t, tt.want, Rank(tt.status)) + assert.Equal(t, tt.want == 2, IsTerminal(tt.status)) + }) + } +} + +func TestTitle(t *testing.T) { + t.Run("with revision", func(t *testing.T) { + o := Observation{ShortRevision: "a1b2c3d4", Environment: eventv1.Environment_production} + assert.Equal(t, "Deploy payments a1b2c3d4 to production", o.Title("payments")) + }) + + t.Run("without revision", func(t *testing.T) { + o := Observation{Environment: eventv1.Environment_production} + assert.Equal(t, "Deploy payments to production", o.Title("payments")) + }) + + t.Run("preproduction environment", func(t *testing.T) { + o := Observation{ShortRevision: "a1b2c3d4", Environment: eventv1.Environment_preproduction} + assert.Equal(t, "Deploy payments a1b2c3d4 to preproduction", o.Title("payments")) + }) +} diff --git a/internal/integrations/repository.go b/internal/integrations/repository.go new file mode 100644 index 0000000..bbb026b --- /dev/null +++ b/internal/integrations/repository.go @@ -0,0 +1,45 @@ +package integrations + +import ( + "net/url" + "regexp" + "strings" +) + +var scpLikeRepo = regexp.MustCompile(`^[A-Za-z0-9._-]+@([A-Za-z0-9.-]+):(.+)$`) + +// NormalizeRepoURL turns the SSH and HTTP forms of a repository URL into +// https://host/path: scheme and host lower case, path kept as is, trailing +// slashes then the .git suffix removed, credentials dropped. It returns "" +// for anything it cannot parse, which then matches nothing. +func NormalizeRepoURL(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + var scheme, host, path string + if m := scpLikeRepo.FindStringSubmatch(raw); m != nil && !strings.Contains(raw, "://") { + scheme, host, path = "https", m[1], m[2] + } else { + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return "" + } + switch strings.ToLower(u.Scheme) { + case "ssh", "git+ssh": + scheme, host = "https", u.Hostname() + case "http", "https": + scheme, host = strings.ToLower(u.Scheme), u.Host + default: + return "" + } + path = u.Path + } + path = strings.TrimLeft(path, "/") + path = strings.TrimRight(path, "/") + path = strings.TrimSuffix(path, ".git") + if host == "" || path == "" { + return "" + } + return scheme + "://" + strings.ToLower(host) + "/" + path +} diff --git a/internal/integrations/repository_test.go b/internal/integrations/repository_test.go new file mode 100644 index 0000000..537df35 --- /dev/null +++ b/internal/integrations/repository_test.go @@ -0,0 +1,36 @@ +package integrations + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNormalizeRepoURL(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"scp-like uppercase host", "git@GitLab.Example.com:team/payments.git", "https://gitlab.example.com/team/payments"}, + {"ssh scheme with port", "ssh://git@gitlab.example.com:2222/team/payments.git", "https://gitlab.example.com/team/payments"}, + {"https trailing slash", "https://GitLab.Example.com/Team/Payments/", "https://gitlab.example.com/Team/Payments"}, + {"https git suffix and trailing slash", "https://gitlab.example.com/team/payments.git/", "https://gitlab.example.com/team/payments"}, + {"uppercase scheme", "HTTPS://gitlab.example.com/team/payments", "https://gitlab.example.com/team/payments"}, + {"credentials dropped", "https://oauth2:tok@gitlab.example.com/team/payments.git", "https://gitlab.example.com/team/payments"}, + {"custom port kept", "https://gitlab.example.com:8443/team/payments", "https://gitlab.example.com:8443/team/payments"}, + {"http scheme kept", "http://gitlab.local/team/payments", "http://gitlab.local/team/payments"}, + {"scp-like nested path with whitespace", " git@host:group/sub/proj ", "https://host/group/sub/proj"}, + {"empty", "", ""}, + {"not a url", "not a url", ""}, + {"unsupported scheme", "ftp://host/x", ""}, + {"no path", "https://gitlab.example.com", ""}, + {"root path only", "https://gitlab.example.com/", ""}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, NormalizeRepoURL(tt.in)) + }) + } +} From abd2bcd4fb0edfcc5390ada1b60b99e9ced05036 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:23:52 +0200 Subject: [PATCH 05/22] feat(integrations): map GitLab deployment events to observations Deployment Hook payloads become observations keyed by instance host and deployment id, with the environment mapping, the status table, the title and the message of the Tracker event. Refs #208 --- internal/integrations/gitlab.go | 231 ++++++++++++++++++++++++ internal/integrations/gitlab_test.go | 258 +++++++++++++++++++++++++++ 2 files changed, 489 insertions(+) create mode 100644 internal/integrations/gitlab.go create mode 100644 internal/integrations/gitlab_test.go diff --git a/internal/integrations/gitlab.go b/internal/integrations/gitlab.go new file mode 100644 index 0000000..06be6be --- /dev/null +++ b/internal/integrations/gitlab.go @@ -0,0 +1,231 @@ +package integrations + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +// Headers and event name for GitLab "Deployment Hook" webhooks. +const ( + HeaderGitLabEvent = "X-Gitlab-Event" + HeaderGitLabInstance = "X-Gitlab-Instance" + GitLabDeploymentHook = "Deployment Hook" +) + +// gitlabDeployment is the subset of a GitLab "Deployment Hook" payload +// ParseGitLab needs. +type gitlabDeployment struct { + ObjectKind string `json:"object_kind"` + Status string `json:"status"` + StatusChangedAt string `json:"status_changed_at"` + DeploymentID int64 `json:"deployment_id"` + DeployableURL string `json:"deployable_url"` + Environment string `json:"environment"` + EnvironmentTier string `json:"environment_tier"` + EnvironmentExternalURL string `json:"environment_external_url"` + ShortSHA string `json:"short_sha"` + CommitURL string `json:"commit_url"` + CommitTitle string `json:"commit_title"` + User struct { + Username string `json:"username"` + } `json:"user"` + Project struct { + WebURL string `json:"web_url"` + GitHTTPURL string `json:"git_http_url"` + PathWithNamespace string `json:"path_with_namespace"` + } `json:"project"` +} + +// gitlabStatus maps a GitLab deployment status to a Tracker status and its +// optional comment. +type gitlabStatus struct { + status eventv1.Status + comment string +} + +var gitlabStatuses = map[string]gitlabStatus{ + "running": {status: eventv1.Status_start}, + "success": {status: eventv1.Status_success}, + "failed": {status: eventv1.Status_failure}, + "canceled": {status: eventv1.Status_warning, comment: "Deployment canceled"}, + "blocked": {status: eventv1.Status_waiting_approval}, + "rejected": {status: eventv1.Status_close}, +} + +// gitlabTimeLayouts are tried in order to parse status_changed_at. RFC3339 +// also accepts the fractional seconds GitLab documents +// (2021-04-28T21:50:00.000+02:00), since Go parses an optional fractional +// second even when the layout does not include one. +var gitlabTimeLayouts = []string{time.RFC3339, "2006-01-02 15:04:05 -0700", "2006-01-02 15:04:05 MST"} + +// ParseGitLab maps a GitLab "Deployment Hook" payload to an Observation. +// eventHeader and instanceHeader are the X-Gitlab-Event and X-Gitlab-Instance +// header values. It returns *IgnoredError for a notification Tracker does +// not record, and *InvalidError for a malformed or incomplete payload. +func ParseGitLab(eventHeader, instanceHeader string, body []byte, cfg Config) (Observation, error) { + if strings.TrimSpace(eventHeader) != GitLabDeploymentHook { + return Observation{}, &IgnoredError{Reason: ReasonUnsupportedEvent} + } + + var p gitlabDeployment + if err := json.Unmarshal(body, &p); err != nil { + return Observation{}, &InvalidError{Reason: "body is not valid JSON"} + } + + if p.ObjectKind != "deployment" { + return Observation{}, &IgnoredError{Reason: ReasonUnsupportedObjectKind} + } + + if p.DeploymentID <= 0 { + return Observation{}, &InvalidError{Reason: "missing field deployment_id"} + } + if strings.TrimSpace(p.Status) == "" { + return Observation{}, &InvalidError{Reason: "missing field status"} + } + if strings.TrimSpace(p.StatusChangedAt) == "" { + return Observation{}, &InvalidError{Reason: "missing field status_changed_at"} + } + if strings.TrimSpace(p.Environment) == "" { + return Observation{}, &InvalidError{Reason: "missing field environment"} + } + pathWithNamespace := strings.TrimSpace(p.Project.PathWithNamespace) + if pathWithNamespace == "" { + return Observation{}, &InvalidError{Reason: "missing field project.path_with_namespace"} + } + + at, err := parseGitLabTime(p.StatusChangedAt) + if err != nil { + return Observation{}, &InvalidError{Reason: "status_changed_at is not a valid date"} + } + + host := gitlabHost(instanceHeader, p.Project.WebURL) + if host == "" { + return Observation{}, &InvalidError{Reason: "cannot determine the GitLab instance host"} + } + + environment, ok := lookupGitLabEnvironment(cfg, p.Environment, p.EnvironmentTier) + if !ok { + return Observation{}, &IgnoredError{Reason: ReasonUnmappedEnvironment} + } + + if p.Status == "approved" { + return Observation{}, &IgnoredError{Reason: ReasonApprovalIgnored} + } + status, ok := gitlabStatuses[p.Status] + if !ok { + return Observation{}, &IgnoredError{Reason: ReasonUnsupportedStatus} + } + + owner := strings.TrimSpace(p.User.Username) + user := "gitlab:" + owner + if owner == "" { + owner = "gitlab" + user = "gitlab:unknown" + } + + return Observation{ + Key: fmt.Sprintf("gitlab:%s:%d", host, p.DeploymentID), + Source: SourceGitLab, + Status: status.status, + At: at, + Terminal: IsTerminal(status.status), + Environment: environment, + Service: gitlabServiceHint(p), + ShortRevision: strings.TrimSpace(p.ShortSHA), + Message: gitlabMessage(p), + Owner: owner, + User: user, + Comment: status.comment, + }, nil +} + +// parseGitLabTime parses v with the first layout in gitlabTimeLayouts that +// matches, and returns it in UTC. +func parseGitLabTime(v string) (time.Time, error) { + for _, layout := range gitlabTimeLayouts { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC(), nil + } + } + return time.Time{}, fmt.Errorf("status_changed_at is not a valid date") +} + +// gitlabHost returns the lower-cased host of the first of instance and +// webURL that parses to a non-empty host. +func gitlabHost(instance, webURL string) string { + for _, v := range []string{instance, webURL} { + v = strings.TrimSpace(v) + if v == "" { + continue + } + u, err := url.Parse(v) + if err != nil || u.Host == "" { + continue + } + return strings.ToLower(u.Host) + } + return "" +} + +// lookupGitLabEnvironment resolves a GitLab environment to a Tracker +// environment, first from the segment of env before "/", then from tier. +func lookupGitLabEnvironment(cfg Config, env, tier string) (eventv1.Environment, bool) { + first, _, _ := strings.Cut(env, "/") + if e, ok := cfg.LookupEnvironment(first); ok { + return e, true + } + return cfg.LookupEnvironment(tier) +} + +// gitlabServiceHint builds the ServiceHint from the project fields: +// deduplicated, normalized repository URLs, and the last path segment as +// the fallback service name. +func gitlabServiceHint(p gitlabDeployment) ServiceHint { + var urls []string + seen := make(map[string]bool) + for _, raw := range []string{p.Project.WebURL, p.Project.GitHTTPURL} { + u := NormalizeRepoURL(raw) + if u == "" || seen[u] { + continue + } + seen[u] = true + urls = append(urls, u) + } + + path := strings.Trim(p.Project.PathWithNamespace, "/") + name := path + if idx := strings.LastIndex(path, "/"); idx >= 0 { + name = path[idx+1:] + } + + return ServiceHint{RepositoryURLs: urls, Name: name} +} + +// gitlabMessage joins the non-empty lines describing the deployment. +func gitlabMessage(p gitlabDeployment) string { + lines := []string{p.CommitTitle} + if v := strings.TrimSpace(p.ShortSHA); v != "" { + lines = append(lines, "Commit: "+v) + } + lines = append(lines, p.CommitURL) + if v := strings.TrimSpace(p.DeployableURL); v != "" { + lines = append(lines, "Job: "+v) + } + lines = append(lines, "GitLab environment: "+p.Environment) + if v := strings.TrimSpace(p.EnvironmentExternalURL); v != "" { + lines = append(lines, "URL: "+v) + } + + nonEmpty := lines[:0] + for _, l := range lines { + if l != "" { + nonEmpty = append(nonEmpty, l) + } + } + return strings.Join(nonEmpty, "\n") +} diff --git a/internal/integrations/gitlab_test.go b/internal/integrations/gitlab_test.go new file mode 100644 index 0000000..d444844 --- /dev/null +++ b/internal/integrations/gitlab_test.go @@ -0,0 +1,258 @@ +package integrations + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +func gitlabBody(t *testing.T, mutate func(m map[string]any)) []byte { + t.Helper() + m := map[string]any{ + "object_kind": "deployment", "status": "running", + "status_changed_at": "2026-09-28 10:00:00 +0200", "deployment_id": 42, + "deployable_url": "https://gitlab.example.com/team/payments/-/jobs/7", + "environment": "production", "environment_tier": "production", + "environment_external_url": "https://payments.example.com", + "short_sha": "a1b2c3d4", "commit_title": "Fix rounding", + "commit_url": "https://gitlab.example.com/team/payments/-/commit/a1b2c3d4e5", + "user": map[string]any{"username": "jdoe"}, + "project": map[string]any{ + "web_url": "https://gitlab.example.com/team/payments", + "git_http_url": "https://gitlab.example.com/team/payments.git", + "path_with_namespace": "team/payments", + }, + } + if mutate != nil { + mutate(m) + } + b, err := json.Marshal(m) + require.NoError(t, err) + return b +} + +func gitlabConfig(t *testing.T) Config { + t.Helper() + cfg, err := LoadConfig(func(string) (string, bool) { return "", false }) + require.NoError(t, err) + return cfg +} + +func TestParseGitLab(t *testing.T) { + t.Run("nominal", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, nil) + + obs, err := ParseGitLab(GitLabDeploymentHook, "https://GitLab.Example.com", body, cfg) + require.NoError(t, err) + + assert.Equal(t, "gitlab:gitlab.example.com:42", obs.Key) + assert.Equal(t, SourceGitLab, obs.Source) + assert.Equal(t, eventv1.Status_start, obs.Status) + assert.False(t, obs.Terminal) + assert.True(t, obs.At.Equal(time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC))) + assert.Equal(t, eventv1.Environment_production, obs.Environment) + assert.Equal(t, ServiceHint{ + RepositoryURLs: []string{"https://gitlab.example.com/team/payments"}, + Name: "payments", + }, obs.Service) + assert.Equal(t, "a1b2c3d4", obs.ShortRevision) + assert.Equal(t, "jdoe", obs.Owner) + assert.Equal(t, "gitlab:jdoe", obs.User) + assert.Equal(t, "", obs.Comment) + assert.Equal(t, "Fix rounding\nCommit: a1b2c3d4\nhttps://gitlab.example.com/team/payments/-/commit/a1b2c3d4e5\nJob: https://gitlab.example.com/team/payments/-/jobs/7\nGitLab environment: production\nURL: https://payments.example.com", obs.Message) + assert.Equal(t, "Deploy payments a1b2c3d4 to production", obs.Title("payments")) + }) + + t.Run("empty instance header falls back to project.web_url host", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, nil) + + obs, err := ParseGitLab(GitLabDeploymentHook, "", body, cfg) + require.NoError(t, err) + assert.Equal(t, "gitlab:gitlab.example.com:42", obs.Key) + }) + + t.Run("status_changed_at as RFC3339", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, func(m map[string]any) { + m["status_changed_at"] = "2026-09-28T08:00:00Z" + }) + + obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + require.NoError(t, err) + assert.True(t, obs.At.Equal(time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC))) + }) + + t.Run("status table", func(t *testing.T) { + cfg := gitlabConfig(t) + + cases := []struct { + status string + want eventv1.Status + terminal bool + comment string + }{ + {"running", eventv1.Status_start, false, ""}, + {"success", eventv1.Status_success, true, ""}, + {"failed", eventv1.Status_failure, true, ""}, + {"canceled", eventv1.Status_warning, true, "Deployment canceled"}, + {"blocked", eventv1.Status_waiting_approval, false, ""}, + {"rejected", eventv1.Status_close, true, ""}, + } + + for _, tt := range cases { + t.Run(tt.status, func(t *testing.T) { + body := gitlabBody(t, func(m map[string]any) { m["status"] = tt.status }) + obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + require.NoError(t, err) + assert.Equal(t, tt.want, obs.Status) + assert.Equal(t, tt.terminal, obs.Terminal) + assert.Equal(t, tt.comment, obs.Comment) + }) + } + + t.Run("approved", func(t *testing.T) { + body := gitlabBody(t, func(m map[string]any) { m["status"] = "approved" }) + _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + var ig *IgnoredError + require.ErrorAs(t, err, &ig) + assert.Equal(t, ReasonApprovalIgnored, ig.Reason) + }) + + t.Run("created", func(t *testing.T) { + body := gitlabBody(t, func(m map[string]any) { m["status"] = "created" }) + _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + var ig *IgnoredError + require.ErrorAs(t, err, &ig) + assert.Equal(t, ReasonUnsupportedStatus, ig.Reason) + }) + }) + + t.Run("environments", func(t *testing.T) { + cfg := gitlabConfig(t) + + mapped := []struct { + name string + environment string + tier string + want eventv1.Environment + }{ + {"production/eu", "production/eu", "production", eventv1.Environment_production}, + {"production-eu with production tier", "production-eu", "production", eventv1.Environment_production}, + {"staging", "staging", "staging", eventv1.Environment_preproduction}, + {"Production", "Production", "Production", eventv1.Environment_production}, + } + for _, tt := range mapped { + t.Run(tt.name, func(t *testing.T) { + body := gitlabBody(t, func(m map[string]any) { + m["environment"] = tt.environment + m["environment_tier"] = tt.tier + }) + obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + require.NoError(t, err) + assert.Equal(t, tt.want, obs.Environment) + }) + } + + unmapped := []struct { + name string + environment string + tier string + }{ + {"review/feature-x with development tier", "review/feature-x", "development"}, + {"integration with testing tier", "integration", "testing"}, + } + for _, tt := range unmapped { + t.Run(tt.name, func(t *testing.T) { + body := gitlabBody(t, func(m map[string]any) { + m["environment"] = tt.environment + m["environment_tier"] = tt.tier + }) + _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + var ig *IgnoredError + require.ErrorAs(t, err, &ig) + assert.Equal(t, ReasonUnmappedEnvironment, ig.Reason) + }) + } + }) + + t.Run("unsupported event", func(t *testing.T) { + cfg := gitlabConfig(t) + _, err := ParseGitLab("Push Hook", "https://gitlab.example.com", []byte("{"), cfg) + var ig *IgnoredError + require.ErrorAs(t, err, &ig) + assert.Equal(t, ReasonUnsupportedEvent, ig.Reason) + }) + + t.Run("unsupported object_kind", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, func(m map[string]any) { m["object_kind"] = "build" }) + _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + var ig *IgnoredError + require.ErrorAs(t, err, &ig) + assert.Equal(t, ReasonUnsupportedObjectKind, ig.Reason) + }) + + t.Run("invalid JSON body", func(t *testing.T) { + cfg := gitlabConfig(t) + _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", []byte("{"), cfg) + var iv *InvalidError + require.ErrorAs(t, err, &iv) + }) + + t.Run("required fields", func(t *testing.T) { + cfg := gitlabConfig(t) + + mutations := []func(m map[string]any){ + func(m map[string]any) { delete(m, "deployment_id") }, + func(m map[string]any) { m["status"] = "" }, + func(m map[string]any) { m["status_changed_at"] = "" }, + func(m map[string]any) { m["environment"] = "" }, + func(m map[string]any) { m["project"].(map[string]any)["path_with_namespace"] = "" }, + func(m map[string]any) { m["status_changed_at"] = "yesterday" }, + } + for _, mutate := range mutations { + body := gitlabBody(t, mutate) + _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + var iv *InvalidError + require.ErrorAs(t, err, &iv) + } + }) + + t.Run("no instance host determinable", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, func(m map[string]any) { + m["project"].(map[string]any)["web_url"] = "" + }) + _, err := ParseGitLab(GitLabDeploymentHook, "", body, cfg) + var iv *InvalidError + require.ErrorAs(t, err, &iv) + }) + + t.Run("empty username falls back to unknown", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, func(m map[string]any) { + m["user"].(map[string]any)["username"] = "" + }) + obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + require.NoError(t, err) + assert.Equal(t, "gitlab", obs.Owner) + assert.Equal(t, "gitlab:unknown", obs.User) + }) + + t.Run("null environment_external_url omits URL line", func(t *testing.T) { + cfg := gitlabConfig(t) + body := gitlabBody(t, func(m map[string]any) { + m["environment_external_url"] = nil + }) + obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + require.NoError(t, err) + assert.NotContains(t, obs.Message, "URL:") + }) +} From 99816316cb6b721c72ba1a5f381bef1681af21f4 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:35:39 +0200 Subject: [PATCH 06/22] feat(integrations): map Flux notifications to observations Kustomization and HelmRelease events become observations keyed by environment, object and revision, with the severity and reason table, the freshness check on the signed timestamp and the short revision. Refs #208 --- internal/integrations/flux.go | 197 ++++++++++++++++ internal/integrations/flux_test.go | 360 +++++++++++++++++++++++++++++ 2 files changed, 557 insertions(+) create mode 100644 internal/integrations/flux.go create mode 100644 internal/integrations/flux_test.go diff --git a/internal/integrations/flux.go b/internal/integrations/flux.go new file mode 100644 index 0000000..44d99bb --- /dev/null +++ b/internal/integrations/flux.go @@ -0,0 +1,197 @@ +package integrations + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +const fluxEventMetadataPrefix = "event.toolkit.fluxcd.io/" + +var fluxRevisionKeys = map[string]string{ + "Kustomization": "kustomize.toolkit.fluxcd.io/revision", + "HelmRelease": "helm.toolkit.fluxcd.io/revision", +} + +// fluxReasons maps a Flux notification severity and reason to a Tracker +// status. Verified against fluxcd/kustomize-controller and +// fluxcd/helm-controller (see flux_test.go and the task report for sources). +var fluxReasons = map[string]map[string]eventv1.Status{ + "info": { + "Progressing": eventv1.Status_start, + "ReconciliationSucceeded": eventv1.Status_success, + "InstallSucceeded": eventv1.Status_success, + "UpgradeSucceeded": eventv1.Status_success, + // helm-controller emits RollbackSucceeded with corev1.EventTypeNormal + // (severity info, not error: see rollback_remediation.go and + // fluxcd/pkg runtime/events recorder.go). A rollback still means the + // preceding upgrade failed, so it is kept as a failure. + "RollbackSucceeded": eventv1.Status_failure, + }, + "error": { + "ReconciliationFailed": eventv1.Status_failure, + "HealthCheckFailed": eventv1.Status_failure, + "BuildFailed": eventv1.Status_failure, + "ValidationFailed": eventv1.Status_failure, + "ArtifactFailed": eventv1.Status_failure, + "InstallFailed": eventv1.Status_failure, + "UpgradeFailed": eventv1.Status_failure, + "TestFailed": eventv1.Status_failure, + }, +} + +type fluxEvent struct { + InvolvedObject struct { + Kind string `json:"kind"` + Namespace string `json:"namespace"` + Name string `json:"name"` + } `json:"involvedObject"` + Severity string `json:"severity"` + Timestamp string `json:"timestamp"` + Message string `json:"message"` + Reason string `json:"reason"` + Metadata map[string]string `json:"metadata"` +} + +// fluxMetadata reads key, then event.toolkit.fluxcd.io/key. In practice a +// genuine Flux notification only ever carries the bare key: per RFC-0008, +// notification-controller strips the event.toolkit.fluxcd.io/ prefix from +// object annotations before dispatching to any provider, generic-hmac +// included. The prefixed fallback is kept defensively. +func fluxMetadata(m map[string]string, key string) string { + if v := strings.TrimSpace(m[key]); v != "" { + return v + } + return strings.TrimSpace(m[fluxEventMetadataPrefix+key]) +} + +// ParseFlux maps a Flux notification-controller event to an Observation. It +// returns *InvalidError for a malformed or incomplete payload, +// ErrStaleTimestamp when the event timestamp is outside cfg.Tolerance, and +// *IgnoredError for a notification Tracker does not record. +func ParseFlux(body []byte, cfg Config, now time.Time) (Observation, error) { + var e fluxEvent + if err := json.Unmarshal(body, &e); err != nil { + return Observation{}, &InvalidError{Reason: "body is not valid JSON"} + } + + if strings.TrimSpace(e.InvolvedObject.Kind) == "" { + return Observation{}, &InvalidError{Reason: "missing field involvedObject.kind"} + } + if strings.TrimSpace(e.InvolvedObject.Name) == "" { + return Observation{}, &InvalidError{Reason: "missing field involvedObject.name"} + } + if strings.TrimSpace(e.InvolvedObject.Namespace) == "" { + return Observation{}, &InvalidError{Reason: "missing field involvedObject.namespace"} + } + if strings.TrimSpace(e.Severity) == "" { + return Observation{}, &InvalidError{Reason: "missing field severity"} + } + if strings.TrimSpace(e.Reason) == "" { + return Observation{}, &InvalidError{Reason: "missing field reason"} + } + if strings.TrimSpace(e.Timestamp) == "" { + return Observation{}, &InvalidError{Reason: "missing field timestamp"} + } + + at, err := time.Parse(time.RFC3339, e.Timestamp) + if err != nil { + return Observation{}, &InvalidError{Reason: "timestamp is not a valid date"} + } + at = at.UTC() + + if err := CheckFreshness(at, now, cfg.Tolerance); err != nil { + return Observation{}, err + } + + kind := e.InvolvedObject.Kind + revisionKey, ok := fluxRevisionKeys[kind] + if !ok { + return Observation{}, &IgnoredError{Reason: ReasonUnsupportedKind} + } + + status, ok := fluxReasons[e.Severity][e.Reason] + if !ok { + return Observation{}, &IgnoredError{Reason: ReasonUnsupportedReason} + } + + md := e.Metadata + if md == nil { + md = map[string]string{} + } + + environment, ok := cfg.LookupEnvironment(fluxMetadata(md, "environment")) + if !ok { + return Observation{}, &IgnoredError{Reason: ReasonUnmappedEnvironment} + } + + revision := strings.TrimSpace(md[revisionKey]) + if revision == "" { + return Observation{}, &IgnoredError{Reason: ReasonMissingRevision} + } + + ns := e.InvolvedObject.Namespace + name := e.InvolvedObject.Name + + serviceName := fluxMetadata(md, "service") + if serviceName == "" { + serviceName = name + } + + lines := []string{ + fmt.Sprintf("%s %s/%s", kind, ns, name), + "Revision: " + revision, + "Reason: " + e.Reason, + } + if v := strings.TrimSpace(e.Message); v != "" { + lines = append(lines, v) + } + + return Observation{ + Key: fmt.Sprintf("flux:%s:%s/%s/%s@%s", environment.String(), kind, ns, name, revision), + Source: SourceFlux, + Status: status, + At: at, + Terminal: IsTerminal(status), + Environment: environment, + Service: ServiceHint{Name: serviceName}, + ShortRevision: ShortRevision(revision), + Message: strings.Join(lines, "\n"), + Owner: "flux", + User: "flux:" + ns + "/" + name, + }, nil +} + +// ShortRevision shortens a Flux revision for display: the part after the +// last ':' (or, failing that, the last '/'), truncated to 8 characters when +// it is a hexadecimal string longer than that (a SHA); a chart version such +// as "1.2.3" is kept as is. +func ShortRevision(rev string) string { + s := rev + if i := strings.LastIndex(s, ":"); i >= 0 { + s = s[i+1:] + } else if i := strings.LastIndex(s, "/"); i >= 0 { + s = s[i+1:] + } + if len(s) > 8 && isHex(s) { + return s[:8] + } + return s +} + +// isHex reports whether s contains only hexadecimal digits. +func isHex(s string) bool { + for _, r := range s { + switch { + case r >= '0' && r <= '9': + case r >= 'a' && r <= 'f': + case r >= 'A' && r <= 'F': + default: + return false + } + } + return true +} diff --git a/internal/integrations/flux_test.go b/internal/integrations/flux_test.go new file mode 100644 index 0000000..cbff567 --- /dev/null +++ b/internal/integrations/flux_test.go @@ -0,0 +1,360 @@ +package integrations + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" +) + +func fluxBody(t *testing.T, mutate func(m map[string]any)) []byte { + t.Helper() + m := map[string]any{ + "involvedObject": map[string]any{"kind": "Kustomization", "namespace": "apps", "name": "payments"}, + "severity": "info", "timestamp": "2026-09-28T08:00:00Z", + "message": "Reconciliation finished", "reason": "ReconciliationSucceeded", + "metadata": map[string]any{ + "kustomize.toolkit.fluxcd.io/revision": "main@sha1:731f7eaddfb6af01cb2173e18f0f75b0ba780ef1", + "environment": "production", + }, + } + if mutate != nil { + mutate(m) + } + b, err := json.Marshal(m) + require.NoError(t, err) + return b +} + +func fluxConfig(t *testing.T) Config { + t.Helper() + cfg, err := LoadConfig(func(string) (string, bool) { return "", false }) + require.NoError(t, err) + return cfg +} + +func TestParseFlux(t *testing.T) { + now := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + + t.Run("nominal", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, nil) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + + assert.Equal(t, "flux:production:Kustomization/apps/payments@main@sha1:731f7eaddfb6af01cb2173e18f0f75b0ba780ef1", obs.Key) + assert.Equal(t, SourceFlux, obs.Source) + assert.Equal(t, eventv1.Status_success, obs.Status) + assert.True(t, obs.Terminal) + assert.True(t, obs.At.Equal(now)) + assert.Equal(t, eventv1.Environment_production, obs.Environment) + assert.Equal(t, ServiceHint{Name: "payments"}, obs.Service) + assert.Equal(t, "731f7ead", obs.ShortRevision) + assert.Equal(t, "flux", obs.Owner) + assert.Equal(t, "flux:apps/payments", obs.User) + assert.Equal(t, "Kustomization apps/payments\nRevision: main@sha1:731f7eaddfb6af01cb2173e18f0f75b0ba780ef1\nReason: ReconciliationSucceeded\nReconciliation finished", obs.Message) + assert.Equal(t, "Deploy payments 731f7ead to production", obs.Title("payments")) + }) + + t.Run("environment via event.toolkit.fluxcd.io/environment", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + md := m["metadata"].(map[string]any) + delete(md, "environment") + md["event.toolkit.fluxcd.io/environment"] = "staging" + }) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + assert.Equal(t, eventv1.Environment_preproduction, obs.Environment) + }) + + t.Run("environment lookup is case-insensitive", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + md := m["metadata"].(map[string]any) + md["environment"] = "Production" + }) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + assert.Equal(t, eventv1.Environment_production, obs.Environment) + }) + + t.Run("service via metadata service", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + md := m["metadata"].(map[string]any) + md["service"] = "payments-api" + }) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + assert.Equal(t, "payments-api", obs.Service.Name) + }) + + t.Run("service via event.toolkit.fluxcd.io/service", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + md := m["metadata"].(map[string]any) + md["event.toolkit.fluxcd.io/service"] = "payments-api" + }) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + assert.Equal(t, "payments-api", obs.Service.Name) + }) + + t.Run("HelmRelease revision is kept as is", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["involvedObject"] = map[string]any{"kind": "HelmRelease", "namespace": "apps", "name": "payments"} + m["metadata"] = map[string]any{ + "helm.toolkit.fluxcd.io/revision": "1.2.3", + "environment": "production", + } + }) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + assert.Equal(t, "1.2.3", obs.ShortRevision) + }) + + t.Run("Kustomization with only the Helm revision key is missing revision", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["metadata"] = map[string]any{ + "helm.toolkit.fluxcd.io/revision": "1.2.3", + "environment": "production", + } + }) + + _, err := ParseFlux(body, cfg, now) + var ignored *IgnoredError + require.ErrorAs(t, err, &ignored) + assert.Equal(t, ReasonMissingRevision, ignored.Reason) + }) + + t.Run("unsupported kind GitRepository is ignored", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["involvedObject"] = map[string]any{"kind": "GitRepository", "namespace": "apps", "name": "payments"} + }) + + _, err := ParseFlux(body, cfg, now) + var ignored *IgnoredError + require.ErrorAs(t, err, &ignored) + assert.Equal(t, ReasonUnsupportedKind, ignored.Reason) + }) + + t.Run("reason table", func(t *testing.T) { + cases := []struct { + severity string + reason string + status eventv1.Status + }{ + {"info", "Progressing", eventv1.Status_start}, + {"info", "ReconciliationSucceeded", eventv1.Status_success}, + {"info", "InstallSucceeded", eventv1.Status_success}, + {"info", "UpgradeSucceeded", eventv1.Status_success}, + {"error", "ReconciliationFailed", eventv1.Status_failure}, + {"error", "HealthCheckFailed", eventv1.Status_failure}, + {"error", "BuildFailed", eventv1.Status_failure}, + {"error", "ValidationFailed", eventv1.Status_failure}, + {"error", "ArtifactFailed", eventv1.Status_failure}, + {"error", "InstallFailed", eventv1.Status_failure}, + {"error", "UpgradeFailed", eventv1.Status_failure}, + {"error", "TestFailed", eventv1.Status_failure}, + // helm-controller emits RollbackSucceeded at severity info + // (corev1.EventTypeNormal), not error: see flux.go and the task + // report. A rollback still means the upgrade failed. + {"info", "RollbackSucceeded", eventv1.Status_failure}, + } + for _, c := range cases { + t.Run(c.severity+"/"+c.reason, func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["severity"] = c.severity + m["reason"] = c.reason + }) + + obs, err := ParseFlux(body, cfg, now) + require.NoError(t, err) + assert.Equal(t, c.status, obs.Status) + }) + } + }) + + t.Run("unsupported reasons are ignored", func(t *testing.T) { + cases := []struct { + severity string + reason string + }{ + {"info", "DependencyNotReady"}, + {"info", "UninstallSucceeded"}, + {"error", "Progressing"}, + {"info", "ReconciliationFailed"}, + // Verified: helm-controller emits RollbackSucceeded at severity + // info, never error, so this combination never occurs in + // practice, but the table must still reject it. + {"error", "RollbackSucceeded"}, + } + for _, c := range cases { + t.Run(c.severity+"/"+c.reason, func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["severity"] = c.severity + m["reason"] = c.reason + }) + + _, err := ParseFlux(body, cfg, now) + var ignored *IgnoredError + require.ErrorAs(t, err, &ignored) + assert.Equal(t, ReasonUnsupportedReason, ignored.Reason) + }) + } + }) + + t.Run("unmapped environment", func(t *testing.T) { + cases := []struct { + name string + mutate func(m map[string]any) + }{ + {"missing", func(m map[string]any) { + delete(m["metadata"].(map[string]any), "environment") + }}, + {"review", func(m map[string]any) { + m["metadata"].(map[string]any)["environment"] = "review" + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, c.mutate) + + _, err := ParseFlux(body, cfg, now) + var ignored *IgnoredError + require.ErrorAs(t, err, &ignored) + assert.Equal(t, ReasonUnmappedEnvironment, ignored.Reason) + }) + } + }) + + t.Run("missing required fields", func(t *testing.T) { + cases := []struct { + name string + mutate func(m map[string]any) + }{ + {"involvedObject.kind", func(m map[string]any) { + m["involvedObject"].(map[string]any)["kind"] = "" + }}, + {"involvedObject.name", func(m map[string]any) { + m["involvedObject"].(map[string]any)["name"] = "" + }}, + {"involvedObject.namespace", func(m map[string]any) { + m["involvedObject"].(map[string]any)["namespace"] = "" + }}, + {"severity", func(m map[string]any) { + m["severity"] = "" + }}, + {"reason", func(m map[string]any) { + m["reason"] = "" + }}, + {"timestamp", func(m map[string]any) { + m["timestamp"] = "" + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, c.mutate) + + _, err := ParseFlux(body, cfg, now) + var invalid *InvalidError + require.ErrorAs(t, err, &invalid) + }) + } + }) + + t.Run("timestamp is not a valid date", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["timestamp"] = "not a date" + }) + + _, err := ParseFlux(body, cfg, now) + var invalid *InvalidError + require.ErrorAs(t, err, &invalid) + }) + + t.Run("invalid JSON", func(t *testing.T) { + cfg := fluxConfig(t) + + _, err := ParseFlux([]byte("{"), cfg, now) + var invalid *InvalidError + require.ErrorAs(t, err, &invalid) + }) + + t.Run("freshness", func(t *testing.T) { + cases := []struct { + name string + timestamp time.Time + wantErr error + }{ + {"at tolerance boundary", now.Add(-5 * time.Minute), nil}, + {"just past lower bound", now.Add(-5*time.Minute - time.Second), ErrStaleTimestamp}, + {"just past upper bound", now.Add(5*time.Minute + time.Second), ErrStaleTimestamp}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["timestamp"] = c.timestamp.Format(time.RFC3339) + }) + + _, err := ParseFlux(body, cfg, now) + if c.wantErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, c.wantErr) + } + }) + } + + t.Run("freshness is checked before kind", func(t *testing.T) { + cfg := fluxConfig(t) + body := fluxBody(t, func(m map[string]any) { + m["involvedObject"] = map[string]any{"kind": "GitRepository", "namespace": "apps", "name": "payments"} + m["timestamp"] = now.Add(-10 * time.Minute).Format(time.RFC3339) + }) + + _, err := ParseFlux(body, cfg, now) + require.ErrorIs(t, err, ErrStaleTimestamp) + }) + }) +} + +func TestShortRevision(t *testing.T) { + cases := []struct { + rev string + want string + }{ + {"main@sha1:731f7eaddfb6af01cb2173e18f0f75b0ba780ef1", "731f7ead"}, + {"main/731f7eaddfb6af01cb2173e18f0f75b0ba780ef1", "731f7ead"}, + {"1.2.3", "1.2.3"}, + {"sha256:ABCDEF0123456789", "ABCDEF01"}, + {"abc", "abc"}, + {"v1.2.3", "v1.2.3"}, + {"", ""}, + } + for _, c := range cases { + t.Run(c.rev, func(t *testing.T) { + assert.Equal(t, c.want, ShortRevision(c.rev)) + }) + } +} From 0f661fc5ed6b469e5ce42323b04edf2633a02fde Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:41:41 +0200 Subject: [PATCH 07/22] feat(stores): add integration_deployments collection Correlates webhook notifications with Tracker events. Claim relies on the unique _id, Advance is a conditional FindOneAndUpdate on lastEventAt and rank, Revert restores the previous state. Indexes on eventId and a 90 day TTL on updatedAt. Refs #208 --- internal/stores/indexes.go | 27 ++ internal/stores/integration_deployments.go | 137 +++++++++ .../stores/integration_deployments_test.go | 280 ++++++++++++++++++ 3 files changed, 444 insertions(+) create mode 100644 internal/stores/integration_deployments.go create mode 100644 internal/stores/integration_deployments_test.go diff --git a/internal/stores/indexes.go b/internal/stores/indexes.go index 970f648..ce8e16c 100644 --- a/internal/stores/indexes.go +++ b/internal/stores/indexes.go @@ -41,6 +41,11 @@ func EnsureIndexes(ctx context.Context, db *mongo.Database) error { return err } + // Index pour la collection integration_deployments + if err := ensureIntegrationDeploymentIndexes(ctx, db, logger); err != nil { + return err + } + logger.Info("All database indexes ensured successfully") return nil } @@ -230,6 +235,28 @@ func ensureAuthIndexes(ctx context.Context, db *mongo.Database, logger *slog.Log return nil } +func ensureIntegrationDeploymentIndexes(ctx context.Context, db *mongo.Database, logger *slog.Logger) error { + collection := db.Collection(IntegrationDeploymentsCollection) + + indexes := []mongo.IndexModel{ + // _id is the correlation key (unique by construction). + // Reverse lookup from a Tracker event, for debugging. + { + Keys: bson.D{{Key: "eventId", Value: 1}}, + Options: options.Index().SetName("idx_integration_event_id"), + }, + // A correlation only matters while a deployment runs: purge after 90 days. + { + Keys: bson.D{{Key: "updatedAt", Value: 1}}, + Options: options.Index(). + SetName("idx_integration_updated_at_ttl"). + SetExpireAfterSeconds(int32(IntegrationDeploymentTTL / time.Second)), + }, + } + + return createIndexes(ctx, collection, indexes, logger, IntegrationDeploymentsCollection) +} + func createIndexes(ctx context.Context, collection *mongo.Collection, indexes []mongo.IndexModel, logger *slog.Logger, collectionName string) error { // Créer un contexte avec timeout pour éviter les blocages ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second) diff --git a/internal/stores/integration_deployments.go b/internal/stores/integration_deployments.go new file mode 100644 index 0000000..9416072 --- /dev/null +++ b/internal/stores/integration_deployments.go @@ -0,0 +1,137 @@ +package store + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const ( + IntegrationDeploymentsCollection = "integration_deployments" + // IntegrationDeploymentTTL bounds how long a correlation is kept after its last update. + IntegrationDeploymentTTL = 90 * 24 * time.Hour +) + +// IntegrationDeployment correlates the notifications of one deployment with +// its Tracker event. Rank orders statuses sharing the same instant. +type IntegrationDeployment struct { + Key string `bson:"_id"` + EventID string `bson:"eventId"` + Source string `bson:"source"` + Status string `bson:"status"` + Rank int `bson:"rank"` + LastEventAt time.Time `bson:"lastEventAt"` + CreatedAt time.Time `bson:"createdAt"` + UpdatedAt time.Time `bson:"updatedAt"` +} + +// NormalizeEventTime matches the millisecond precision of BSON dates. +func NormalizeEventTime(t time.Time) time.Time { return t.UTC().Truncate(time.Millisecond) } + +// IntegrationDeploymentStore persists the correlation between webhook +// notifications and Tracker events. +type IntegrationDeploymentStore struct { + coll *mongo.Collection + now func() time.Time +} + +func NewIntegrationDeploymentStore() *IntegrationDeploymentStore { + return NewIntegrationDeploymentStoreFromCollection(NewClient(IntegrationDeploymentsCollection)) +} + +func NewIntegrationDeploymentStoreFromCollection(coll *mongo.Collection) *IntegrationDeploymentStore { + return &IntegrationDeploymentStore{coll: coll, now: time.Now} +} + +// Claim inserts the correlation. When the key already exists it returns +// created=false and the stored document. +func (s *IntegrationDeploymentStore) Claim(ctx context.Context, key, source, status string, at time.Time, rank int) (bool, *IntegrationDeployment, error) { + now := s.now().UTC() + doc := &IntegrationDeployment{Key: key, Source: source, Status: status, Rank: rank, LastEventAt: NormalizeEventTime(at), CreatedAt: now, UpdatedAt: now} + _, err := s.coll.InsertOne(ctx, doc) + if err == nil { + return true, doc, nil + } + if !mongo.IsDuplicateKeyError(err) { + return false, nil, err + } + existing, err := s.Get(ctx, key) + if err != nil { + return false, nil, err + } + return false, existing, nil +} + +// Advance applies status atomically when it is newer than the stored state +// (see spec 5.3). Applied: returns the previous state. Not applied: returns +// the current state so that the caller can tell duplicate from stale. +func (s *IntegrationDeploymentStore) Advance(ctx context.Context, key, status string, at time.Time, rank int) (bool, *IntegrationDeployment, error) { + at = NormalizeEventTime(at) + filter := bson.M{"_id": key, "$or": bson.A{ + bson.M{"lastEventAt": bson.M{"$lt": at}, "rank": bson.M{"$lte": rank}}, + bson.M{"lastEventAt": at, "rank": bson.M{"$lt": rank}}, + }} + update := bson.M{"$set": bson.M{"status": status, "rank": rank, "lastEventAt": at, "updatedAt": s.now().UTC()}} + var prev IntegrationDeployment + err := s.coll.FindOneAndUpdate(ctx, filter, update, options.FindOneAndUpdate().SetReturnDocument(options.Before)).Decode(&prev) + if err == nil { + return true, &prev, nil + } + if !errors.Is(err, mongo.ErrNoDocuments) { + return false, nil, err + } + current, err := s.Get(ctx, key) + if err != nil { + return false, nil, err + } + return false, current, nil +} + +// SetEventID records the Tracker event created for this correlation. +func (s *IntegrationDeploymentStore) SetEventID(ctx context.Context, key, eventID string) error { + res, err := s.coll.UpdateOne(ctx, bson.M{"_id": key}, bson.M{"$set": bson.M{"eventId": eventID, "updatedAt": s.now().UTC()}}) + if err != nil { + return err + } + if res.MatchedCount == 0 { + return ErrNotFound + } + return nil +} + +// Revert restores the previous state after a Tracker write fails, but only +// if the stored state still matches at: a newer state already applied means +// there is nothing to undo. +func (s *IntegrationDeploymentStore) Revert(ctx context.Context, key string, prev *IntegrationDeployment, at time.Time) error { + if prev == nil { + return errors.New("revert: previous state is required") + } + _, err := s.coll.UpdateOne(ctx, + bson.M{"_id": key, "lastEventAt": NormalizeEventTime(at)}, + bson.M{"$set": bson.M{"status": prev.Status, "rank": prev.Rank, "lastEventAt": prev.LastEventAt, "updatedAt": s.now().UTC()}}, + ) + return err +} + +// Delete removes a correlation, e.g. once it is no longer needed. +func (s *IntegrationDeploymentStore) Delete(ctx context.Context, key string) error { + _, err := s.coll.DeleteOne(ctx, bson.M{"_id": key}) + return err +} + +// Get returns the stored correlation, or ErrNotFound. +func (s *IntegrationDeploymentStore) Get(ctx context.Context, key string) (*IntegrationDeployment, error) { + var doc IntegrationDeployment + err := s.coll.FindOne(ctx, bson.M{"_id": key}).Decode(&doc) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + return &doc, nil +} diff --git a/internal/stores/integration_deployments_test.go b/internal/stores/integration_deployments_test.go new file mode 100644 index 0000000..b66fd6c --- /dev/null +++ b/internal/stores/integration_deployments_test.go @@ -0,0 +1,280 @@ +package store + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" +) + +func newIDStore(t *testing.T) *IntegrationDeploymentStore { + return NewIntegrationDeploymentStoreFromCollection(testDatabase(t).Collection(IntegrationDeploymentsCollection)) +} + +func TestIntegrationDeploymentClaim(t *testing.T) { + s := newIDStore(t) + ctx := context.Background() + t0 := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + k := "claim-key" + + created, doc, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + assert.True(t, created) + assert.Equal(t, "", doc.EventID) + + created, doc, err = s.Claim(ctx, k, "gitlab", "success", t0, 1) + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, "start", doc.Status) + assert.Equal(t, 1, doc.Rank) + assert.True(t, doc.LastEventAt.Equal(t0)) + assert.Equal(t, "gitlab", doc.Source) +} + +func TestIntegrationDeploymentClaimConcurrent(t *testing.T) { + s := newIDStore(t) + ctx := context.Background() + t0 := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + k := "claim-concurrent-key" + + const n = 16 + start := make(chan struct{}) + var wg sync.WaitGroup + created := make([]bool, n) + errs := make([]error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + c, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + created[i] = c + errs[i] = err + }(i) + } + close(start) + wg.Wait() + + createdCount := 0 + for i := 0; i < n; i++ { + require.NoError(t, errs[i]) + if created[i] { + createdCount++ + } + } + assert.Equal(t, 1, createdCount) +} + +func TestIntegrationDeploymentAdvance(t *testing.T) { + ctx := context.Background() + t0 := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + + t.Run("duplicate and stale", func(t *testing.T) { + s := newIDStore(t) + k := "advance-duplicate-stale" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + applied, prev, err := s.Advance(ctx, k, "success", t0.Add(time.Second), 2) + require.NoError(t, err) + assert.True(t, applied) + assert.Equal(t, "start", prev.Status) + + got, err := s.Get(ctx, k) + require.NoError(t, err) + assert.Equal(t, "success", got.Status) + assert.Equal(t, 2, got.Rank) + assert.True(t, got.LastEventAt.Equal(t0.Add(time.Second))) + + // same advance again: duplicate, not applied. + applied, doc, err := s.Advance(ctx, k, "success", t0.Add(time.Second), 2) + require.NoError(t, err) + assert.False(t, applied) + assert.Equal(t, "success", doc.Status) + assert.True(t, doc.LastEventAt.Equal(t0.Add(time.Second))) + + // non terminal on a later instant never applies over a terminal status. + applied, doc, err = s.Advance(ctx, k, "start", t0.Add(2*time.Second), 1) + require.NoError(t, err) + assert.False(t, applied) + assert.Equal(t, "success", doc.Status) + + // older instant, not applied. + applied, doc, err = s.Advance(ctx, k, "failure", t0, 2) + require.NoError(t, err) + assert.False(t, applied) + assert.Equal(t, "success", doc.Status) + }) + + t.Run("same instant higher rank applies", func(t *testing.T) { + s := newIDStore(t) + k := "advance-same-instant-higher-rank" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + applied, prev, err := s.Advance(ctx, k, "success", t0, 2) + require.NoError(t, err) + assert.True(t, applied) + assert.Equal(t, "start", prev.Status) + }) + + t.Run("same instant lower rank does not apply", func(t *testing.T) { + s := newIDStore(t) + k := "advance-same-instant-lower-rank" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + applied, doc, err := s.Advance(ctx, k, "waiting_approval", t0, 1) + require.NoError(t, err) + assert.False(t, applied) + assert.Equal(t, "start", doc.Status) + }) + + t.Run("terminal replaces older terminal", func(t *testing.T) { + s := newIDStore(t) + k := "advance-terminal-replaces-terminal" + _, _, err := s.Claim(ctx, k, "gitlab", "success", t0, 2) + require.NoError(t, err) + + applied, prev, err := s.Advance(ctx, k, "failure", t0.Add(time.Second), 2) + require.NoError(t, err) + assert.True(t, applied) + assert.Equal(t, "success", prev.Status) + }) + + t.Run("truncates to millisecond", func(t *testing.T) { + s := newIDStore(t) + k := "advance-truncation" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + at := t0.Add(1*time.Second + 123456789*time.Nanosecond) + applied, _, err := s.Advance(ctx, k, "success", at, 2) + require.NoError(t, err) + assert.True(t, applied) + + applied, doc, err := s.Advance(ctx, k, "success", at, 2) + require.NoError(t, err) + assert.False(t, applied) + assert.True(t, doc.LastEventAt.Equal(NormalizeEventTime(at))) + }) + + t.Run("unknown key", func(t *testing.T) { + s := newIDStore(t) + _, _, err := s.Advance(ctx, "unknown-key", "success", t0, 1) + assert.ErrorIs(t, err, ErrNotFound) + }) +} + +func TestIntegrationDeploymentRevert(t *testing.T) { + ctx := context.Background() + t0 := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + + t.Run("restores previous state", func(t *testing.T) { + s := newIDStore(t) + k := "revert-key" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + applied, prev, err := s.Advance(ctx, k, "success", t0.Add(time.Second), 2) + require.NoError(t, err) + require.True(t, applied) + + require.NoError(t, s.Revert(ctx, k, prev, t0.Add(time.Second))) + + got, err := s.Get(ctx, k) + require.NoError(t, err) + assert.Equal(t, "start", got.Status) + assert.Equal(t, 1, got.Rank) + assert.True(t, got.LastEventAt.Equal(t0)) + }) + + t.Run("no-op when a newer state already applied", func(t *testing.T) { + s := newIDStore(t) + k := "revert-conditional-key" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + applied, prev1, err := s.Advance(ctx, k, "success", t0.Add(time.Second), 2) + require.NoError(t, err) + require.True(t, applied) + + applied, _, err = s.Advance(ctx, k, "failure", t0.Add(2*time.Second), 2) + require.NoError(t, err) + require.True(t, applied) + + require.NoError(t, s.Revert(ctx, k, prev1, t0.Add(time.Second))) + + got, err := s.Get(ctx, k) + require.NoError(t, err) + assert.Equal(t, "failure", got.Status) + assert.True(t, got.LastEventAt.Equal(t0.Add(2*time.Second))) + }) + + t.Run("nil previous state is an error", func(t *testing.T) { + s := newIDStore(t) + err := s.Revert(ctx, "revert-nil-key", nil, t0) + assert.Error(t, err) + }) +} + +func TestIntegrationDeploymentSetEventIDGetDelete(t *testing.T) { + s := newIDStore(t) + ctx := context.Background() + t0 := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + k := "set-event-id-key" + + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + require.NoError(t, s.SetEventID(ctx, k, "evt-1")) + got, err := s.Get(ctx, k) + require.NoError(t, err) + assert.Equal(t, "evt-1", got.EventID) + assert.False(t, got.UpdatedAt.IsZero()) + + require.NoError(t, s.Delete(ctx, k)) + _, err = s.Get(ctx, k) + assert.ErrorIs(t, err, ErrNotFound) + + assert.ErrorIs(t, s.SetEventID(ctx, "missing", "x"), ErrNotFound) + assert.NoError(t, s.Delete(ctx, "missing")) +} + +func TestIntegrationDeploymentIndexes(t *testing.T) { + ctx := context.Background() + db := testDatabase(t) + + cursor, err := db.Collection(IntegrationDeploymentsCollection).Indexes().List(ctx) + require.NoError(t, err) + var results []bson.M + require.NoError(t, cursor.All(ctx, &results)) + + found := map[string]bson.M{} + for _, idx := range results { + if name, ok := idx["name"].(string); ok { + found[name] = idx + } + } + + require.Contains(t, found, "idx_integration_event_id") + require.Contains(t, found, "idx_integration_updated_at_ttl") + + ttlIdx := found["idx_integration_updated_at_ttl"] + var ttl int64 + switch v := ttlIdx["expireAfterSeconds"].(type) { + case int32: + ttl = int64(v) + case int64: + ttl = v + case float64: + ttl = int64(v) + default: + t.Fatalf("unexpected type for expireAfterSeconds: %T", v) + } + assert.Equal(t, int64(7776000), ttl) +} From ef6820ddd90788bb9eec15dbbdb64378d5fbc591 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 13:52:55 +0200 Subject: [PATCH 08/22] refactor(server): extract unauthorized event and lock cores CreateEvent, UpdateEvent, CreateLock and UpdateLock keep authz.Authorize as their first statement and delegate to unexported cores (createEvent, updateEvent, createLock, updateLock). createEvent gains an observe mode that records a lock conflict instead of failing, and updateEvent measures the duration at an explicit instant. Characterization tests pin the RPC behaviour. Behaviour change: an event that takes a lock now counts one authorization decision in tracker_auth_requests_total instead of three, because the nested lock calls go through the cores. Refs #208 --- internal/stores/event.go | 5 + internal/stores/lock.go | 5 + server/event.go | 405 +++++++++++++++++++++-------------- server/event_core_test.go | 319 +++++++++++++++++++++++++++ server/event_testing_test.go | 95 ++++++++ server/lock.go | 19 ++ 6 files changed, 688 insertions(+), 160 deletions(-) create mode 100644 server/event_core_test.go create mode 100644 server/event_testing_test.go diff --git a/internal/stores/event.go b/internal/stores/event.go index 85badab..4350b6c 100644 --- a/internal/stores/event.go +++ b/internal/stores/event.go @@ -24,6 +24,11 @@ func NewStoreEvent(collection string) (c *EventStoreClient) { } } +// NewStoreEventFromCollection wraps an existing collection (tests, custom wiring). +func NewStoreEventFromCollection(coll *mongo.Collection) *EventStoreClient { + return &EventStoreClient{collection: coll} +} + // List takes label and field selectors, and returns the list of Events that match those selectors. func (c *EventStoreClient) List(ctx context.Context) (results []*v1alpha1.Event, err error) { cursor, err := c.collection.Find(context.TODO(), bson.D{}) diff --git a/internal/stores/lock.go b/internal/stores/lock.go index f97ed23..d1e2c14 100644 --- a/internal/stores/lock.go +++ b/internal/stores/lock.go @@ -23,6 +23,11 @@ func NewStoreLock(collection string) (c *LockStoreClient) { } } +// NewStoreLockFromCollection wraps an existing collection (tests, custom wiring). +func NewStoreLockFromCollection(coll *mongo.Collection) *LockStoreClient { + return &LockStoreClient{collection: coll} +} + // List takes label and field selectors, and returns the list of Locks that match those selectors. func (c *LockStoreClient) List(ctx context.Context) (results []*v1alpha1.Lock, err error) { cursor, err := c.collection.Find(context.TODO(), bson.D{}) diff --git a/server/event.go b/server/event.go index b6b5709..61261c8 100644 --- a/server/event.go +++ b/server/event.go @@ -45,15 +45,28 @@ type Event struct { logger *slog.Logger } -func NewEvent() *Event { +func newEventFromStores(events *store.EventStoreClient, locks *store.LockStoreClient, logger *slog.Logger) *Event { return &Event{ UnimplementedEventServiceServer: v1alpha1.UnimplementedEventServiceServer{}, - store: store.NewStoreEvent(config.ConfigDatabase.EventCollection), - lockService: NewLock(), - logger: slog.New(slog.NewJSONHandler(os.Stdout, nil)), + store: events, + lockService: &Lock{ + UnimplementedLockServiceServer: lock.UnimplementedLockServiceServer{}, + store: *locks, + eventStore: events, + logger: logger, + }, + logger: logger, } } +func NewEvent() *Event { + return newEventFromStores( + store.NewStoreEvent(config.ConfigDatabase.EventCollection), + store.NewStoreLock(config.ConfigDatabase.LockCollection), + slog.New(slog.NewJSONHandler(os.Stdout, nil)), + ) +} + // addChangelogEntry adds a new entry to the event's changelog func addChangelogEntry(event *v1alpha1.Event, changeType v1alpha1.ChangeType, user, field, oldValue, newValue, comment string) { if event.Changelog == nil { @@ -99,6 +112,24 @@ func getResourceType(eventType v1alpha1.Type) string { } } +// lockMode tells createEvent what to do when the service is already locked. +type lockMode int + +const ( + // lockStrict is the RPC behaviour: a conflict makes the creation fail. + lockStrict lockMode = iota + // lockObserve is the integration behaviour: the conflict is recorded, + // the event is created without a lock. + lockObserve +) + +// lockConflict names the holder of the lock an observed deployment could not take. +type lockConflict struct{ Who string } + +func lockConflictComment(service, environment, who string) string { + return fmt.Sprintf("Deployed while %s was locked in %s by %s", service, environment, who) +} + func (e *Event) CreateEvent( ctx context.Context, i *v1alpha1.CreateEventRequest, @@ -149,121 +180,159 @@ func (e *Event) CreateEvent( } - eventCounter.With(prometheus.Labels{"status": i.Attributes.Status.String(), "service": i.Attributes.Service, "environment": i.Attributes.Environment.String()}).Inc() - - // Add initial changelog entry user := "system" if i.Attributes.Owner != "" { user = i.Attributes.Owner } + + // The lock taken for a deployment or an operation is part of creating the + // event: it is covered by the event:write check above and goes through the + // unauthorized lock core, so the request counts a single decision in + // tracker_auth_requests_total. + created, _, err := e.createEvent(ctx, event, user, "", lockStrict) + if err != nil { + return nil, err + } + return &v1alpha1.CreateEventResponse{Event: created}, nil +} + +// createEvent is CreateEvent without authorization. mode controls what +// happens when the service is already locked: lockStrict fails the creation +// (the RPC behaviour), lockObserve records the conflict on the event instead +// (the integration behaviour) and creates it without taking the lock. +func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, comment string, mode lockMode) (*v1alpha1.Event, *lockConflict, error) { + attrs := event.Attributes + environment := attrs.Environment.String() + resource := getResourceType(attrs.Type) + + eventCounter.With(prometheus.Labels{"status": attrs.Status.String(), "service": attrs.Service, "environment": environment}).Inc() + addChangelogEntry(event, v1alpha1.ChangeType_created, user, "", "", "", "Event created") - // Vérifier et créer un lock si nécessaire AVANT de créer l'événement - if shouldCreateLock(i.Attributes.Type, i.Attributes.Status) { - lockReq := &lock.CreateLockRequest{ - Service: i.Attributes.Service, - Who: user, - Environment: i.Attributes.Environment.String(), - Resource: getResourceType(i.Attributes.Type), - EventId: "", // Sera mis à jour après la création de l'événement - } + var lockID string + var conflict *lockConflict + switch mode { + case lockStrict: + if shouldCreateLock(attrs.Type, attrs.Status) { + res, err := e.lockService.createLock(ctx, &lock.CreateLockRequest{ + Service: attrs.Service, + Who: user, + Environment: environment, + Resource: resource, + EventId: "", + }) + if err != nil { + e.logger.Error("failed to create lock", + "service", attrs.Service, + "environment", environment, + "resource", resource, + "error", err, + ) - // This is an internal call: it reuses the request context, so the - // method name authz sees stays "/tracker.event.v1alpha1.EventService/ - // CreateEvent", not CreateLock. The lock is therefore authorized by - // event:write, not lock:write, even though spec 3.1 files CreateLock - // under lock:write. That is deliberate: creating an event that locks a - // service is one operation from the caller's point of view. - // Side effect: tracker_auth_requests_total counts such a request twice - // under the same method label, once for CreateEvent and once for the - // nested Authorize below. Same for the UpdateLock call further down. - _, err := e.lockService.CreateLock(ctx, lockReq) - if err != nil { - e.logger.Error("failed to create lock", - "service", i.Attributes.Service, - "environment", i.Attributes.Environment.String(), - "resource", getResourceType(i.Attributes.Type), - "error", err, - ) + if strings.Contains(err.Error(), "already locked") { + return nil, nil, fmt.Errorf("cannot create event: service %s is already locked in %s. Please unlock it first", + attrs.Service, environment) + } - // Améliorer le message d'erreur pour être plus explicite - if strings.Contains(err.Error(), "already locked") { - return nil, fmt.Errorf("cannot create event: service %s is already locked in %s. Please unlock it first", - i.Attributes.Service, i.Attributes.Environment.String()) + return nil, nil, fmt.Errorf("cannot create event: failed to create lock - %v", err) + } + lockID = res.Lock.Id + } + case lockObserve: + if resource != "unknown" { + var err error + lockID, conflict, err = e.observeLock(ctx, attrs.Service, environment, resource, user, shouldCreateLock(attrs.Type, attrs.Status)) + if err != nil { + return nil, nil, err } - - return nil, fmt.Errorf("cannot create event: failed to create lock - %v", err) } } - var eventResult = &v1alpha1.CreateEventResponse{} - var err error - eventResult.Event, err = e.store.Create(context.Background(), event) - if err != nil { - return nil, err + if conflict != nil { + addChangelogEntry(event, v1alpha1.ChangeType_commented, user, "", "", "", lockConflictComment(attrs.Service, environment, conflict.Who)) + } + if comment != "" { + addChangelogEntry(event, v1alpha1.ChangeType_commented, user, "", "", "", comment) } - // Mettre à jour le lock avec l'event_id - if shouldCreateLock(i.Attributes.Type, i.Attributes.Status) { - // Récupérer le lock correspondant et mettre à jour son event_id - filter := map[string]interface{}{ - "service": i.Attributes.Service, - "environment": i.Attributes.Environment.String(), - "resource": getResourceType(i.Attributes.Type), - } - - existingLock, err2 := e.lockService.store.Get(ctx, filter) - if err2 == nil && existingLock != nil && existingLock.Id != "" { - // Internal call on the request context, see the comment above the - // CreateLock call: authorized by event:write and counted a second - // time in tracker_auth_requests_total under CreateEvent. - _, errUpd := e.lockService.UpdateLock(ctx, &lock.UpdateLockRequest{ - Id: existingLock.Id, - EventId: eventResult.Event.Metadata.Id, - }) - if errUpd != nil { - e.logger.Warn("failed to update lock with event_id", - "lock_id", existingLock.Id, - "event_id", eventResult.Event.Metadata.Id, - "service", i.Attributes.Service, - "environment", i.Attributes.Environment.String(), - "error", errUpd, - ) - } else { - e.logger.Info("lock updated with event_id", - "lock_id", existingLock.Id, - "event_id", eventResult.Event.Metadata.Id, - "service", i.Attributes.Service, - "environment", i.Attributes.Environment.String(), + created, err := e.store.Create(context.Background(), event) + if err != nil { + if mode == lockObserve && lockID != "" { + if _, unlockErr := e.lockService.store.Unlock(ctx, map[string]interface{}{"id": lockID}); unlockErr != nil { + e.logger.Error("failed to release lock after failed event creation", + "lock_id", lockID, + "service", attrs.Service, + "environment", environment, + "error", unlockErr, ) } + } + return nil, nil, err + } + + if lockID != "" { + _, errUpd := e.lockService.updateLock(ctx, &lock.UpdateLockRequest{ + Id: lockID, + EventId: created.Metadata.Id, + }) + if errUpd != nil { + e.logger.Warn("failed to update lock with event_id", + "lock_id", lockID, + "event_id", created.Metadata.Id, + "service", attrs.Service, + "environment", environment, + "error", errUpd, + ) } else { - e.logger.Warn("lock not found for event to update", - "service", i.Attributes.Service, - "environment", i.Attributes.Environment.String(), - "resource", getResourceType(i.Attributes.Type), - "error", err2, + e.logger.Info("lock updated with event_id", + "lock_id", lockID, + "event_id", created.Metadata.Id, + "service", attrs.Service, + "environment", environment, ) } } // log event to json format e.logger.Info("event created", - "title", eventResult.Event.Title, - "message", eventResult.Event.Attributes.Message, - "priority", eventResult.Event.Attributes.Priority.String(), - "environment", eventResult.Event.Attributes.Environment.String(), - "owner", eventResult.Event.Attributes.Owner, - "impact", eventResult.Event.Attributes.Impact, - "service", eventResult.Event.Attributes.Service, - "status", eventResult.Event.Attributes.Status.String(), - "type", eventResult.Event.Attributes.Type.String(), - "pull_request", eventResult.Event.Links.PullRequestLink, - "id", eventResult.Event.Metadata.Id, - "created_at", eventResult.Event.Metadata.CreatedAt.AsTime(), + "title", created.Title, + "message", created.Attributes.Message, + "priority", created.Attributes.Priority.String(), + "environment", created.Attributes.Environment.String(), + "owner", created.Attributes.Owner, + "impact", created.Attributes.Impact, + "service", created.Attributes.Service, + "status", created.Attributes.Status.String(), + "type", created.Attributes.Type.String(), + "pull_request", created.Links.PullRequestLink, + "id", created.Metadata.Id, + "created_at", created.Metadata.CreatedAt.AsTime(), ) - return eventResult, nil + return created, conflict, nil +} + +// observeLock takes the lock when take is true and nobody holds it. A lock +// held by someone else is reported, never returned as an error. +func (e *Event) observeLock(ctx context.Context, service, environment, resource, who string, take bool) (string, *lockConflict, error) { + if held := e.lockService.findLock(ctx, service, environment, resource); held != nil { + return "", &lockConflict{Who: held.Who}, nil + } + if !take { + return "", nil, nil + } + res, err := e.lockService.createLock(ctx, &lock.CreateLockRequest{Service: service, Who: who, Environment: environment, Resource: resource}) + if err != nil { + if strings.Contains(err.Error(), "already locked") { + holder := "unknown" + if held := e.lockService.findLock(ctx, service, environment, resource); held != nil { + holder = held.Who + } + return "", &lockConflict{Who: holder}, nil + } + return "", nil, fmt.Errorf("cannot create lock: %w", err) + } + return res.Lock.Id, nil, nil } func (e *Event) GetEvent( @@ -373,7 +442,6 @@ func (e *Event) UpdateEvent( return nil, err } - var eventResult = &v1alpha1.UpdateEventResponse{} var eventDatabase = &v1alpha1.GetEventResponse{} var err error @@ -425,86 +493,131 @@ func (e *Event) UpdateEvent( }, } - if event.Attributes.Status == 2 || event.Attributes.Status == 3 { - duration := time.Since(eventDatabase.Event.Metadata.CreatedAt.AsTime()) - event.Metadata.Duration = durationpb.New(duration) - if eventDatabase.Event.Attributes.Status != event.Attributes.Status { - recordEvent(event.Attributes.Status.String(), event.Attributes.Service, event.Attributes.Environment.String(), duration) - } - } - - // Preserve existing changelog - event.Changelog = eventDatabase.Event.Changelog - // Track changes and add changelog entries user := "system" if i.Attributes.Owner != "" { user = i.Attributes.Owner } + // Use the appropriate filter based on whether SlackId or Id is provided + var filter map[string]interface{} + if i.SlackId != "" { + filter = map[string]interface{}{"metadata.slackid": i.SlackId} + } else { + filter = map[string]interface{}{"metadata.id": i.Id} + } + + updated, err := e.updateEvent(ctx, eventDatabase.Event, event, filter, user, "", time.Now()) + if err != nil { + return nil, err + } + + // Libérer le lock si l'événement se termine + if shouldReleaseLock(event.Attributes.Type, event.Attributes.Status) { + err = e.lockService.UnlockByEventId(ctx, updated.Metadata.Id) + if err != nil { + e.logger.Warn("failed to release lock", + "event_id", updated.Metadata.Id, + "service", event.Attributes.Service, + "error", err, + ) + // Ne pas retourner d'erreur, l'événement est déjà mis à jour + } else { + e.logger.Info("lock released for event", + "event_id", updated.Metadata.Id, + "service", event.Attributes.Service, + "status", event.Attributes.Status.String(), + ) + } + } + + return &v1alpha1.UpdateEventResponse{Event: updated}, nil +} + +// updateEvent is UpdateEvent without authorization. current is the event as +// stored, next is the requested state (metadata already carries the id, +// created_at and previous duration); at is the instant the duration is +// measured against, and comment, when set, is appended as a final changelog +// entry after the update summary. Only callers of the server package that +// already authorized or authenticated the operation may use it. +func (e *Event) updateEvent(ctx context.Context, current, next *v1alpha1.Event, filter map[string]interface{}, user, comment string, at time.Time) (*v1alpha1.Event, error) { + if next.Attributes.Status == v1alpha1.Status_failure || next.Attributes.Status == v1alpha1.Status_success { + duration := at.Sub(current.Metadata.CreatedAt.AsTime()) + if duration < 0 { + duration = 0 + } + next.Metadata.Duration = durationpb.New(duration) + if current.Attributes.Status != next.Attributes.Status { + recordEvent(next.Attributes.Status.String(), next.Attributes.Service, next.Attributes.Environment.String(), duration) + } + } + + // Preserve existing changelog + next.Changelog = current.Changelog + // Detect if this is an approval (only owner changed, nothing else) - isApproval := eventDatabase.Event.Attributes.Owner != event.Attributes.Owner && - eventDatabase.Event.Attributes.Status == event.Attributes.Status && - eventDatabase.Event.Attributes.Priority == event.Attributes.Priority && - eventDatabase.Event.Title == event.Title + isApproval := current.Attributes.Owner != next.Attributes.Owner && + current.Attributes.Status == next.Attributes.Status && + current.Attributes.Priority == next.Attributes.Priority && + current.Title == next.Title // Check for status change - if eventDatabase.Event.Attributes.Status != event.Attributes.Status { + if current.Attributes.Status != next.Attributes.Status { addChangelogEntry( - event, + next, v1alpha1.ChangeType_status_changed, user, "status", - eventDatabase.Event.Attributes.Status.String(), - event.Attributes.Status.String(), + current.Attributes.Status.String(), + next.Attributes.Status.String(), "Status updated", ) } // Check for ticket link change - if eventDatabase.Event.Links.Ticket != event.Links.Ticket && event.Links.Ticket != "" { + if current.Links.Ticket != next.Links.Ticket && next.Links.Ticket != "" { addChangelogEntry( - event, + next, v1alpha1.ChangeType_linked, user, "ticket", - eventDatabase.Event.Links.Ticket, - event.Links.Ticket, + current.Links.Ticket, + next.Links.Ticket, "Jira ticket linked", ) } // Check for priority change - if eventDatabase.Event.Attributes.Priority != event.Attributes.Priority { + if current.Attributes.Priority != next.Attributes.Priority { addChangelogEntry( - event, + next, v1alpha1.ChangeType_updated, user, "priority", - eventDatabase.Event.Attributes.Priority.String(), - event.Attributes.Priority.String(), + current.Attributes.Priority.String(), + next.Attributes.Priority.String(), "Priority updated", ) } // Check for title change - if eventDatabase.Event.Title != event.Title { + if current.Title != next.Title { addChangelogEntry( - event, + next, v1alpha1.ChangeType_updated, user, "title", - eventDatabase.Event.Title, - event.Title, + current.Title, + next.Title, "Title updated", ) } // Add general update entry if no specific changes were tracked - if len(event.Changelog) == len(eventDatabase.Event.Changelog) { + if len(next.Changelog) == len(current.Changelog) { if isApproval { addChangelogEntry( - event, + next, v1alpha1.ChangeType_approved, user, "", @@ -514,7 +627,7 @@ func (e *Event) UpdateEvent( ) } else { addChangelogEntry( - event, + next, v1alpha1.ChangeType_updated, user, "", @@ -525,39 +638,11 @@ func (e *Event) UpdateEvent( } } - // Use the appropriate filter based on whether SlackId or Id is provided - var filter map[string]interface{} - if i.SlackId != "" { - filter = map[string]interface{}{"metadata.slackid": i.SlackId} - } else { - filter = map[string]interface{}{"metadata.id": i.Id} + if comment != "" { + addChangelogEntry(next, v1alpha1.ChangeType_commented, user, "", "", "", comment) } - eventResult.Event, err = e.store.Update(context.Background(), filter, event) - if err != nil { - return nil, err - } - - // Libérer le lock si l'événement se termine - if shouldReleaseLock(event.Attributes.Type, event.Attributes.Status) { - err = e.lockService.UnlockByEventId(ctx, eventResult.Event.Metadata.Id) - if err != nil { - e.logger.Warn("failed to release lock", - "event_id", eventResult.Event.Metadata.Id, - "service", event.Attributes.Service, - "error", err, - ) - // Ne pas retourner d'erreur, l'événement est déjà mis à jour - } else { - e.logger.Info("lock released for event", - "event_id", eventResult.Event.Metadata.Id, - "service", event.Attributes.Service, - "status", event.Attributes.Status.String(), - ) - } - } - - return eventResult, nil + return e.store.Update(context.Background(), filter, next) } // DeleteEvents implements the EventService.DeleteEvents RPC. The method is diff --git a/server/event_core_test.go b/server/event_core_test.go new file mode 100644 index 0000000..4dfa646 --- /dev/null +++ b/server/event_core_test.go @@ -0,0 +1,319 @@ +package server + +import ( + "context" + "testing" + "time" + + v1alpha1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" + lockv1 "github.com/bananaops/tracker/generated/proto/lock/v1alpha1" + store "github.com/bananaops/tracker/internal/stores" + "github.com/stretchr/testify/require" +) + +// baseCreateEventRequest returns the CreateEventRequest used by every +// characterization test: a deployment start on svc/production owned by alice. +func baseCreateEventRequest() *v1alpha1.CreateEventRequest { + return &v1alpha1.CreateEventRequest{ + Title: "deploy svc", + Attributes: &v1alpha1.EventAttributes{ + Type: v1alpha1.Type_deployment, + Status: v1alpha1.Status_start, + Service: "svc", + Environment: v1alpha1.Environment_production, + Owner: "alice", + Priority: v1alpha1.Priority_P3, + Source: "manual", + }, + Links: &v1alpha1.EventLinks{}, + } +} + +func TestCreateEventRPCTakesAndLinksLock(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + resp, err := e.CreateEvent(eventRPCCtx("CreateEvent"), baseCreateEventRequest()) + require.NoError(t, err) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "alice", locks[0].Who) + require.Equal(t, "production", locks[0].Environment) + require.Equal(t, "deployment", locks[0].Resource) + require.Equal(t, resp.Event.Metadata.Id, locks[0].EventId) + + require.NotEmpty(t, resp.Event.Changelog) + require.Equal(t, v1alpha1.ChangeType_created, resp.Event.Changelog[0].ChangeType) + require.Equal(t, "alice", resp.Event.Changelog[0].User) + require.Equal(t, "Event created", resp.Event.Changelog[0].Comment) +} + +func TestCreateEventRPCRefusesLockedService(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + _, err := store.NewStoreLockFromCollection(db.Collection("locks")).Create(context.Background(), &lockv1.Lock{ + Service: "svc", Environment: "production", Resource: "deployment", Who: "bob", + }) + require.NoError(t, err) + + _, err = e.CreateEvent(eventRPCCtx("CreateEvent"), baseCreateEventRequest()) + require.Error(t, err) + require.EqualError(t, err, "cannot create event: service svc is already locked in production. Please unlock it first") + + events, err := store.NewStoreEventFromCollection(db.Collection("events")).List(context.Background()) + require.NoError(t, err) + require.Len(t, events, 0) +} + +func TestUpdateEventRPCReleasesLockOnSuccess(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + created, err := e.CreateEvent(eventRPCCtx("CreateEvent"), baseCreateEventRequest()) + require.NoError(t, err) + id := created.Event.Metadata.Id + + req := baseCreateEventRequest() + req.Attributes.Status = v1alpha1.Status_success + _, err = e.UpdateEvent(eventRPCCtx("UpdateEvent"), &v1alpha1.UpdateEventRequest{ + Id: id, + Title: "deploy svc", + Attributes: req.Attributes, + Links: &v1alpha1.EventLinks{}, + }) + require.NoError(t, err) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 0) + + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": id}) + require.NoError(t, err) + require.Equal(t, v1alpha1.Status_success, updated.Attributes.Status) + require.NotNil(t, updated.Metadata.Duration) + + var statusChanged *v1alpha1.ChangelogEntry + for _, entry := range updated.Changelog { + if entry.ChangeType == v1alpha1.ChangeType_status_changed { + statusChanged = entry + } + } + require.NotNil(t, statusChanged) + require.Equal(t, "status", statusChanged.Field) + require.Equal(t, "start", statusChanged.OldValue) + require.Equal(t, "success", statusChanged.NewValue) + require.Equal(t, "alice", statusChanged.User) +} + +func TestCreateEventRPCIncidentTakesNoLock(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + req := baseCreateEventRequest() + req.Attributes.Type = v1alpha1.Type_incident + _, err := e.CreateEvent(eventRPCCtx("CreateEvent"), req) + require.NoError(t, err) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 0) +} + +// -- Step 4: new tests, red until the core refactor lands. -- + +func TestCreateEventObserveRecordsConflict(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + _, err := store.NewStoreLockFromCollection(db.Collection("locks")).Create(context.Background(), &lockv1.Lock{ + Service: "svc", Environment: "production", Resource: "deployment", Who: "alice", + }) + require.NoError(t, err) + + ev := &v1alpha1.Event{ + Title: "deploy svc", + Attributes: &v1alpha1.EventAttributes{ + Type: v1alpha1.Type_deployment, + Status: v1alpha1.Status_start, + Service: "svc", + Environment: v1alpha1.Environment_production, + Owner: "alice", + Priority: v1alpha1.Priority_P3, + Source: "manual", + }, + Links: &v1alpha1.EventLinks{}, + Metadata: &v1alpha1.EventMetadata{}, + } + + created, conflict, err := e.createEvent(context.Background(), ev, "gitlab:bob", "", lockObserve) + require.NoError(t, err) + require.NotNil(t, conflict) + require.Equal(t, "alice", conflict.Who) + + var commented *v1alpha1.ChangelogEntry + for _, entry := range created.Changelog { + if entry.ChangeType == v1alpha1.ChangeType_commented { + commented = entry + } + } + require.NotNil(t, commented) + require.Equal(t, "Deployed while svc was locked in production by alice", commented.Comment) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "alice", locks[0].Who) + require.Equal(t, "", locks[0].EventId) +} + +func TestCreateEventObserveTakesLock(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + ev := &v1alpha1.Event{ + Title: "deploy svc", + Attributes: &v1alpha1.EventAttributes{ + Type: v1alpha1.Type_deployment, + Status: v1alpha1.Status_start, + Service: "svc", + Environment: v1alpha1.Environment_production, + Owner: "alice", + Priority: v1alpha1.Priority_P3, + Source: "manual", + }, + Links: &v1alpha1.EventLinks{}, + Metadata: &v1alpha1.EventMetadata{}, + } + + created, conflict, err := e.createEvent(context.Background(), ev, "gitlab:bob", "", lockObserve) + require.NoError(t, err) + require.Nil(t, conflict) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "gitlab:bob", locks[0].Who) + require.Equal(t, created.Metadata.Id, locks[0].EventId) +} + +func TestCreateEventObserveTerminalTakesNoLock(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + ev := &v1alpha1.Event{ + Title: "deploy svc", + Attributes: &v1alpha1.EventAttributes{ + Type: v1alpha1.Type_deployment, + Status: v1alpha1.Status_success, + Service: "svc", + Environment: v1alpha1.Environment_production, + Owner: "alice", + Priority: v1alpha1.Priority_P3, + Source: "manual", + }, + Links: &v1alpha1.EventLinks{}, + Metadata: &v1alpha1.EventMetadata{}, + } + + _, conflict, err := e.createEvent(context.Background(), ev, "gitlab:bob", "", lockObserve) + require.NoError(t, err) + require.Nil(t, conflict) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 0) +} + +func TestUpdateEventUsesExplicitTime(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + ev := &v1alpha1.Event{ + Title: "deploy svc", + Attributes: &v1alpha1.EventAttributes{ + Type: v1alpha1.Type_deployment, + Status: v1alpha1.Status_start, + Service: "svc", + Environment: v1alpha1.Environment_production, + Owner: "alice", + Priority: v1alpha1.Priority_P3, + Source: "manual", + }, + Links: &v1alpha1.EventLinks{}, + Metadata: &v1alpha1.EventMetadata{}, + } + created, _, err := e.createEvent(context.Background(), ev, "alice", "", lockStrict) + require.NoError(t, err) + id := created.Metadata.Id + + current, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": id}) + require.NoError(t, err) + + next := &v1alpha1.Event{ + Title: current.Title, + Attributes: &v1alpha1.EventAttributes{ + Message: current.Attributes.Message, + Source: current.Attributes.Source, + Type: current.Attributes.Type, + Priority: current.Attributes.Priority, + Impact: current.Attributes.Impact, + Environment: current.Attributes.Environment, + Owner: current.Attributes.Owner, + RelatedId: current.Attributes.RelatedId, + Service: current.Attributes.Service, + Status: v1alpha1.Status_success, + StartDate: current.Attributes.StartDate, + EndDate: current.Attributes.EndDate, + StakeHolders: current.Attributes.StakeHolders, + Notification: current.Attributes.Notification, + Notifications: current.Attributes.Notifications, + }, + Links: &v1alpha1.EventLinks{ + PullRequestLink: current.Links.PullRequestLink, + Ticket: current.Links.Ticket, + }, + Metadata: &v1alpha1.EventMetadata{ + Id: current.Metadata.Id, + CreatedAt: current.Metadata.CreatedAt, + SlackId: current.Metadata.SlackId, + }, + } + + at := current.Metadata.CreatedAt.AsTime().Add(90 * time.Second) + _, err = e.updateEvent(context.Background(), current, next, map[string]interface{}{"metadata.id": id}, "gitlab:bob", "Deployment canceled", at) + require.NoError(t, err) + + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": id}) + require.NoError(t, err) + require.NotNil(t, updated.Metadata.Duration) + require.Equal(t, 90*time.Second, updated.Metadata.Duration.AsDuration()) + + last := updated.Changelog[len(updated.Changelog)-1] + require.Equal(t, v1alpha1.ChangeType_commented, last.ChangeType) + require.Equal(t, "Deployment canceled", last.Comment) + + var statusChanged *v1alpha1.ChangelogEntry + for _, entry := range updated.Changelog { + if entry.ChangeType == v1alpha1.ChangeType_status_changed { + statusChanged = entry + } + } + require.NotNil(t, statusChanged) + require.Equal(t, "gitlab:bob", statusChanged.User) +} + +func TestCreateEventCountsOneAuthorization(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + before := gatheredCounter(t, "tracker_auth_requests_total", map[string]string{"principal": "user", "result": "allowed"}) + + _, err := e.CreateEvent(eventRPCCtx("CreateEvent"), baseCreateEventRequest()) + require.NoError(t, err) + + after := gatheredCounter(t, "tracker_auth_requests_total", map[string]string{"principal": "user", "result": "allowed"}) + require.Equal(t, float64(1), after-before) +} diff --git a/server/event_testing_test.go b/server/event_testing_test.go new file mode 100644 index 0000000..7090187 --- /dev/null +++ b/server/event_testing_test.go @@ -0,0 +1,95 @@ +package server + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/bananaops/tracker/internal/auth" + store "github.com/bananaops/tracker/internal/stores" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "google.golang.org/grpc" +) + +// testMongoDatabase connects to MONGO_TEST_URI, creates a throwaway database +// with all indexes and drops it at the end of the test. +func testMongoDatabase(t *testing.T) *mongo.Database { + t.Helper() + uri := os.Getenv("MONGO_TEST_URI") + if uri == "" { + t.Skip("MONGO_TEST_URI not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + require.NoError(t, err) + db := client.Database(fmt.Sprintf("tracker_test_%d", time.Now().UnixNano())) + t.Cleanup(func() { + c, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = db.Drop(c) + _ = client.Disconnect(c) + }) + require.NoError(t, store.EnsureIndexes(ctx, db)) + return db +} + +func newTestEvent(t *testing.T, db *mongo.Database) *Event { + t.Helper() + return newEventFromStores( + store.NewStoreEventFromCollection(db.Collection("events")), + store.NewStoreLockFromCollection(db.Collection("locks")), + slog.New(slog.NewJSONHandler(io.Discard, nil)), + ) +} + +func writerPrincipal() auth.Principal { + return auth.Principal{Kind: auth.KindUser, Username: "tester", + Permissions: auth.NewPermissionSet(auth.AllPermissions()...), Scope: auth.ScopeAll()} +} + +func eventRPCCtx(method string) context.Context { + ctx := grpc.NewContextWithServerTransportStream(context.Background(), + fakeTransportStream{method: "/tracker.event.v1alpha1.EventService/" + method}) + return auth.WithPrincipal(ctx, writerPrincipal()) +} + +// gatheredCounter reads a counter of the default registry whose labels +// include every pair of labels (0 when absent). +func gatheredCounter(t *testing.T, name string, labels map[string]string) float64 { + t.Helper() + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + for _, mf := range mfs { + if mf.GetName() != name { + continue + } + for _, m := range mf.GetMetric() { + match := true + for k, v := range labels { + found := false + for _, lp := range m.GetLabel() { + if lp.GetName() == k && lp.GetValue() == v { + found = true + break + } + } + if !found { + match = false + break + } + } + if match { + return m.GetCounter().GetValue() + } + } + } + return 0 +} diff --git a/server/lock.go b/server/lock.go index 18b0492..37d6416 100644 --- a/server/lock.go +++ b/server/lock.go @@ -37,7 +37,12 @@ func (e *Lock) CreateLock( if err := authz.Authorize(ctx); err != nil { return nil, err } + return e.createLock(ctx, i) +} +// createLock is CreateLock without authorization. Only callers of the server +// package that already authorized or authenticated the operation may use it. +func (e *Lock) createLock(ctx context.Context, i *v1alpha1.CreateLockRequest) (*v1alpha1.CreateLockResponse, error) { var lock = &v1alpha1.Lock{ Service: i.Service, Who: i.Who, @@ -143,7 +148,12 @@ func (e *Lock) UpdateLock( if err := authz.Authorize(ctx); err != nil { return nil, err } + return e.updateLock(ctx, i) +} +// updateLock is UpdateLock without authorization. Only callers of the server +// package that already authorized or authenticated the operation may use it. +func (e *Lock) updateLock(ctx context.Context, i *v1alpha1.UpdateLockRequest) (*v1alpha1.UpdateLockResponse, error) { // Retrieve existing lock by id existing, err := e.store.Get(ctx, map[string]interface{}{"id": i.Id}) if err != nil { @@ -311,3 +321,12 @@ func (e *Lock) UnlockByEventId(ctx context.Context, eventId string) error { return nil } + +// findLock returns the lock held on service, environment and resource, or nil. +func (e *Lock) findLock(ctx context.Context, service, environment, resource string) *v1alpha1.Lock { + l, err := e.store.Get(ctx, map[string]interface{}{"service": service, "environment": environment, "resource": resource}) + if err != nil || l == nil || l.Service == "" { + return nil + } + return l +} From 3257707e1396af72176d0646d86beabf810c51cc Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 14:20:57 +0200 Subject: [PATCH 09/22] feat(server): apply deployment observations to events and locks IntegrationProcessor claims the correlation key, creates the Tracker event once and updates it afterwards through the unauthorized cores, in observe mode for locks: a conflict is recorded as a changelog comment and never fails the request. Terminal statuses release only the lock of their own event. Service names come from a 60 second snapshot of the catalog. Refs #208 --- internal/stores/catalog.go | 5 + server/integrations_processor.go | 384 +++++++++++++++++++++++++ server/integrations_processor_test.go | 395 ++++++++++++++++++++++++++ 3 files changed, 784 insertions(+) create mode 100644 server/integrations_processor.go create mode 100644 server/integrations_processor_test.go diff --git a/internal/stores/catalog.go b/internal/stores/catalog.go index 09c38f6..e4d04e0 100644 --- a/internal/stores/catalog.go +++ b/internal/stores/catalog.go @@ -20,6 +20,11 @@ func NewStoreCatalog(collection string) (c *CatalogStoreClient) { } } +// NewStoreCatalogFromCollection wraps an existing collection (tests, custom wiring). +func NewStoreCatalogFromCollection(coll *mongo.Collection) *CatalogStoreClient { + return &CatalogStoreClient{collection: coll} +} + // List takes label and field selectors, and returns the list of Catalogs that match those selectors. func (c *CatalogStoreClient) List(ctx context.Context) (results []*v1alpha1.Catalog, err error) { cursor, err := c.collection.Find(context.TODO(), bson.D{}) diff --git a/server/integrations_processor.go b/server/integrations_processor.go new file mode 100644 index 0000000..b6d06ba --- /dev/null +++ b/server/integrations_processor.go @@ -0,0 +1,384 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + catalogv1 "github.com/bananaops/tracker/generated/proto/catalog/v1alpha1" + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" + lockv1 "github.com/bananaops/tracker/generated/proto/lock/v1alpha1" + "github.com/bananaops/tracker/internal/integrations" + store "github.com/bananaops/tracker/internal/stores" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + catalogCacheTTL = 60 * time.Second + claimPollInterval = 100 * time.Millisecond + claimPollTimeout = 2 * time.Second + staleClaimAfter = 30 * time.Second + integrationLockResource = "deployment" + outcomeRecorded = "recorded" + outcomeLockConflict = "lock_conflict" + outcomeDuplicate = "duplicate" + outcomeStale = "stale" +) + +// errClaimPending signals that a correlation was claimed but the Tracker +// event it will point to is not recorded yet (either still being created, or +// abandoned and dropped). +var errClaimPending = errors.New("deployment creation still in progress") + +// DeploymentStore is the subset of IntegrationDeploymentStore the processor +// needs, narrowed for testing. +type DeploymentStore interface { + Claim(ctx context.Context, key, source, status string, at time.Time, rank int) (bool, *store.IntegrationDeployment, error) + Advance(ctx context.Context, key, status string, at time.Time, rank int) (bool, *store.IntegrationDeployment, error) + SetEventID(ctx context.Context, key, eventID string) error + Revert(ctx context.Context, key string, prev *store.IntegrationDeployment, at time.Time) error + Delete(ctx context.Context, key string) error + Get(ctx context.Context, key string) (*store.IntegrationDeployment, error) +} + +// CatalogLister is the subset of CatalogStoreClient the service resolver +// needs, narrowed for testing. +type CatalogLister interface { + List(ctx context.Context) ([]*catalogv1.Catalog, error) +} + +// ProcessResult reports what Process did with one observation. +type ProcessResult struct { + Outcome string + EventID string +} + +type catalogEntry struct{ name, repository string } + +// serviceResolver keeps a snapshot of (name, normalized repository) for the +// whole catalog, reloaded at most once per ttl. A failed reload keeps the +// previous snapshot. +type serviceResolver struct { + catalog CatalogLister + ttl time.Duration + now func() time.Time + logger *slog.Logger + + mu sync.RWMutex + loadedAt time.Time + entries []catalogEntry +} + +// snapshot returns the current catalog snapshot, reloading it when it is +// older than ttl. At most one reload happens per ttl, whether it succeeds or +// fails. +func (r *serviceResolver) snapshot(ctx context.Context) []catalogEntry { + r.mu.RLock() + if !r.loadedAt.IsZero() && r.now().Sub(r.loadedAt) < r.ttl { + entries := r.entries + r.mu.RUnlock() + return entries + } + r.mu.RUnlock() + + r.mu.Lock() + defer r.mu.Unlock() + if !r.loadedAt.IsZero() && r.now().Sub(r.loadedAt) < r.ttl { + return r.entries + } + + catalogs, err := r.catalog.List(ctx) + r.loadedAt = r.now() + if err != nil { + r.logger.Error("integration: catalog reload failed, keeping the previous snapshot", "error", err) + return r.entries + } + + entries := make([]catalogEntry, 0, len(catalogs)) + for _, c := range catalogs { + if c == nil || c.Name == "" { + continue + } + entries = append(entries, catalogEntry{name: c.Name, repository: integrations.NormalizeRepoURL(c.Repository)}) + } + r.entries = entries + return r.entries +} + +// Resolve matches hint's repository URLs against the catalog first, then its +// name, and falls back to the name unchanged. +func (r *serviceResolver) Resolve(ctx context.Context, hint integrations.ServiceHint) string { + entries := r.snapshot(ctx) + for _, url := range hint.RepositoryURLs { + if url == "" { + continue + } + for _, e := range entries { + if e.repository != "" && e.repository == url { + return e.name + } + } + } + for _, e := range entries { + if e.name == hint.Name { + return e.name + } + } + return hint.Name +} + +// IntegrationProcessor turns a normalized deployment observation into one +// Tracker event, claiming the correlation key so that concurrent +// notifications of the same deployment agree on a single event. +type IntegrationProcessor struct { + events *Event + store DeploymentStore + services *serviceResolver + logger *slog.Logger + + pollInterval time.Duration + pollTimeout time.Duration + staleClaim time.Duration + now func() time.Time +} + +func NewIntegrationProcessor(events *Event, deployments DeploymentStore, catalog CatalogLister, logger *slog.Logger) *IntegrationProcessor { + return &IntegrationProcessor{ + events: events, + store: deployments, + services: &serviceResolver{catalog: catalog, ttl: catalogCacheTTL, now: time.Now, logger: logger}, + logger: logger, + pollInterval: claimPollInterval, + pollTimeout: claimPollTimeout, + staleClaim: staleClaimAfter, + now: time.Now, + } +} + +// Process claims obs's correlation key, then creates or updates the Tracker +// event it correlates to. +func (p *IntegrationProcessor) Process(ctx context.Context, obs integrations.Observation) (ProcessResult, error) { + status := obs.Status.String() + rank := integrations.Rank(obs.Status) + created, doc, err := p.store.Claim(ctx, obs.Key, obs.Source, status, obs.At, rank) + if err != nil { + return ProcessResult{}, fmt.Errorf("claim deployment: %w", err) + } + if created { + return p.create(ctx, obs) + } + if doc.EventID == "" { + if p.now().Sub(doc.CreatedAt) > p.staleClaim { + p.logger.Error("integration: dropping an abandoned claim", "key", obs.Key) + if err := p.store.Delete(ctx, obs.Key); err != nil { + p.logger.Error("integration: cannot drop the abandoned claim", "key", obs.Key, "error", err) + } + return ProcessResult{}, errClaimPending + } + if _, err = p.waitForEventID(ctx, obs.Key); err != nil { + return ProcessResult{}, err + } + } + applied, state, err := p.store.Advance(ctx, obs.Key, status, obs.At, rank) + if err != nil { + return ProcessResult{}, fmt.Errorf("advance deployment: %w", err) + } + if !applied { + outcome := outcomeStale + if state.Status == status && state.LastEventAt.Equal(store.NormalizeEventTime(obs.At)) { + outcome = outcomeDuplicate + } + return ProcessResult{Outcome: outcome, EventID: state.EventID}, nil + } + return p.update(ctx, obs, state) +} + +// waitForEventID polls the claim until it carries an event id, the poll +// timeout elapses (errClaimPending), or the context is done. +func (p *IntegrationProcessor) waitForEventID(ctx context.Context, key string) (*store.IntegrationDeployment, error) { + deadline := p.now().Add(p.pollTimeout) + for { + timer := time.NewTimer(p.pollInterval) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + + doc, err := p.store.Get(ctx, key) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, errClaimPending + } + return nil, err + } + if doc.EventID != "" { + return doc, nil + } + if p.now().After(deadline) { + return nil, errClaimPending + } + } +} + +// create resolves the service, creates the Tracker event and records its id +// on the claim. A failure at either step drops the claim so a later +// notification can retry from scratch. +func (p *IntegrationProcessor) create(ctx context.Context, obs integrations.Observation) (ProcessResult, error) { + service := p.services.Resolve(ctx, obs.Service) + ev := newIntegrationEvent(obs, service) + created, conflict, err := p.events.createEvent(ctx, ev, obs.User, obs.Comment, lockObserve) + if err != nil { + if delErr := p.store.Delete(ctx, obs.Key); delErr != nil { + p.logger.Error("integration: cannot drop the claim after a failed event creation", "key", obs.Key, "error", delErr) + } + return ProcessResult{}, fmt.Errorf("create event: %w", err) + } + if err := p.store.SetEventID(ctx, obs.Key, created.Metadata.Id); err != nil { + p.logger.Error("integration: cannot record the event id for the claim", "key", obs.Key, "eventId", created.Metadata.Id, "error", err) + if delErr := p.store.Delete(ctx, obs.Key); delErr != nil { + p.logger.Error("integration: cannot drop the claim after a failed event id update", "key", obs.Key, "error", delErr) + } + return ProcessResult{}, err + } + return ProcessResult{Outcome: outcomeFor(conflict), EventID: created.Metadata.Id}, nil +} + +func newIntegrationEvent(obs integrations.Observation, service string) *eventv1.Event { + attrs := &eventv1.EventAttributes{ + Message: obs.Message, + Source: obs.Source, + Type: eventv1.Type_deployment, + Priority: eventv1.Priority_P3, + Impact: false, + Environment: obs.Environment, + Owner: obs.Owner, + Service: service, + Status: obs.Status, + StartDate: timestamppb.New(obs.At), + } + if obs.Terminal { + attrs.EndDate = timestamppb.New(obs.At) + } + return &eventv1.Event{ + Title: obs.Title(service), + Attributes: attrs, + Links: &eventv1.EventLinks{}, + Metadata: &eventv1.EventMetadata{}, + } +} + +// update applies obs to the event already correlated with the claim. A lock +// conflict is recorded as a changelog comment and never fails the request; +// only a store or Tracker write failure does, and then the claim is reverted +// to its previous state so a later notification can retry. +func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Observation, prev *store.IntegrationDeployment) (ProcessResult, error) { + fail := func(err error) (ProcessResult, error) { + if revertErr := p.store.Revert(ctx, obs.Key, prev, obs.At); revertErr != nil { + p.logger.Error("integration: cannot revert the claim after a failed event update", "key", obs.Key, "eventId", prev.EventID, "error", revertErr) + } + return ProcessResult{}, err + } + + current, err := p.events.store.Get(ctx, map[string]interface{}{"metadata.id": prev.EventID}) + if err != nil { + return fail(err) + } + + service := current.Attributes.Service + environment := current.Attributes.Environment.String() + + var conflict *lockConflict + comment := obs.Comment + takeLock := obs.Status == eventv1.Status_start && current.Attributes.Status != eventv1.Status_start + if takeLock { + if held := p.events.lockService.findLock(ctx, service, environment, integrationLockResource); held != nil { + conflict = &lockConflict{Who: held.Who} + takeLock = false + comment = lockConflictComment(service, environment, held.Who) + } + } + + next := nextIntegrationEvent(current, obs) + + if _, err := p.events.updateEvent(ctx, current, next, map[string]interface{}{"metadata.id": prev.EventID}, obs.User, comment, obs.At); err != nil { + return fail(err) + } + + if takeLock { + lockID, c, err := p.events.observeLock(ctx, service, environment, integrationLockResource, obs.User, true) + if err != nil { + p.logger.Warn("integration: cannot take lock after status change", "key", obs.Key, "eventId", prev.EventID, "error", err) + } else if c != nil { + conflict = c + p.logger.Warn("integration: lock conflict while taking lock after status change", "key", obs.Key, "eventId", prev.EventID, "who", c.Who) + } else if lockID != "" { + if _, err := p.events.lockService.updateLock(ctx, &lockv1.UpdateLockRequest{Id: lockID, EventId: prev.EventID}); err != nil { + p.logger.Warn("integration: cannot link lock to event", "key", obs.Key, "eventId", prev.EventID, "lockId", lockID, "error", err) + } + } + } + + if obs.Terminal { + if err := p.events.lockService.UnlockByEventId(ctx, prev.EventID); err != nil { + p.logger.Warn("integration: cannot release lock for terminal status", "key", obs.Key, "eventId", prev.EventID, "error", err) + } + } + + return ProcessResult{Outcome: outcomeFor(conflict), EventID: prev.EventID}, nil +} + +// nextIntegrationEvent builds the requested state for updateEvent: current +// copied field by field, with the observed status and, for a terminal +// status, an end date. +func nextIntegrationEvent(current *eventv1.Event, obs integrations.Observation) *eventv1.Event { + attrs := &eventv1.EventAttributes{ + Message: current.Attributes.Message, + Source: current.Attributes.Source, + Type: current.Attributes.Type, + Priority: current.Attributes.Priority, + RelatedId: current.Attributes.RelatedId, + Service: current.Attributes.Service, + Status: obs.Status, + Environment: current.Attributes.Environment, + Impact: current.Attributes.Impact, + StartDate: current.Attributes.StartDate, + EndDate: current.Attributes.EndDate, + Owner: current.Attributes.Owner, + StakeHolders: current.Attributes.StakeHolders, + Notification: current.Attributes.Notification, + Notifications: current.Attributes.Notifications, + } + if obs.Terminal { + attrs.EndDate = timestamppb.New(obs.At) + } + + links := &eventv1.EventLinks{} + if current.Links != nil { + links.PullRequestLink = current.Links.PullRequestLink + links.Ticket = current.Links.Ticket + } + + return &eventv1.Event{ + Title: current.Title, + Attributes: attrs, + Links: links, + Metadata: &eventv1.EventMetadata{ + SlackId: current.Metadata.SlackId, + CreatedAt: current.Metadata.CreatedAt, + Duration: current.Metadata.Duration, + Id: current.Metadata.Id, + }, + } +} + +func outcomeFor(conflict *lockConflict) string { + if conflict != nil { + return outcomeLockConflict + } + return outcomeRecorded +} diff --git a/server/integrations_processor_test.go b/server/integrations_processor_test.go new file mode 100644 index 0000000..6b46686 --- /dev/null +++ b/server/integrations_processor_test.go @@ -0,0 +1,395 @@ +package server + +import ( + "context" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + catalogv1 "github.com/bananaops/tracker/generated/proto/catalog/v1alpha1" + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" + lockv1 "github.com/bananaops/tracker/generated/proto/lock/v1alpha1" + "github.com/bananaops/tracker/internal/integrations" + store "github.com/bananaops/tracker/internal/stores" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/mongo" +) + +type fakeCatalog struct { + mu sync.Mutex + entries []*catalogv1.Catalog + err error + calls int +} + +func (f *fakeCatalog) List(context.Context) ([]*catalogv1.Catalog, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + if f.err != nil { + return nil, f.err + } + return f.entries, nil +} + +func (f *fakeCatalog) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +type procEnv struct { + db *mongo.Database + events *Event + store *store.IntegrationDeploymentStore + proc *IntegrationProcessor + cat *fakeCatalog +} + +func newProcEnv(t *testing.T) *procEnv { + t.Helper() + db := testMongoDatabase(t) + events := newTestEvent(t, db) + st := store.NewIntegrationDeploymentStoreFromCollection(db.Collection("integration_deployments")) + cat := &fakeCatalog{} + proc := NewIntegrationProcessor(events, st, cat, slog.New(slog.NewJSONHandler(io.Discard, nil))) + return &procEnv{db: db, events: events, store: st, proc: proc, cat: cat} +} + +func obs(status eventv1.Status, at time.Time) integrations.Observation { + return integrations.Observation{ + Key: "gitlab:gitlab.example.com:42", + Source: "gitlab", + Status: status, + At: at, + Terminal: integrations.IsTerminal(status), + Environment: eventv1.Environment_production, + Service: integrations.ServiceHint{Name: "payments"}, + ShortRevision: "a1b2c3d4", + Message: "m", + Owner: "jdoe", + User: "gitlab:jdoe", + } +} + +func TestProcessCreateThenUpdate(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + require.NotEmpty(t, res.EventID) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + require.Equal(t, "Deploy payments a1b2c3d4 to production", ev.Title) + require.Equal(t, "gitlab", ev.Attributes.Source) + require.Equal(t, eventv1.Type_deployment, ev.Attributes.Type) + require.Equal(t, eventv1.Priority_P3, ev.Attributes.Priority) + require.Equal(t, "jdoe", ev.Attributes.Owner) + require.True(t, ev.Attributes.StartDate.AsTime().Equal(t0)) + require.Nil(t, ev.Attributes.EndDate) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "gitlab:jdoe", locks[0].Who) + require.Equal(t, res.EventID, locks[0].EventId) + + t1 := t0.Add(30 * time.Second) + res2, err := env.proc.Process(ctx, obs(eventv1.Status_success, t1)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res2.Outcome) + require.Equal(t, res.EventID, res2.EventID) + + updated, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + require.Equal(t, eventv1.Status_success, updated.Attributes.Status) + require.True(t, updated.Attributes.EndDate.AsTime().Equal(t1)) + + locks, err = store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) + + doc, err := env.store.Get(ctx, obs(eventv1.Status_start, t0).Key) + require.NoError(t, err) + require.Equal(t, "success", doc.Status) + require.Equal(t, res.EventID, doc.EventID) +} + +func TestProcessDuplicateAndStale(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + _, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + + t1 := t0.Add(1 * time.Second) + res, err := env.proc.Process(ctx, obs(eventv1.Status_success, t1)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + changelogLen := len(ev.Changelog) + + res2, err := env.proc.Process(ctx, obs(eventv1.Status_success, t1)) + require.NoError(t, err) + require.Equal(t, outcomeDuplicate, res2.Outcome) + + res3, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0.Add(2*time.Second))) + require.NoError(t, err) + require.Equal(t, outcomeStale, res3.Outcome) + + ev2, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + require.Len(t, ev2.Changelog, changelogLen) +} + +func TestProcessTerminalFirstTakesNoLock(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_success, t0)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + require.True(t, ev.Attributes.StartDate.AsTime().Equal(t0)) + require.True(t, ev.Attributes.EndDate.AsTime().Equal(t0)) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) + + res2, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0.Add(-10*time.Second))) + require.NoError(t, err) + require.Equal(t, outcomeStale, res2.Outcome) + + locks, err = store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) +} + +func TestProcessLockConflict(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + _, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).Create(ctx, &lockv1.Lock{ + Service: "payments", Environment: "production", Resource: "deployment", Who: "alice", + }) + require.NoError(t, err) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + require.Equal(t, outcomeLockConflict, res.Outcome) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + var commented *eventv1.ChangelogEntry + for _, e := range ev.Changelog { + if e.ChangeType == eventv1.ChangeType_commented { + commented = e + } + } + require.NotNil(t, commented) + require.Equal(t, "Deployed while payments was locked in production by alice", commented.Comment) + + res2, err := env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res2.Outcome) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "alice", locks[0].Who) + require.Equal(t, "", locks[0].EventId) +} + +func TestProcessCanceledReleasesLock(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + _, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + + warn := obs(eventv1.Status_warning, t0.Add(time.Second)) + warn.Comment = "Deployment canceled" + res, err := env.proc.Process(ctx, warn) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + last := ev.Changelog[len(ev.Changelog)-1] + require.Equal(t, eventv1.ChangeType_commented, last.ChangeType) + require.Equal(t, "Deployment canceled", last.Comment) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) +} + +func TestProcessApprovalThenStartTakesLock(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + _, err := env.proc.Process(ctx, obs(eventv1.Status_waiting_approval, t0)) + require.NoError(t, err) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) + + t1 := t0.Add(5 * time.Second) + res, err := env.proc.Process(ctx, obs(eventv1.Status_start, t1)) + require.NoError(t, err) + + locks, err = store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "gitlab:jdoe", locks[0].Who) + require.Equal(t, res.EventID, locks[0].EventId) + + t2 := t0.Add(10 * time.Second) + _, err = env.proc.Process(ctx, obs(eventv1.Status_success, t2)) + require.NoError(t, err) + + locks, err = store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) +} + +func TestProcessUpdateFailureReverts(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + + require.NoError(t, env.events.store.Delete(ctx, map[string]interface{}{"metadata.id": res.EventID})) + + _, err = env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) + require.Error(t, err) + + doc, err := env.store.Get(ctx, obs(eventv1.Status_start, t0).Key) + require.NoError(t, err) + require.Equal(t, "start", doc.Status) + require.True(t, doc.LastEventAt.Equal(t0)) +} + +func TestProcessWaitsForPendingClaim(t *testing.T) { + env := newProcEnv(t) + env.proc.pollInterval = 20 * time.Millisecond + env.proc.pollTimeout = 500 * time.Millisecond + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + key := obs(eventv1.Status_start, t0).Key + + _, _, err := env.store.Claim(ctx, key, "gitlab", "start", t0, 1) + require.NoError(t, err) + + idCh := make(chan string, 1) + errCh := make(chan error, 1) + go func() { + ev := newIntegrationEvent(obs(eventv1.Status_start, t0), "payments") + created, _, err := env.events.createEvent(ctx, ev, "gitlab:jdoe", "", lockObserve) + if err != nil { + errCh <- err + return + } + idCh <- created.Metadata.Id + time.Sleep(100 * time.Millisecond) + errCh <- env.store.SetEventID(ctx, key, created.Metadata.Id) + }() + + var expected string + select { + case expected = <-idCh: + case err := <-errCh: + t.Fatalf("create event: %v", err) + } + + res, err := env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + require.Equal(t, expected, res.EventID) +} + +func TestProcessPendingClaimTimesOut(t *testing.T) { + env := newProcEnv(t) + env.proc.pollTimeout = 200 * time.Millisecond + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + key := obs(eventv1.Status_start, t0).Key + + _, _, err := env.store.Claim(ctx, key, "gitlab", "start", t0, 1) + require.NoError(t, err) + + _, err = env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) + require.True(t, errors.Is(err, errClaimPending)) +} + +func TestProcessAbandonedClaimIsDropped(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + key := obs(eventv1.Status_start, t0).Key + + _, _, err := env.store.Claim(ctx, key, "gitlab", "start", t0, 1) + require.NoError(t, err) + + env.proc.now = func() time.Time { return time.Now().Add(time.Minute) } + + _, err = env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) + require.True(t, errors.Is(err, errClaimPending)) + + _, err = env.store.Get(ctx, key) + require.ErrorIs(t, err, store.ErrNotFound) +} + +func TestServiceResolver(t *testing.T) { + cat := &fakeCatalog{entries: []*catalogv1.Catalog{ + {Name: "payments-api", Repository: "git@gitlab.example.com:team/payments.git"}, + {Name: "billing"}, + }} + now := time.Now() + resolver := &serviceResolver{catalog: cat, ttl: 60 * time.Second, now: func() time.Time { return now }, logger: slog.New(slog.NewJSONHandler(io.Discard, nil))} + ctx := context.Background() + + name := resolver.Resolve(ctx, integrations.ServiceHint{RepositoryURLs: []string{"https://gitlab.example.com/team/payments"}, Name: "payments"}) + require.Equal(t, "payments-api", name) + + name = resolver.Resolve(ctx, integrations.ServiceHint{Name: "billing"}) + require.Equal(t, "billing", name) + + name = resolver.Resolve(ctx, integrations.ServiceHint{Name: "unknown"}) + require.Equal(t, "unknown", name) + + require.Equal(t, 1, cat.callCount()) + + now = now.Add(61 * time.Second) + name = resolver.Resolve(ctx, integrations.ServiceHint{Name: "billing"}) + require.Equal(t, "billing", name) + require.Equal(t, 2, cat.callCount()) + + cat.mu.Lock() + cat.err = errors.New("boom") + cat.mu.Unlock() + now = now.Add(61 * time.Second) + name = resolver.Resolve(ctx, integrations.ServiceHint{RepositoryURLs: []string{"https://gitlab.example.com/team/payments"}, Name: "payments"}) + require.Equal(t, "payments-api", name) + require.Equal(t, 3, cat.callCount()) +} From ba29f6f259eb6d22e4a6d9711bd8e4d74de46985 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 14:37:14 +0200 Subject: [PATCH 10/22] fix(stores): make integration Revert conditional on the applied state Revert's filter now also matches the status and rank of the Advance being undone, in addition to lastEventAt, so it can never clobber a newer state that shares the same instant but carries a higher rank. Refs #208 --- internal/stores/integration_deployments.go | 9 ++--- .../stores/integration_deployments_test.go | 33 +++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/internal/stores/integration_deployments.go b/internal/stores/integration_deployments.go index 9416072..0ee5b25 100644 --- a/internal/stores/integration_deployments.go +++ b/internal/stores/integration_deployments.go @@ -104,14 +104,15 @@ func (s *IntegrationDeploymentStore) SetEventID(ctx context.Context, key, eventI } // Revert restores the previous state after a Tracker write fails, but only -// if the stored state still matches at: a newer state already applied means -// there is nothing to undo. -func (s *IntegrationDeploymentStore) Revert(ctx context.Context, key string, prev *IntegrationDeployment, at time.Time) error { +// if the stored state still matches the Advance being undone: status, rank +// and lastEventAt all have to match, so a Revert can never clobber a newer +// state that shares the same instant but carries a higher rank. +func (s *IntegrationDeploymentStore) Revert(ctx context.Context, key string, prev *IntegrationDeployment, status string, rank int, at time.Time) error { if prev == nil { return errors.New("revert: previous state is required") } _, err := s.coll.UpdateOne(ctx, - bson.M{"_id": key, "lastEventAt": NormalizeEventTime(at)}, + bson.M{"_id": key, "status": status, "rank": rank, "lastEventAt": NormalizeEventTime(at)}, bson.M{"$set": bson.M{"status": prev.Status, "rank": prev.Rank, "lastEventAt": prev.LastEventAt, "updatedAt": s.now().UTC()}}, ) return err diff --git a/internal/stores/integration_deployments_test.go b/internal/stores/integration_deployments_test.go index b66fd6c..a6fa1e3 100644 --- a/internal/stores/integration_deployments_test.go +++ b/internal/stores/integration_deployments_test.go @@ -184,7 +184,7 @@ func TestIntegrationDeploymentRevert(t *testing.T) { require.NoError(t, err) require.True(t, applied) - require.NoError(t, s.Revert(ctx, k, prev, t0.Add(time.Second))) + require.NoError(t, s.Revert(ctx, k, prev, "success", 2, t0.Add(time.Second))) got, err := s.Get(ctx, k) require.NoError(t, err) @@ -207,7 +207,7 @@ func TestIntegrationDeploymentRevert(t *testing.T) { require.NoError(t, err) require.True(t, applied) - require.NoError(t, s.Revert(ctx, k, prev1, t0.Add(time.Second))) + require.NoError(t, s.Revert(ctx, k, prev1, "success", 2, t0.Add(time.Second))) got, err := s.Get(ctx, k) require.NoError(t, err) @@ -215,9 +215,36 @@ func TestIntegrationDeploymentRevert(t *testing.T) { assert.True(t, got.LastEventAt.Equal(t0.Add(2*time.Second))) }) + t.Run("no-op when a higher rank state applied at the same instant", func(t *testing.T) { + s := newIDStore(t) + k := "revert-same-instant-higher-rank-key" + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + + // A request observing "success" at t0 advances the claim (same + // instant, higher rank). Its own write to the previous, lower-rank + // state must never clobber it: the filter has to match status and + // rank too, not just lastEventAt. + applied, prev, err := s.Advance(ctx, k, "success", t0, 2) + require.NoError(t, err) + require.True(t, applied) + require.Equal(t, "start", prev.Status) + require.Equal(t, 1, prev.Rank) + + // Revert as if undoing the original "start" claim (status/rank from + // before the Advance) at the same lastEventAt: must not apply since + // the stored status/rank no longer match "start"/1. + require.NoError(t, s.Revert(ctx, k, &IntegrationDeployment{Status: "queued", Rank: 0}, "start", 1, t0)) + + got, err := s.Get(ctx, k) + require.NoError(t, err) + assert.Equal(t, "success", got.Status) + assert.Equal(t, 2, got.Rank) + }) + t.Run("nil previous state is an error", func(t *testing.T) { s := newIDStore(t) - err := s.Revert(ctx, "revert-nil-key", nil, t0) + err := s.Revert(ctx, "revert-nil-key", nil, "start", 1, t0) assert.Error(t, err) }) } From 19c3dcf493ce46e4eb963693fee65a939e280f67 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 14:37:22 +0200 Subject: [PATCH 11/22] fix(server): converge integration events on concurrent updates After every successful event write for a key, the processor re-reads the claim and, if a concurrent observation has moved it further, re-applies the claim's own status and end date through the same unauthorized update core, up to 3 times, without any per-key mutex or Mongo lease; the reported outcome always stays the request's own. The lock taken on a start transition is now created with its event id in the same write instead of a separate attach step, is released immediately if the claim has already gone terminal by the time it is taken, and is released by its id if the post-take re-read itself fails. SetEventID now retries up to 3 times (50/100/200ms) after createEvent succeeds; if it still fails, the created event and any lock attached to it are removed before the claim is dropped, instead of leaving them orphaned. The service resolver reloads the catalog against a context detached from the caller, single-flighted so concurrent callers never wait on the reload or block on the exclusive lock, and retries a failed reload after 5s instead of the full 60s ttl. Refs #208 --- server/event.go | 14 +- server/integrations_processor.go | 228 ++++++++++++++++++++------ server/integrations_processor_test.go | 136 +++++++++++++++ 3 files changed, 323 insertions(+), 55 deletions(-) diff --git a/server/event.go b/server/event.go index 61261c8..f5e50cb 100644 --- a/server/event.go +++ b/server/event.go @@ -240,8 +240,10 @@ func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, co } case lockObserve: if resource != "unknown" { + // The event does not have an id yet at this point, so the lock + // (if taken) is attached to it below, once the event is created. var err error - lockID, conflict, err = e.observeLock(ctx, attrs.Service, environment, resource, user, shouldCreateLock(attrs.Type, attrs.Status)) + lockID, conflict, err = e.observeLock(ctx, attrs.Service, environment, resource, user, "", shouldCreateLock(attrs.Type, attrs.Status)) if err != nil { return nil, nil, err } @@ -312,16 +314,18 @@ func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, co return created, conflict, nil } -// observeLock takes the lock when take is true and nobody holds it. A lock -// held by someone else is reported, never returned as an error. -func (e *Event) observeLock(ctx context.Context, service, environment, resource, who string, take bool) (string, *lockConflict, error) { +// observeLock takes the lock when take is true and nobody holds it, in the +// same write as eventId when the caller already knows it (an empty eventId +// leaves the lock unattached; the caller links it once the event exists). A +// lock held by someone else is reported, never returned as an error. +func (e *Event) observeLock(ctx context.Context, service, environment, resource, who, eventID string, take bool) (string, *lockConflict, error) { if held := e.lockService.findLock(ctx, service, environment, resource); held != nil { return "", &lockConflict{Who: held.Who}, nil } if !take { return "", nil, nil } - res, err := e.lockService.createLock(ctx, &lock.CreateLockRequest{Service: service, Who: who, Environment: environment, Resource: resource}) + res, err := e.lockService.createLock(ctx, &lock.CreateLockRequest{Service: service, Who: who, Environment: environment, Resource: resource, EventId: eventID}) if err != nil { if strings.Contains(err.Error(), "already locked") { holder := "unknown" diff --git a/server/integrations_processor.go b/server/integrations_processor.go index b6d06ba..5b790a7 100644 --- a/server/integrations_processor.go +++ b/server/integrations_processor.go @@ -10,24 +10,29 @@ import ( catalogv1 "github.com/bananaops/tracker/generated/proto/catalog/v1alpha1" eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" - lockv1 "github.com/bananaops/tracker/generated/proto/lock/v1alpha1" "github.com/bananaops/tracker/internal/integrations" store "github.com/bananaops/tracker/internal/stores" "google.golang.org/protobuf/types/known/timestamppb" ) const ( - catalogCacheTTL = 60 * time.Second - claimPollInterval = 100 * time.Millisecond - claimPollTimeout = 2 * time.Second - staleClaimAfter = 30 * time.Second - integrationLockResource = "deployment" - outcomeRecorded = "recorded" - outcomeLockConflict = "lock_conflict" - outcomeDuplicate = "duplicate" - outcomeStale = "stale" + catalogCacheTTL = 60 * time.Second + catalogRetryAfterFailure = 5 * time.Second + claimPollInterval = 100 * time.Millisecond + claimPollTimeout = 2 * time.Second + staleClaimAfter = 30 * time.Second + maxConvergeAttempts = 3 + integrationLockResource = "deployment" + outcomeRecorded = "recorded" + outcomeLockConflict = "lock_conflict" + outcomeDuplicate = "duplicate" + outcomeStale = "stale" ) +// setEventIDRetryDelays are the waits between SetEventID attempts, after the +// first one: 3 retries, 4 attempts in total. +var setEventIDRetryDelays = [...]time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond} + // errClaimPending signals that a correlation was claimed but the Tracker // event it will point to is not recorded yet (either still being created, or // abandoned and dropped). @@ -39,7 +44,7 @@ type DeploymentStore interface { Claim(ctx context.Context, key, source, status string, at time.Time, rank int) (bool, *store.IntegrationDeployment, error) Advance(ctx context.Context, key, status string, at time.Time, rank int) (bool, *store.IntegrationDeployment, error) SetEventID(ctx context.Context, key, eventID string) error - Revert(ctx context.Context, key string, prev *store.IntegrationDeployment, at time.Time) error + Revert(ctx context.Context, key string, prev *store.IntegrationDeployment, status string, rank int, at time.Time) error Delete(ctx context.Context, key string) error Get(ctx context.Context, key string) (*store.IntegrationDeployment, error) } @@ -59,42 +64,59 @@ type ProcessResult struct { type catalogEntry struct{ name, repository string } // serviceResolver keeps a snapshot of (name, normalized repository) for the -// whole catalog, reloaded at most once per ttl. A failed reload keeps the -// previous snapshot. +// whole catalog. A reload is single-flighted: while one caller reloads, the +// others get the previous snapshot immediately instead of waiting on it or +// on the exclusive lock. A successful reload is valid for ttl; a failed one +// keeps the previous snapshot and is retried after catalogRetryAfterFailure +// rather than the full ttl. type serviceResolver struct { catalog CatalogLister ttl time.Duration now func() time.Time logger *slog.Logger - mu sync.RWMutex - loadedAt time.Time - entries []catalogEntry + mu sync.Mutex + loading bool + nextReload time.Time + entries []catalogEntry } -// snapshot returns the current catalog snapshot, reloading it when it is -// older than ttl. At most one reload happens per ttl, whether it succeeds or -// fails. -func (r *serviceResolver) snapshot(ctx context.Context) []catalogEntry { - r.mu.RLock() - if !r.loadedAt.IsZero() && r.now().Sub(r.loadedAt) < r.ttl { +// snapshot returns the current catalog snapshot, triggering at most one +// concurrent reload. The reload itself runs outside the lock, against a +// context detached from the caller's request so that request cancellation +// never aborts a reload other callers rely on. +func (r *serviceResolver) snapshot() []catalogEntry { + r.mu.Lock() + if (!r.nextReload.IsZero() && r.now().Before(r.nextReload)) || r.loading { entries := r.entries - r.mu.RUnlock() + r.mu.Unlock() return entries } - r.mu.RUnlock() + r.loading = true + r.mu.Unlock() + + r.reload() r.mu.Lock() - defer r.mu.Unlock() - if !r.loadedAt.IsZero() && r.now().Sub(r.loadedAt) < r.ttl { - return r.entries - } + entries := r.entries + r.mu.Unlock() + return entries +} - catalogs, err := r.catalog.List(ctx) - r.loadedAt = r.now() +// reload calls the catalog once and installs the result, or keeps the +// previous snapshot and schedules a quick retry on failure. +func (r *serviceResolver) reload() { + reloadCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + catalogs, err := r.catalog.List(reloadCtx) + + r.mu.Lock() + defer r.mu.Unlock() + r.loading = false if err != nil { r.logger.Error("integration: catalog reload failed, keeping the previous snapshot", "error", err) - return r.entries + r.nextReload = r.now().Add(catalogRetryAfterFailure) + return } entries := make([]catalogEntry, 0, len(catalogs)) @@ -105,13 +127,13 @@ func (r *serviceResolver) snapshot(ctx context.Context) []catalogEntry { entries = append(entries, catalogEntry{name: c.Name, repository: integrations.NormalizeRepoURL(c.Repository)}) } r.entries = entries - return r.entries + r.nextReload = r.now().Add(r.ttl) } // Resolve matches hint's repository URLs against the catalog first, then its // name, and falls back to the name unchanged. -func (r *serviceResolver) Resolve(ctx context.Context, hint integrations.ServiceHint) string { - entries := r.snapshot(ctx) +func (r *serviceResolver) Resolve(_ context.Context, hint integrations.ServiceHint) string { + entries := r.snapshot() for _, url := range hint.RepositoryURLs { if url == "" { continue @@ -225,9 +247,33 @@ func (p *IntegrationProcessor) waitForEventID(ctx context.Context, key string) ( } } +// setEventIDWithRetry retries SetEventID after a transient failure, waiting +// setEventIDRetryDelays between attempts (4 attempts in total). +func (p *IntegrationProcessor) setEventIDWithRetry(ctx context.Context, key, eventID string) error { + var err error + for attempt := 0; ; attempt++ { + if err = p.store.SetEventID(ctx, key, eventID); err == nil { + return nil + } + if attempt >= len(setEventIDRetryDelays) { + return err + } + p.logger.Warn("integration: retrying SetEventID", "key", key, "eventId", eventID, "attempt", attempt+1, "error", err) + timer := time.NewTimer(setEventIDRetryDelays[attempt]) + select { + case <-ctx.Done(): + timer.Stop() + return err + case <-timer.C: + } + } +} + // create resolves the service, creates the Tracker event and records its id -// on the claim. A failure at either step drops the claim so a later -// notification can retry from scratch. +// on the claim. A failure creating the event drops the claim so a later +// notification can retry from scratch. A failure recording the event id, +// even after retrying, instead compensates: the event and any lock it holds +// are removed before the claim is dropped, so nothing is left orphaned. func (p *IntegrationProcessor) create(ctx context.Context, obs integrations.Observation) (ProcessResult, error) { service := p.services.Resolve(ctx, obs.Service) ev := newIntegrationEvent(obs, service) @@ -238,13 +284,23 @@ func (p *IntegrationProcessor) create(ctx context.Context, obs integrations.Obse } return ProcessResult{}, fmt.Errorf("create event: %w", err) } - if err := p.store.SetEventID(ctx, obs.Key, created.Metadata.Id); err != nil { - p.logger.Error("integration: cannot record the event id for the claim", "key", obs.Key, "eventId", created.Metadata.Id, "error", err) + + if err := p.setEventIDWithRetry(ctx, obs.Key, created.Metadata.Id); err != nil { + p.logger.Error("integration: giving up recording the event id for the claim, compensating", "key", obs.Key, "eventId", created.Metadata.Id, "error", err) + if unlockErr := p.events.lockService.UnlockByEventId(ctx, created.Metadata.Id); unlockErr != nil { + p.logger.Error("integration: cannot release the lock for the orphaned event", "eventId", created.Metadata.Id, "error", unlockErr) + } + if delErr := p.events.store.Delete(ctx, map[string]interface{}{"metadata.id": created.Metadata.Id}); delErr != nil { + p.logger.Error("integration: cannot delete the orphaned event", "eventId", created.Metadata.Id, "error", delErr) + } if delErr := p.store.Delete(ctx, obs.Key); delErr != nil { p.logger.Error("integration: cannot drop the claim after a failed event id update", "key", obs.Key, "error", delErr) } return ProcessResult{}, err } + + p.converge(ctx, obs.Key, created.Metadata.Id, obs.Status, obs.At, integrations.Rank(obs.Status)) + return ProcessResult{Outcome: outcomeFor(conflict), EventID: created.Metadata.Id}, nil } @@ -275,10 +331,16 @@ func newIntegrationEvent(obs integrations.Observation, service string) *eventv1. // update applies obs to the event already correlated with the claim. A lock // conflict is recorded as a changelog comment and never fails the request; // only a store or Tracker write failure does, and then the claim is reverted -// to its previous state so a later notification can retry. +// to its previous state so a later notification can retry. The result +// reported to the caller always reflects this request's own observation, +// even though converge (called last) may keep advancing the event if a +// concurrent observation has already moved the claim further. func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Observation, prev *store.IntegrationDeployment) (ProcessResult, error) { + status := obs.Status.String() + rank := integrations.Rank(obs.Status) + fail := func(err error) (ProcessResult, error) { - if revertErr := p.store.Revert(ctx, obs.Key, prev, obs.At); revertErr != nil { + if revertErr := p.store.Revert(ctx, obs.Key, prev, status, rank, obs.At); revertErr != nil { p.logger.Error("integration: cannot revert the claim after a failed event update", "key", obs.Key, "eventId", prev.EventID, "error", revertErr) } return ProcessResult{}, err @@ -303,22 +365,36 @@ func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Obse } } - next := nextIntegrationEvent(current, obs) + next := nextIntegrationEvent(current, obs.Status, obs.At, obs.Terminal) if _, err := p.events.updateEvent(ctx, current, next, map[string]interface{}{"metadata.id": prev.EventID}, obs.User, comment, obs.At); err != nil { return fail(err) } if takeLock { - lockID, c, err := p.events.observeLock(ctx, service, environment, integrationLockResource, obs.User, true) + // The lock is created with the event id already attached, in the + // same write: there is no separate attach step, and so no window + // where the lock exists without pointing at the event it guards. + lockID, c, err := p.events.observeLock(ctx, service, environment, integrationLockResource, obs.User, prev.EventID, true) if err != nil { p.logger.Warn("integration: cannot take lock after status change", "key", obs.Key, "eventId", prev.EventID, "error", err) } else if c != nil { conflict = c p.logger.Warn("integration: lock conflict while taking lock after status change", "key", obs.Key, "eventId", prev.EventID, "who", c.Who) } else if lockID != "" { - if _, err := p.events.lockService.updateLock(ctx, &lockv1.UpdateLockRequest{Id: lockID, EventId: prev.EventID}); err != nil { - p.logger.Warn("integration: cannot link lock to event", "key", obs.Key, "eventId", prev.EventID, "lockId", lockID, "error", err) + // A concurrent, later observation may have already brought the + // claim to a terminal status while this lock was being taken: + // release it right away rather than leave it stuck. + reread, rerr := p.store.Get(ctx, obs.Key) + if rerr != nil { + p.logger.Warn("integration: cannot re-read claim after taking lock, releasing it", "key", obs.Key, "eventId", prev.EventID, "lockId", lockID, "error", rerr) + if _, unlockErr := p.events.lockService.store.Unlock(ctx, map[string]interface{}{"id": lockID}); unlockErr != nil { + p.logger.Warn("integration: cannot release lock after a failed re-read", "key", obs.Key, "eventId", prev.EventID, "lockId", lockID, "error", unlockErr) + } + } else if val, ok := eventv1.Status_value[reread.Status]; ok && integrations.IsTerminal(eventv1.Status(val)) { + if err := p.events.lockService.UnlockByEventId(ctx, prev.EventID); err != nil { + p.logger.Warn("integration: cannot release lock taken for an already terminal claim", "key", obs.Key, "eventId", prev.EventID, "error", err) + } } } } @@ -329,13 +405,65 @@ func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Obse } } + p.converge(ctx, obs.Key, prev.EventID, obs.Status, obs.At, rank) + return ProcessResult{Outcome: outcomeFor(conflict), EventID: prev.EventID}, nil } +// converge re-reads the claim after a successful event write for key and, if +// a concurrent observation has since moved it further (a race this +// processor resolves without any per-key mutex or Mongo lease), re-applies +// the claim's own state to the event so the two never stay diverged. It +// repeats the re-read/re-apply up to maxConvergeAttempts times in total. It +// never changes the ProcessResult reported for this request's own +// observation: any failure here is logged and swallowed. +func (p *IntegrationProcessor) converge(ctx context.Context, key, eventID string, appliedStatus eventv1.Status, appliedAt time.Time, appliedRank int) { + status := appliedStatus.String() + at := store.NormalizeEventTime(appliedAt) + rank := appliedRank + + for i := 0; i < maxConvergeAttempts; i++ { + claim, err := p.store.Get(ctx, key) + if err != nil { + p.logger.Warn("integration: cannot re-read claim to converge", "key", key, "eventId", eventID, "error", err) + return + } + if claim.Status == status && claim.LastEventAt.Equal(at) && claim.Rank == rank { + return + } + + claimStatusValue, ok := eventv1.Status_value[claim.Status] + if !ok { + p.logger.Warn("integration: unknown claim status while converging", "key", key, "eventId", eventID, "status", claim.Status) + return + } + claimStatus := eventv1.Status(claimStatusValue) + terminal := integrations.IsTerminal(claimStatus) + + current, err := p.events.store.Get(ctx, map[string]interface{}{"metadata.id": eventID}) + if err != nil { + p.logger.Warn("integration: cannot read event to converge", "key", key, "eventId", eventID, "error", err) + return + } + next := nextIntegrationEvent(current, claimStatus, claim.LastEventAt, terminal) + if _, err := p.events.updateEvent(ctx, current, next, map[string]interface{}{"metadata.id": eventID}, "system", "", claim.LastEventAt); err != nil { + p.logger.Warn("integration: cannot converge event to claim state", "key", key, "eventId", eventID, "error", err) + return + } + if terminal { + if err := p.events.lockService.UnlockByEventId(ctx, eventID); err != nil { + p.logger.Warn("integration: cannot release lock while converging", "key", key, "eventId", eventID, "error", err) + } + } + + status, at, rank = claim.Status, claim.LastEventAt, claim.Rank + } +} + // nextIntegrationEvent builds the requested state for updateEvent: current -// copied field by field, with the observed status and, for a terminal -// status, an end date. -func nextIntegrationEvent(current *eventv1.Event, obs integrations.Observation) *eventv1.Event { +// copied field by field, with status and, for a terminal status, an end +// date set to at (the existing start date is kept either way). +func nextIntegrationEvent(current *eventv1.Event, status eventv1.Status, at time.Time, terminal bool) *eventv1.Event { attrs := &eventv1.EventAttributes{ Message: current.Attributes.Message, Source: current.Attributes.Source, @@ -343,7 +471,7 @@ func nextIntegrationEvent(current *eventv1.Event, obs integrations.Observation) Priority: current.Attributes.Priority, RelatedId: current.Attributes.RelatedId, Service: current.Attributes.Service, - Status: obs.Status, + Status: status, Environment: current.Attributes.Environment, Impact: current.Attributes.Impact, StartDate: current.Attributes.StartDate, @@ -353,8 +481,8 @@ func nextIntegrationEvent(current *eventv1.Event, obs integrations.Observation) Notification: current.Attributes.Notification, Notifications: current.Attributes.Notifications, } - if obs.Terminal { - attrs.EndDate = timestamppb.New(obs.At) + if terminal { + attrs.EndDate = timestamppb.New(at) } links := &eventv1.EventLinks{} diff --git a/server/integrations_processor_test.go b/server/integrations_processor_test.go index 6b46686..b3cdf39 100644 --- a/server/integrations_processor_test.go +++ b/server/integrations_processor_test.go @@ -3,6 +3,7 @@ package server import ( "context" "errors" + "fmt" "io" "log/slog" "sync" @@ -15,6 +16,7 @@ import ( "github.com/bananaops/tracker/internal/integrations" store "github.com/bananaops/tracker/internal/stores" "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" ) @@ -112,6 +114,17 @@ func TestProcessCreateThenUpdate(t *testing.T) { require.Equal(t, eventv1.Status_success, updated.Attributes.Status) require.True(t, updated.Attributes.EndDate.AsTime().Equal(t1)) + var statusChanged *eventv1.ChangelogEntry + for _, e := range updated.Changelog { + if e.ChangeType == eventv1.ChangeType_status_changed { + statusChanged = e + } + } + require.NotNil(t, statusChanged) + require.Equal(t, "start", statusChanged.OldValue) + require.Equal(t, "success", statusChanged.NewValue) + require.Equal(t, "gitlab:jdoe", statusChanged.User) + locks, err = store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) require.NoError(t, err) require.Len(t, locks, 0) @@ -326,6 +339,8 @@ func TestProcessWaitsForPendingClaim(t *testing.T) { require.NoError(t, err) require.Equal(t, outcomeRecorded, res.Outcome) require.Equal(t, expected, res.EventID) + + require.NoError(t, <-errCh) } func TestProcessPendingClaimTimesOut(t *testing.T) { @@ -360,6 +375,127 @@ func TestProcessAbandonedClaimIsDropped(t *testing.T) { require.ErrorIs(t, err, store.ErrNotFound) } +// failingSetEventIDStore wraps a real store and always fails SetEventID, to +// exercise create()'s compensation path (retry, then delete the event and +// its lock, then drop the claim). +type failingSetEventIDStore struct { + *store.IntegrationDeploymentStore +} + +func (f *failingSetEventIDStore) SetEventID(context.Context, string, string) error { + return errors.New("set event id always fails") +} + +func TestProcessSetEventIDFailureCompensates(t *testing.T) { + db := testMongoDatabase(t) + events := newTestEvent(t, db) + real := store.NewIntegrationDeploymentStoreFromCollection(db.Collection("integration_deployments")) + failing := &failingSetEventIDStore{IntegrationDeploymentStore: real} + cat := &fakeCatalog{} + proc := NewIntegrationProcessor(events, failing, cat, slog.New(slog.NewJSONHandler(io.Discard, nil))) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + _, err := proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.Error(t, err) + + remainingEvents, err := events.store.List(ctx) + require.NoError(t, err) + require.Len(t, remainingEvents, 0) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 0) + + _, err = real.Get(ctx, obs(eventv1.Status_start, t0).Key) + require.ErrorIs(t, err, store.ErrNotFound) +} + +// TestProcessConcurrentSameInstantConvergesOnSuccess fires "running" and +// "success" concurrently for the same deployment, same status_changed_at +// second, on a fresh key each time. Whichever request's own write lands +// first, convergence (no per-key mutex, no Mongo lease) must always leave +// exactly one event at status success and no lock behind. +func TestProcessConcurrentSameInstantConvergesOnSuccess(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + + for i := 0; i < 20; i++ { + key := fmt.Sprintf("gitlab:gitlab.example.com:concurrent-%d", i) + rev := fmt.Sprintf("rev%d", i) + t0 := time.Now().UTC().Truncate(time.Second) + + running := obs(eventv1.Status_start, t0) + running.Key, running.ShortRevision = key, rev + success := obs(eventv1.Status_success, t0) + success.Key, success.ShortRevision = key, rev + + var wg sync.WaitGroup + errs := make([]error, 2) + wg.Add(2) + go func() { defer wg.Done(); _, errs[0] = env.proc.Process(ctx, running) }() + go func() { defer wg.Done(); _, errs[1] = env.proc.Process(ctx, success) }() + wg.Wait() + require.NoError(t, errs[0], "iteration %d", i) + require.NoError(t, errs[1], "iteration %d", i) + + title := "Deploy payments " + rev + " to production" + count, err := env.db.Collection("events").CountDocuments(ctx, bson.M{"title": title}) + require.NoError(t, err) + require.Equal(t, int64(1), count, "iteration %d: expected exactly one event", i) + + doc, err := env.store.Get(ctx, key) + require.NoError(t, err) + require.Equal(t, "success", doc.Status, "iteration %d", i) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": doc.EventID}) + require.NoError(t, err) + require.Equal(t, eventv1.Status_success, ev.Attributes.Status, "iteration %d", i) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + for _, l := range locks { + require.NotEqual(t, doc.EventID, l.EventId, "iteration %d: lock still attached to event", i) + } + } +} + +// TestProcessOutOfOrderSuccessBeforeRunningStaysSuccess delivers "success" +// strictly before "running" for the same deployment and instant: the later, +// lower-rank "running" must never move the already-terminal event backwards. +func TestProcessOutOfOrderSuccessBeforeRunningStaysSuccess(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + key := "gitlab:gitlab.example.com:out-of-order" + + success := obs(eventv1.Status_success, t0) + success.Key = key + res, err := env.proc.Process(ctx, success) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + + running := obs(eventv1.Status_start, t0) + running.Key = key + res2, err := env.proc.Process(ctx, running) + require.NoError(t, err) + require.Equal(t, outcomeStale, res2.Outcome) + + doc, err := env.store.Get(ctx, key) + require.NoError(t, err) + require.Equal(t, "success", doc.Status) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": doc.EventID}) + require.NoError(t, err) + require.Equal(t, eventv1.Status_success, ev.Attributes.Status) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + for _, l := range locks { + require.NotEqual(t, doc.EventID, l.EventId) + } +} + func TestServiceResolver(t *testing.T) { cat := &fakeCatalog{entries: []*catalogv1.Catalog{ {Name: "payments-api", Repository: "git@gitlab.example.com:team/payments.git"}, From e7d0cf7eb729df1f095ebab0f4e598b69856efac Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 14:52:17 +0200 Subject: [PATCH 12/22] fix(integrations): rank waiting_approval below start Rank now gives waiting_approval 0 instead of 1, tied with start: a later start always takes over an approval, matching the processor's create/update flow. Refs #208 --- internal/integrations/observation.go | 7 ++++--- internal/integrations/observation_test.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/integrations/observation.go b/internal/integrations/observation.go index 5828d78..529764e 100644 --- a/internal/integrations/observation.go +++ b/internal/integrations/observation.go @@ -50,11 +50,12 @@ func (o Observation) Title(service string) string { return strings.Join(parts, " ") } -// Rank orders statuses for the same instant: 1 for start and -// waiting_approval, 2 for terminal statuses, 0 for anything else. +// Rank orders statuses for the same instant: 1 for start, 2 for terminal +// statuses, 0 for anything else, including waiting_approval, which is +// ranked below start so a later start always takes over an approval. func Rank(s eventv1.Status) int { switch s { - case eventv1.Status_start, eventv1.Status_waiting_approval: + case eventv1.Status_start: return 1 case eventv1.Status_success, eventv1.Status_failure, eventv1.Status_warning, eventv1.Status_close: return 2 diff --git a/internal/integrations/observation_test.go b/internal/integrations/observation_test.go index 65016bb..c94503e 100644 --- a/internal/integrations/observation_test.go +++ b/internal/integrations/observation_test.go @@ -14,7 +14,7 @@ func TestRank(t *testing.T) { want int }{ {eventv1.Status_start, 1}, - {eventv1.Status_waiting_approval, 1}, + {eventv1.Status_waiting_approval, 0}, {eventv1.Status_success, 2}, {eventv1.Status_failure, 2}, {eventv1.Status_warning, 2}, From 0c0ba174809239e3902dc956bf3ab634d29c4942 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 14:52:23 +0200 Subject: [PATCH 13/22] fix(server): take integration locks with their event id The observe-mode create path now creates the event first, then takes the lock with the event id already set in the same write, reusing the round 1 observeLock path instead of a separate attach step; a conflict is recorded as a changelog comment via a follow-up write, and any other failure taking the lock is only logged. After taking the lock, the claim is re-read and the lock released immediately if it has already gone terminal, same as the update path. In update, takeLock is now derived from the claim state read before Advance (prev), never from the event's current status, since converge can rewrite it concurrently. Tests: every lock assertion in the concurrent and out-of-order processor tests now checks no lock exists at all for the service/environment, not only by event id; added an update-path race test (blocked, then running/success concurrently, 20 iterations) and a create-path lock-conflict test. Refs #208 --- server/event.go | 139 ++++++++++++++++---------- server/integrations_processor.go | 38 ++++++- server/integrations_processor_test.go | 114 +++++++++++++++++++-- 3 files changed, 228 insertions(+), 63 deletions(-) diff --git a/server/event.go b/server/event.go index f5e50cb..b796ed0 100644 --- a/server/event.go +++ b/server/event.go @@ -198,8 +198,12 @@ func (e *Event) CreateEvent( // createEvent is CreateEvent without authorization. mode controls what // happens when the service is already locked: lockStrict fails the creation -// (the RPC behaviour), lockObserve records the conflict on the event instead -// (the integration behaviour) and creates it without taking the lock. +// (the RPC behaviour, unchanged) and takes its lock before creating the +// event, attaching it afterward once the id is known. lockObserve (the +// integration behaviour) creates the event first, then takes the lock with +// the event id already known, in the same write: a conflict with an +// existing lock is recorded as a changelog comment instead of failing the +// creation, and any other failure taking the lock is only logged. func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, comment string, mode lockMode) (*v1alpha1.Event, *lockConflict, error) { attrs := event.Attributes environment := attrs.Environment.String() @@ -210,65 +214,38 @@ func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, co addChangelogEntry(event, v1alpha1.ChangeType_created, user, "", "", "", "Event created") var lockID string - var conflict *lockConflict - switch mode { - case lockStrict: - if shouldCreateLock(attrs.Type, attrs.Status) { - res, err := e.lockService.createLock(ctx, &lock.CreateLockRequest{ - Service: attrs.Service, - Who: user, - Environment: environment, - Resource: resource, - EventId: "", - }) - if err != nil { - e.logger.Error("failed to create lock", - "service", attrs.Service, - "environment", environment, - "resource", resource, - "error", err, - ) - - if strings.Contains(err.Error(), "already locked") { - return nil, nil, fmt.Errorf("cannot create event: service %s is already locked in %s. Please unlock it first", - attrs.Service, environment) - } - - return nil, nil, fmt.Errorf("cannot create event: failed to create lock - %v", err) - } - lockID = res.Lock.Id - } - case lockObserve: - if resource != "unknown" { - // The event does not have an id yet at this point, so the lock - // (if taken) is attached to it below, once the event is created. - var err error - lockID, conflict, err = e.observeLock(ctx, attrs.Service, environment, resource, user, "", shouldCreateLock(attrs.Type, attrs.Status)) - if err != nil { - return nil, nil, err + if mode == lockStrict && shouldCreateLock(attrs.Type, attrs.Status) { + res, err := e.lockService.createLock(ctx, &lock.CreateLockRequest{ + Service: attrs.Service, + Who: user, + Environment: environment, + Resource: resource, + EventId: "", + }) + if err != nil { + e.logger.Error("failed to create lock", + "service", attrs.Service, + "environment", environment, + "resource", resource, + "error", err, + ) + + if strings.Contains(err.Error(), "already locked") { + return nil, nil, fmt.Errorf("cannot create event: service %s is already locked in %s. Please unlock it first", + attrs.Service, environment) } + + return nil, nil, fmt.Errorf("cannot create event: failed to create lock - %v", err) } + lockID = res.Lock.Id } - if conflict != nil { - addChangelogEntry(event, v1alpha1.ChangeType_commented, user, "", "", "", lockConflictComment(attrs.Service, environment, conflict.Who)) - } if comment != "" { addChangelogEntry(event, v1alpha1.ChangeType_commented, user, "", "", "", comment) } created, err := e.store.Create(context.Background(), event) if err != nil { - if mode == lockObserve && lockID != "" { - if _, unlockErr := e.lockService.store.Unlock(ctx, map[string]interface{}{"id": lockID}); unlockErr != nil { - e.logger.Error("failed to release lock after failed event creation", - "lock_id", lockID, - "service", attrs.Service, - "environment", environment, - "error", unlockErr, - ) - } - } return nil, nil, err } @@ -295,6 +272,11 @@ func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, co } } + var conflict *lockConflict + if mode == lockObserve && resource != "unknown" { + created, conflict = e.observeLockAfterCreate(ctx, created, user, shouldCreateLock(attrs.Type, attrs.Status)) + } + // log event to json format e.logger.Info("event created", "title", created.Title, @@ -339,6 +321,61 @@ func (e *Event) observeLock(ctx context.Context, service, environment, resource, return res.Lock.Id, nil, nil } +// observeLockAfterCreate takes the lock for an event that was just created, +// with the event id already known, in the same write. It never fails the +// creation: a conflict with an existing lock is recorded as a changelog +// comment on the event, and any other failure is only logged. It returns +// the event as currently stored, reloaded when the lock or the comment +// changed it. +func (e *Event) observeLockAfterCreate(ctx context.Context, created *v1alpha1.Event, user string, take bool) (*v1alpha1.Event, *lockConflict) { + attrs := created.Attributes + environment := attrs.Environment.String() + resource := getResourceType(attrs.Type) + + lockID, conflict, err := e.observeLock(ctx, attrs.Service, environment, resource, user, created.Metadata.Id, take) + if err != nil { + e.logger.Warn("failed to take lock after creating event", + "service", attrs.Service, + "environment", environment, + "resource", resource, + "event_id", created.Metadata.Id, + "error", err, + ) + return created, nil + } + + if conflict != nil { + ev, gerr := e.store.Get(context.Background(), map[string]interface{}{"metadata.id": created.Metadata.Id}) + if gerr != nil { + e.logger.Warn("failed to reload event to record lock conflict comment", "event_id", created.Metadata.Id, "error", gerr) + return created, conflict + } + addChangelogEntry(ev, v1alpha1.ChangeType_commented, user, "", "", "", lockConflictComment(attrs.Service, environment, conflict.Who)) + // Update returns the document as it was before the $set (no + // ReturnDocument(After) option), so the caller gets the locally + // mutated copy, not that stale snapshot. + if _, uerr := e.store.Update(context.Background(), map[string]interface{}{"metadata.id": created.Metadata.Id}, ev); uerr != nil { + e.logger.Warn("failed to record lock conflict comment", "event_id", created.Metadata.Id, "error", uerr) + return created, conflict + } + return ev, conflict + } + + if lockID == "" { + return created, nil + } + + // createLock already appended the "Service locked in %s" changelog entry + // since the event id was set in the same write: reload so the returned + // event reflects it. + ev, gerr := e.store.Get(context.Background(), map[string]interface{}{"metadata.id": created.Metadata.Id}) + if gerr != nil { + e.logger.Warn("failed to reload event after taking lock", "event_id", created.Metadata.Id, "error", gerr) + return created, nil + } + return ev, nil +} + func (e *Event) GetEvent( ctx context.Context, i *v1alpha1.GetEventRequest, diff --git a/server/integrations_processor.go b/server/integrations_processor.go index 5b790a7..6adefb4 100644 --- a/server/integrations_processor.go +++ b/server/integrations_processor.go @@ -299,6 +299,19 @@ func (p *IntegrationProcessor) create(ctx context.Context, obs integrations.Obse return ProcessResult{}, err } + // A lock taken while creating the event may already be stale by the time + // the claim carries the event id: re-read it and release the lock right + // away if it has gone terminal, same as the update path. + if shouldCreateLock(eventv1.Type_deployment, obs.Status) { + if reread, rerr := p.store.Get(ctx, obs.Key); rerr != nil { + p.logger.Warn("integration: cannot re-read claim after creating event", "key", obs.Key, "eventId", created.Metadata.Id, "error", rerr) + } else if terminalStatus(reread.Status) { + if err := p.events.lockService.UnlockByEventId(ctx, created.Metadata.Id); err != nil { + p.logger.Warn("integration: cannot release lock taken for an already terminal claim", "key", obs.Key, "eventId", created.Metadata.Id, "error", err) + } + } + } + p.converge(ctx, obs.Key, created.Metadata.Id, obs.Status, obs.At, integrations.Rank(obs.Status)) return ProcessResult{Outcome: outcomeFor(conflict), EventID: created.Metadata.Id}, nil @@ -356,7 +369,11 @@ func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Obse var conflict *lockConflict comment := obs.Comment - takeLock := obs.Status == eventv1.Status_start && current.Attributes.Status != eventv1.Status_start + // takeLock is derived from the claim state read before this Advance + // (prev), never from the event's current status: converge can rewrite + // the event to a later claim state concurrently, which would otherwise + // make this decision flicker. + takeLock := obs.Status == eventv1.Status_start && prev.Status != eventv1.Status_start.String() && !terminalStatus(prev.Status) if takeLock { if held := p.events.lockService.findLock(ctx, service, environment, integrationLockResource); held != nil { conflict = &lockConflict{Who: held.Who} @@ -391,7 +408,7 @@ func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Obse if _, unlockErr := p.events.lockService.store.Unlock(ctx, map[string]interface{}{"id": lockID}); unlockErr != nil { p.logger.Warn("integration: cannot release lock after a failed re-read", "key", obs.Key, "eventId", prev.EventID, "lockId", lockID, "error", unlockErr) } - } else if val, ok := eventv1.Status_value[reread.Status]; ok && integrations.IsTerminal(eventv1.Status(val)) { + } else if terminalStatus(reread.Status) { if err := p.events.lockService.UnlockByEventId(ctx, prev.EventID); err != nil { p.logger.Warn("integration: cannot release lock taken for an already terminal claim", "key", obs.Key, "eventId", prev.EventID, "error", err) } @@ -432,12 +449,11 @@ func (p *IntegrationProcessor) converge(ctx context.Context, key, eventID string return } - claimStatusValue, ok := eventv1.Status_value[claim.Status] + claimStatus, ok := parseStatus(claim.Status) if !ok { p.logger.Warn("integration: unknown claim status while converging", "key", key, "eventId", eventID, "status", claim.Status) return } - claimStatus := eventv1.Status(claimStatusValue) terminal := integrations.IsTerminal(claimStatus) current, err := p.events.store.Get(ctx, map[string]interface{}{"metadata.id": eventID}) @@ -510,3 +526,17 @@ func outcomeFor(conflict *lockConflict) string { } return outcomeRecorded } + +// parseStatus turns a claim's stored status string back into eventv1.Status. +// ok is false for a string that names no known status. +func parseStatus(status string) (s eventv1.Status, ok bool) { + val, known := eventv1.Status_value[status] + return eventv1.Status(val), known +} + +// terminalStatus reports whether a claim's stored status string names a +// terminal eventv1.Status. An unrecognized string is treated as not terminal. +func terminalStatus(status string) bool { + s, ok := parseStatus(status) + return ok && integrations.IsTerminal(s) +} diff --git a/server/integrations_processor_test.go b/server/integrations_processor_test.go index b3cdf39..1253186 100644 --- a/server/integrations_processor_test.go +++ b/server/integrations_processor_test.go @@ -61,6 +61,16 @@ func newProcEnv(t *testing.T) *procEnv { return &procEnv{db: db, events: events, store: st, proc: proc, cat: cat} } +// assertNoLockForService fails the test if any lock exists for +// service/environment at all, regardless of which event id (if any) it +// carries. +func assertNoLockForService(t *testing.T, ctx context.Context, db *mongo.Database, service, environment string) { + t.Helper() + count, err := db.Collection("locks").CountDocuments(ctx, bson.M{"service": service, "environment": environment}) + require.NoError(t, err) + require.Equal(t, int64(0), count, "expected no lock for %s/%s", service, environment) +} + func obs(status eventv1.Status, at time.Time) integrations.Observation { return integrations.Observation{ Key: "gitlab:gitlab.example.com:42", @@ -452,11 +462,7 @@ func TestProcessConcurrentSameInstantConvergesOnSuccess(t *testing.T) { require.NoError(t, err) require.Equal(t, eventv1.Status_success, ev.Attributes.Status, "iteration %d", i) - locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) - require.NoError(t, err) - for _, l := range locks { - require.NotEqual(t, doc.EventID, l.EventId, "iteration %d: lock still attached to event", i) - } + assertNoLockForService(t, ctx, env.db, "payments", "production") } } @@ -489,11 +495,103 @@ func TestProcessOutOfOrderSuccessBeforeRunningStaysSuccess(t *testing.T) { require.NoError(t, err) require.Equal(t, eventv1.Status_success, ev.Attributes.Status) - locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + assertNoLockForService(t, ctx, env.db, "payments", "production") +} + +// TestProcessUpdatePathRaceAfterApproval processes "blocked" (waiting +// approval, no lock) first so the claim and event both exist, then fires +// "running" and "success" concurrently for the same instant: the update +// path's takeLock, derived from the claim state read before Advance, and +// convergence must always settle on exactly one event at status success and +// no lock left over. +func TestProcessUpdatePathRaceAfterApproval(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + + for i := 0; i < 20; i++ { + key := fmt.Sprintf("gitlab:gitlab.example.com:update-race-%d", i) + rev := fmt.Sprintf("uprev%d", i) + t0 := time.Now().UTC().Truncate(time.Second) + + blocked := obs(eventv1.Status_waiting_approval, t0) + blocked.Key, blocked.ShortRevision = key, rev + res, err := env.proc.Process(ctx, blocked) + require.NoError(t, err, "iteration %d", i) + require.Equal(t, outcomeRecorded, res.Outcome, "iteration %d", i) + assertNoLockForService(t, ctx, env.db, "payments", "production") + + t1 := t0.Add(5 * time.Second) + running := obs(eventv1.Status_start, t1) + running.Key, running.ShortRevision = key, rev + success := obs(eventv1.Status_success, t1) + success.Key, success.ShortRevision = key, rev + + var wg sync.WaitGroup + errs := make([]error, 2) + wg.Add(2) + go func() { defer wg.Done(); _, errs[0] = env.proc.Process(ctx, running) }() + go func() { defer wg.Done(); _, errs[1] = env.proc.Process(ctx, success) }() + wg.Wait() + require.NoError(t, errs[0], "iteration %d", i) + require.NoError(t, errs[1], "iteration %d", i) + + title := "Deploy payments " + rev + " to production" + count, err := env.db.Collection("events").CountDocuments(ctx, bson.M{"title": title}) + require.NoError(t, err) + require.Equal(t, int64(1), count, "iteration %d: expected exactly one event", i) + + doc, err := env.store.Get(ctx, key) + require.NoError(t, err) + require.Equal(t, "success", doc.Status, "iteration %d", i) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": doc.EventID}) + require.NoError(t, err) + require.Equal(t, eventv1.Status_success, ev.Attributes.Status, "iteration %d", i) + + assertNoLockForService(t, ctx, env.db, "payments", "production") + } +} + +// TestProcessCreatePathLockConflictKeepsOneLock exercises the observe-mode +// create path when the service is already locked: the event is created and +// recorded regardless, the conflict is recorded as a changelog comment, the +// result is lock_conflict, and the existing lock is left as the only one for +// that service/environment (no second lock taken). +func TestProcessCreatePathLockConflictKeepsOneLock(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + _, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).Create(ctx, &lockv1.Lock{ + Service: "payments", Environment: "production", Resource: "deployment", Who: "alice", + }) require.NoError(t, err) - for _, l := range locks { - require.NotEqual(t, doc.EventID, l.EventId) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + require.Equal(t, outcomeLockConflict, res.Outcome) + require.NotEmpty(t, res.EventID) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + var commented *eventv1.ChangelogEntry + for _, e := range ev.Changelog { + if e.ChangeType == eventv1.ChangeType_commented { + commented = e + } } + require.NotNil(t, commented) + require.Equal(t, "Deployed while payments was locked in production by alice", commented.Comment) + + count, err := env.db.Collection("locks").CountDocuments(ctx, bson.M{"service": "payments", "environment": "production"}) + require.NoError(t, err) + require.Equal(t, int64(1), count, "expected the existing lock to be the only one") + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "alice", locks[0].Who) + require.Equal(t, "", locks[0].EventId) } func TestServiceResolver(t *testing.T) { From bb869234645ea9f52469bbe6e00ebcb9c8159c31 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 14:58:52 +0200 Subject: [PATCH 14/22] test(server): pin approval and start ordering for integrations Adds two processor tests: waiting_approval and start sharing the same status_changed_at second still lets start take the lock (higher rank wins the tie), and start followed by a later waiting_approval is rejected as stale, leaving the event and its lock untouched. Comments only otherwise: the Rank doc comment now states that a later waiting_approval never moves an event back from start, and the observeLock comment no longer describes the round 1 separate-attach behaviour removed in round 2. Refs #208 --- internal/integrations/observation.go | 3 +- server/event.go | 10 +++-- server/integrations_processor_test.go | 56 +++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/internal/integrations/observation.go b/internal/integrations/observation.go index 529764e..a89e2c2 100644 --- a/internal/integrations/observation.go +++ b/internal/integrations/observation.go @@ -52,7 +52,8 @@ func (o Observation) Title(service string) string { // Rank orders statuses for the same instant: 1 for start, 2 for terminal // statuses, 0 for anything else, including waiting_approval, which is -// ranked below start so a later start always takes over an approval. +// ranked below start so a later start always takes over an approval. A +// later waiting_approval, conversely, never moves an event back from start. func Rank(s eventv1.Status) int { switch s { case eventv1.Status_start: diff --git a/server/event.go b/server/event.go index b796ed0..d051094 100644 --- a/server/event.go +++ b/server/event.go @@ -296,10 +296,12 @@ func (e *Event) createEvent(ctx context.Context, event *v1alpha1.Event, user, co return created, conflict, nil } -// observeLock takes the lock when take is true and nobody holds it, in the -// same write as eventId when the caller already knows it (an empty eventId -// leaves the lock unattached; the caller links it once the event exists). A -// lock held by someone else is reported, never returned as an error. +// observeLock takes the lock when take is true and nobody holds it, with +// eventID already set in the same write: every observe-mode caller knows +// the event id up front (observeLockAfterCreate once the event is created, +// the processor's update path from the claim it already correlated), so +// there is no separate attach step. A lock held by someone else is +// reported, never returned as an error. func (e *Event) observeLock(ctx context.Context, service, environment, resource, who, eventID string, take bool) (string, *lockConflict, error) { if held := e.lockService.findLock(ctx, service, environment, resource); held != nil { return "", &lockConflict{Who: held.Who}, nil diff --git a/server/integrations_processor_test.go b/server/integrations_processor_test.go index 1253186..596008b 100644 --- a/server/integrations_processor_test.go +++ b/server/integrations_processor_test.go @@ -294,6 +294,62 @@ func TestProcessApprovalThenStartTakesLock(t *testing.T) { require.Len(t, locks, 0) } +// TestProcessApprovalThenRunningSameInstantTakesLock covers waiting_approval +// and start sharing the same status_changed_at second: start's higher rank +// (1 vs 0) still applies and takes the lock, exactly as when they are a few +// seconds apart. +func TestProcessApprovalThenRunningSameInstantTakesLock(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_waiting_approval, t0)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + + res2, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res2.Outcome) + require.Equal(t, res.EventID, res2.EventID) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + require.Equal(t, eventv1.Status_start, ev.Attributes.Status) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, "payments", locks[0].Service) + require.Equal(t, "production", locks[0].Environment) + require.Equal(t, res.EventID, locks[0].EventId) +} + +// TestProcessRunningThenLaterApprovalIsRejected covers the opposite +// ordering: a waiting_approval arriving after start must never move the +// event back, regardless of how much later it is. +func TestProcessRunningThenLaterApprovalIsRejected(t *testing.T) { + env := newProcEnv(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + res, err := env.proc.Process(ctx, obs(eventv1.Status_start, t0)) + require.NoError(t, err) + require.Equal(t, outcomeRecorded, res.Outcome) + + res2, err := env.proc.Process(ctx, obs(eventv1.Status_waiting_approval, t0.Add(5*time.Second))) + require.NoError(t, err) + require.Equal(t, outcomeStale, res2.Outcome) + + ev, err := env.events.store.Get(ctx, map[string]interface{}{"metadata.id": res.EventID}) + require.NoError(t, err) + require.Equal(t, eventv1.Status_start, ev.Attributes.Status) + + locks, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).List(ctx) + require.NoError(t, err) + require.Len(t, locks, 1) + require.Equal(t, res.EventID, locks[0].EventId) +} + func TestProcessUpdateFailureReverts(t *testing.T) { env := newProcEnv(t) ctx := context.Background() From 2869ac55bfee856fc33a6cc7e28fe6671454b3e3 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:13:20 +0200 Subject: [PATCH 15/22] feat(server): record deployments from GitLab and Flux webhooks POST /api/v1alpha1/integrations/gitlab/webhook and /api/v1alpha1/integrations/flux/webhook are registered only when their source is configured. The signature is the authentication (1 MiB body limit, check before any JSON decoding); responses are 200 recorded, 202 ignored, 400, 401, 413 or 500. Adds tracker_integration_webhooks_total{source,result}. Refs #208 --- cmd/serv.go | 8 + server/integrations.go | 258 +++++++++++ server/integrations_test.go | 654 ++++++++++++++++++++++++++++ server/integrations_testing_test.go | 252 +++++++++++ 4 files changed, 1172 insertions(+) create mode 100644 server/integrations.go create mode 100644 server/integrations_test.go create mode 100644 server/integrations_testing_test.go diff --git a/cmd/serv.go b/cmd/serv.go index d9b2950..2369a77 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -188,6 +188,14 @@ var serv = &cobra.Command{ // Register custom links CRUD endpoints server.RegisterLinksHandler(mux) + // Register the GitLab and Flux deployment webhooks (configured sources only). + // NewIntegrationDeps opens collections, so it only runs when a source is set. + if integrationsCfg.Enabled() { + if err := server.RegisterIntegrationHandlers(mux, integrationsCfg, server.NewIntegrationDeps(events)); err != nil { + log.Fatalf("cannot register integration webhooks: %v", err) + } + } + // Setup Swagger documentation with go-swagger opts := middleware.SwaggerUIOpts{SpecURL: "/swagger.json"} sh := middleware.SwaggerUI(opts, nil) diff --git a/server/integrations.go b/server/integrations.go new file mode 100644 index 0000000..a74a673 --- /dev/null +++ b/server/integrations.go @@ -0,0 +1,258 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "time" + + "github.com/bananaops/tracker/internal/auth" + "github.com/bananaops/tracker/internal/auth/authz" + "github.com/bananaops/tracker/internal/config" + "github.com/bananaops/tracker/internal/integrations" + store "github.com/bananaops/tracker/internal/stores" + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/prometheus/client_golang/prometheus" +) + +// integrationWebhooks counts deployment webhooks received, by source (gitlab, +// flux) and result. The result matches one of the resultXxx constants below. +var integrationWebhooks = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "tracker_integration_webhooks_total", + Help: "Deployment webhooks received, by source and result", + }, + []string{"source", "result"}, +) + +func init() { + prometheus.MustRegister(integrationWebhooks) +} + +const ( + integrationGitLabPath = "/api/v1alpha1/integrations/gitlab/webhook" + integrationFluxPath = "/api/v1alpha1/integrations/flux/webhook" + integrationMaxBodyBytes = 1 << 20 + integrationProcessTimeout = 15 * time.Second + + resultRecorded = "recorded" + resultIgnored = "ignored" + resultDuplicate = "duplicate" + resultUnauthorized = "unauthorized" + resultInvalid = "invalid" + resultError = "error" + resultLockConflict = "lock_conflict" +) + +// IntegrationDeps are the dependencies wired into the GitLab and Flux webhook +// handlers by RegisterIntegrationHandlers. +type IntegrationDeps struct { + Events *Event + Deployments DeploymentStore + Catalog CatalogLister + Logger *slog.Logger + Now func() time.Time +} + +// NewIntegrationDeps builds the production IntegrationDeps for events. It +// opens the deployment and catalog collections, so it must only be called +// once a deployment integration source is actually configured. +func NewIntegrationDeps(events *Event) IntegrationDeps { + return IntegrationDeps{ + Events: events, + Deployments: store.NewIntegrationDeploymentStore(), + Catalog: store.NewStoreCatalog(config.ConfigDatabase.CatalogCollection), + Logger: slog.New(slog.NewJSONHandler(os.Stdout, nil)), + Now: time.Now, + } +} + +// integrationHandler serves the GitLab and Flux webhook endpoints. +type integrationHandler struct { + cfg integrations.Config + processor *IntegrationProcessor + logger *slog.Logger + now func() time.Time +} + +// RegisterIntegrationHandlers registers the GitLab and Flux webhook handlers +// for the sources cfg configures. It registers nothing and returns nil when +// cfg.Enabled() is false. +func RegisterIntegrationHandlers(mux *runtime.ServeMux, cfg integrations.Config, deps IntegrationDeps) error { + if !cfg.Enabled() { + return nil + } + if deps.Events == nil || deps.Deployments == nil || deps.Catalog == nil { + return errors.New("integrations: Events, Deployments and Catalog are required") + } + logger := deps.Logger + if logger == nil { + logger = slog.Default() + } + now := deps.Now + if now == nil { + now = time.Now + } + + h := &integrationHandler{ + cfg: cfg, + processor: NewIntegrationProcessor(deps.Events, deps.Deployments, deps.Catalog, logger), + logger: logger, + now: now, + } + + if cfg.GitLabEnabled() { + if err := mux.HandlePath(http.MethodPost, integrationGitLabPath, authz.RequireHTTP(auth.PermPublic, h.handleGitLab)); err != nil { + return fmt.Errorf("register gitlab webhook: %w", err) + } + } + if cfg.FluxEnabled() { + if err := mux.HandlePath(http.MethodPost, integrationFluxPath, authz.RequireHTTP(auth.PermPublic, h.handleFlux)); err != nil { + return fmt.Errorf("register flux webhook: %w", err) + } + } + return nil +} + +// Response bodies. Field names match the wire contract in the design spec. +type ( + integrationRecorded struct { + Status string `json:"status"` + EventID string `json:"eventId"` + } + integrationIgnored struct { + Status string `json:"status"` + Reason string `json:"reason"` + } + integrationError struct { + Error string `json:"error"` + } +) + +// integrationVerify authenticates a request; a non nil error is always a +// signature problem (missing, malformed or not matching). +type integrationVerify func(header http.Header, body []byte) error + +// integrationParse turns a verified request into an Observation, or +// *integrations.IgnoredError, *integrations.InvalidError or +// integrations.ErrStaleTimestamp. +type integrationParse func(header http.Header, body []byte) (integrations.Observation, error) + +func (h *integrationHandler) handleGitLab(w http.ResponseWriter, r *http.Request, _ map[string]string) { + verify := func(header http.Header, body []byte) error { + if len(h.cfg.GitLabSigningKey) > 0 { + return integrations.VerifyGitLabSignature(h.cfg.GitLabSigningKey, header, body, h.now(), h.cfg.Tolerance) + } + return integrations.VerifyGitLabSecretToken(h.cfg.GitLabSecretToken, header.Get(integrations.HeaderGitLabToken)) + } + parse := func(header http.Header, body []byte) (integrations.Observation, error) { + return integrations.ParseGitLab(header.Get(integrations.HeaderGitLabEvent), header.Get(integrations.HeaderGitLabInstance), body, h.cfg) + } + h.serve(w, r, integrations.SourceGitLab, verify, parse) +} + +func (h *integrationHandler) handleFlux(w http.ResponseWriter, r *http.Request, _ map[string]string) { + verify := func(header http.Header, body []byte) error { + return integrations.VerifyFluxSignature(h.cfg.FluxHMACKey, header.Get(integrations.HeaderFluxSignature), body) + } + parse := func(header http.Header, body []byte) (integrations.Observation, error) { + return integrations.ParseFlux(body, h.cfg, h.now()) + } + h.serve(w, r, integrations.SourceFlux, verify, parse) +} + +// serve is the pipeline common to both webhooks: read the body under a size +// limit, verify its signature, parse it into an Observation and process it. +// Exactly one result is counted and logged per request. Never logged: the +// raw body, a parsed message, or a secret/signature header value. +func (h *integrationHandler) serve(w http.ResponseWriter, r *http.Request, source string, verify integrationVerify, parse integrationParse) { + baseAttrs := []any{"source", source} + if source == integrations.SourceGitLab { + baseAttrs = append(baseAttrs, + "idempotencyKey", r.Header.Get("Idempotency-Key"), + "gitlabEventUUID", r.Header.Get("X-Gitlab-Event-UUID"), + "webhookId", r.Header.Get(integrations.HeaderWebhookID), + ) + } + + finish := func(status int, result string, body any, extra ...any) { + integrationWebhooks.WithLabelValues(source, result).Inc() + + attrs := make([]any, 0, len(baseAttrs)+len(extra)+2) + attrs = append(attrs, baseAttrs...) + attrs = append(attrs, "result", result) + attrs = append(attrs, extra...) + + level := slog.LevelInfo + switch result { + case resultUnauthorized, resultInvalid: + level = slog.LevelWarn + case resultError: + level = slog.LevelError + } + h.logger.Log(r.Context(), level, "integration webhook", attrs...) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) + } + + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, integrationMaxBodyBytes)) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + finish(http.StatusRequestEntityTooLarge, resultInvalid, integrationError{Error: "request body too large"}) + return + } + finish(http.StatusBadRequest, resultInvalid, integrationError{Error: "cannot read request body"}) + return + } + + if err := verify(r.Header, body); err != nil { + finish(http.StatusUnauthorized, resultUnauthorized, integrationError{Error: "invalid signature"}, "reason", err.Error()) + return + } + + obs, err := parse(r.Header, body) + if err != nil { + var ignored *integrations.IgnoredError + var invalid *integrations.InvalidError + switch { + case errors.As(err, &ignored): + finish(http.StatusAccepted, resultIgnored, integrationIgnored{Status: "ignored", Reason: ignored.Reason}, "reason", ignored.Reason) + case errors.As(err, &invalid): + finish(http.StatusBadRequest, resultInvalid, integrationError{Error: invalid.Reason}, "reason", invalid.Reason) + case errors.Is(err, integrations.ErrStaleTimestamp): + finish(http.StatusUnauthorized, resultUnauthorized, integrationError{Error: "invalid signature"}, "reason", err.Error()) + default: + finish(http.StatusInternalServerError, resultError, integrationError{Error: "internal error"}, "reason", err.Error()) + } + return + } + + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), integrationProcessTimeout) + defer cancel() + res, err := h.processor.Process(ctx, obs) + if err != nil { + if errors.Is(err, errClaimPending) { + finish(http.StatusInternalServerError, resultError, integrationError{Error: "deployment is being recorded, retry later"}, "key", obs.Key) + return + } + finish(http.StatusInternalServerError, resultError, integrationError{Error: "storage failure"}, "key", obs.Key) + return + } + + switch res.Outcome { + case outcomeDuplicate, outcomeStale: + finish(http.StatusAccepted, resultDuplicate, integrationIgnored{Status: "ignored", Reason: res.Outcome}, "key", obs.Key, "eventId", res.EventID) + case outcomeLockConflict: + finish(http.StatusOK, resultLockConflict, integrationRecorded{Status: "recorded", EventID: res.EventID}, "key", obs.Key, "eventId", res.EventID) + default: + finish(http.StatusOK, resultRecorded, integrationRecorded{Status: "recorded", EventID: res.EventID}, "key", obs.Key, "eventId", res.EventID) + } +} diff --git a/server/integrations_test.go b/server/integrations_test.go new file mode 100644 index 0000000..4ae9130 --- /dev/null +++ b/server/integrations_test.go @@ -0,0 +1,654 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + catalogv1 "github.com/bananaops/tracker/generated/proto/catalog/v1alpha1" + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" + lockv1 "github.com/bananaops/tracker/generated/proto/lock/v1alpha1" + "github.com/bananaops/tracker/internal/integrations" + store "github.com/bananaops/tracker/internal/stores" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" +) + +// fullIntegrationConfig loads a Config with both GitLab (signing token) and +// Flux configured, using the deterministic test secrets. +func fullIntegrationConfig(t *testing.T) integrations.Config { + t.Helper() + sec := newIntegrationSecrets() + return loadIntegrationConfig(t, map[string]string{ + integrations.EnvGitLabSigningToken: sec.signingToken, + integrations.EnvFluxHMACKey: sec.fluxKey, + }) +} + +// gitlabRequestWithSignatureFor builds a GitLab request whose body is body +// but whose signature is computed over signedBody, so the signature never +// matches (used to test a tampered or mismatched signature). +func gitlabRequestWithSignatureFor(t *testing.T, env *integrationEnv, body, signedBody []byte, ts time.Time) *http.Request { + t.Helper() + env.seq++ + id := fmt.Sprintf("msg_%d", env.seq) + tsString := strconv.FormatInt(ts.Unix(), 10) + req, err := http.NewRequest(http.MethodPost, integrationGitLabPath, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(integrations.HeaderGitLabEvent, integrations.GitLabDeploymentHook) + req.Header.Set(integrations.HeaderGitLabInstance, "https://gitlab.example.com") + req.Header.Set(integrations.HeaderWebhookID, id) + req.Header.Set(integrations.HeaderWebhookTimestamp, tsString) + req.Header.Set(integrations.HeaderWebhookSignature, integrations.SignGitLab(env.sec.signingKey, id, tsString, signedBody)) + return req +} + +func TestIntegrationRoutesAbsentWithoutConfig(t *testing.T) { + env := newIntegrationEnv(t, loadIntegrationConfig(t, nil)) + + for _, path := range []string{integrationGitLabPath, integrationFluxPath} { + req, err := http.NewRequest(http.MethodPost, path, bytes.NewReader([]byte("{}"))) + require.NoError(t, err) + rec := env.do(req) + require.Equal(t, http.StatusNotFound, rec.Code, "path %s", path) + } +} + +func TestIntegrationFluxOnly(t *testing.T) { + sec := newIntegrationSecrets() + cfg := loadIntegrationConfig(t, map[string]string{integrations.EnvFluxHMACKey: sec.fluxKey}) + env := newIntegrationEnv(t, cfg) + + req, err := http.NewRequest(http.MethodPost, integrationGitLabPath, bytes.NewReader([]byte("{}"))) + require.NoError(t, err) + rec := env.do(req) + require.Equal(t, http.StatusNotFound, rec.Code) + + body := fluxEventBody(t, "info", "Progressing", env.now, "main@sha1:abcdef1234567890abcdef1234567890abcdef12") + unsigned, err := http.NewRequest(http.MethodPost, integrationFluxPath, bytes.NewReader(body)) + require.NoError(t, err) + unsigned.Header.Set("Content-Type", "application/json") + rec2 := env.do(unsigned) + require.Equal(t, http.StatusUnauthorized, rec2.Code) +} + +func TestGitLabInvalidSignature(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "unauthorized")) + + body := gitlabDeploymentBody(t, 42, "running", time.Now(), "production") + otherBody := gitlabDeploymentBody(t, 43, "running", time.Now(), "production") + req := gitlabRequestWithSignatureFor(t, env, body, otherBody, env.now) + + rec := env.do(req) + require.Equal(t, http.StatusUnauthorized, rec.Code) + require.JSONEq(t, `{"error":"invalid signature"}`, rec.Body.String()) + + after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "unauthorized")) + require.Equal(t, before+1, after) + require.Equal(t, int64(0), env.count(t, "events")) + require.Equal(t, int64(0), env.count(t, "integration_deployments")) +} + +func TestGitLabTimestampOutsideTolerance(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + body := gitlabDeploymentBody(t, 44, "running", time.Now(), "production") + + for _, ts := range []time.Time{ + env.now.Add(-5*time.Minute - time.Second), + env.now.Add(5*time.Minute + time.Second), + } { + req := env.gitlabRequest(t, body, ts) + rec := env.do(req) + require.Equal(t, http.StatusUnauthorized, rec.Code, "ts %s", ts) + } + + require.Equal(t, int64(0), env.count(t, "events")) + require.Equal(t, int64(0), env.count(t, "integration_deployments")) +} + +func TestGitLabRejectedAPIKey(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + + body := gitlabDeploymentBody(t, 45, "running", time.Now(), "production") + req := env.gitlabRequest(t, body, env.now) + req.Header.Set("X-Api-Key", "not-a-key") + + before := totalIntegrationWebhooks(t) + rec := env.do(req) + require.Equal(t, http.StatusUnauthorized, rec.Code) + require.JSONEq(t, `{"error":"invalid or expired credentials"}`, rec.Body.String()) + after := totalIntegrationWebhooks(t) + require.Equal(t, before, after) +} + +func TestGitLabAnonymousWithEmptyPermissions(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + body := gitlabDeploymentBody(t, 46, "running", time.Now(), "review/foo") + req := env.gitlabRequest(t, body, env.now) + + rec := env.do(req) + require.Equal(t, http.StatusAccepted, rec.Code) +} + +func TestGitLabBodyTooLarge(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "invalid")) + + body := bytes.Repeat([]byte("a"), integrationMaxBodyBytes+1) + req := env.gitlabRequest(t, body, env.now) + + rec := env.do(req) + require.Equal(t, http.StatusRequestEntityTooLarge, rec.Code) + + after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "invalid")) + require.Equal(t, before+1, after) +} + +func TestGitLabInvalidJSON(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "invalid")) + + req := env.gitlabRequest(t, []byte(`{`), env.now) + rec := env.do(req) + require.Equal(t, http.StatusBadRequest, rec.Code) + + after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "invalid")) + require.Equal(t, before+1, after) +} + +func TestIntegrationIgnoredCases(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + + assertIgnored := func(t *testing.T, reason string, rec *httptest.ResponseRecorder) { + t.Helper() + require.Equal(t, http.StatusAccepted, rec.Code) + require.JSONEq(t, fmt.Sprintf(`{"status":"ignored","reason":%q}`, reason), rec.Body.String()) + require.Equal(t, int64(0), env.count(t, "events")) + require.Equal(t, int64(0), env.count(t, "integration_deployments")) + } + + t.Run("gitlab push hook", func(t *testing.T) { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored")) + body := gitlabDeploymentBody(t, 100, "running", time.Now(), "production") + req := env.gitlabRequest(t, body, env.now) + req.Header.Set(integrations.HeaderGitLabEvent, "Push Hook") + rec := env.do(req) + assertIgnored(t, "unsupported event", rec) + require.Equal(t, before+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored"))) + }) + + t.Run("gitlab review environment", func(t *testing.T) { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored")) + body := gitlabDeploymentBody(t, 101, "running", time.Now(), "review/foo") + req := env.gitlabRequest(t, body, env.now) + rec := env.do(req) + assertIgnored(t, "unmapped environment", rec) + require.Equal(t, before+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored"))) + }) + + t.Run("gitlab integration environment", func(t *testing.T) { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored")) + body := gitlabDeploymentBody(t, 102, "running", time.Now(), "integration") + req := env.gitlabRequest(t, body, env.now) + rec := env.do(req) + assertIgnored(t, "unmapped environment", rec) + require.Equal(t, before+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored"))) + }) + + t.Run("gitlab approved status", func(t *testing.T) { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored")) + body := gitlabDeploymentBody(t, 103, "approved", time.Now(), "production") + req := env.gitlabRequest(t, body, env.now) + rec := env.do(req) + assertIgnored(t, "approval not tracked", rec) + require.Equal(t, before+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "ignored"))) + }) + + t.Run("flux unsupported kind", func(t *testing.T) { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "ignored")) + body, err := json.Marshal(map[string]any{ + "involvedObject": map[string]any{"kind": "GitRepository", "namespace": "apps", "name": "payments"}, + "severity": "info", + "reason": "Progressing", + "timestamp": env.now.Format(time.RFC3339), + "metadata": map[string]any{"environment": "production"}, + }) + require.NoError(t, err) + req := env.fluxRequest(t, body) + rec := env.do(req) + assertIgnored(t, "unsupported kind", rec) + require.Equal(t, before+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "ignored"))) + }) + + t.Run("flux unsupported reason", func(t *testing.T) { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "ignored")) + body := fluxEventBody(t, "info", "DependencyNotReady", env.now, "main@sha1:abcdef1234567890abcdef1234567890abcdef12") + req := env.fluxRequest(t, body) + rec := env.do(req) + assertIgnored(t, "unsupported reason", rec) + require.Equal(t, before+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "ignored"))) + }) +} + +func TestGitLabDeploymentLifecycle(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + beforeRecorded := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "recorded")) + + t0 := time.Now() + req1 := env.gitlabRequest(t, gitlabDeploymentBody(t, 200, "running", t0, "production"), env.now) + rec1 := env.do(req1) + require.Equal(t, http.StatusOK, rec1.Code) + var resp1 integrationRecorded + require.NoError(t, json.Unmarshal(rec1.Body.Bytes(), &resp1)) + require.Equal(t, "recorded", resp1.Status) + require.NotEmpty(t, resp1.EventID) + + locks := env.locks(t) + require.Len(t, locks, 1) + require.Equal(t, "gitlab:jdoe", locks[0].Who) + require.Equal(t, resp1.EventID, locks[0].EventId) + + t1 := t0.Add(30 * time.Second) + req2 := env.gitlabRequest(t, gitlabDeploymentBody(t, 200, "success", t1, "production"), env.now) + rec2 := env.do(req2) + require.Equal(t, http.StatusOK, rec2.Code) + var resp2 integrationRecorded + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &resp2)) + require.Equal(t, resp1.EventID, resp2.EventID) + + ev := env.onlyEvent(t) + require.Equal(t, eventv1.Type_deployment, ev.Attributes.Type) + require.Equal(t, "gitlab", ev.Attributes.Source) + require.Equal(t, eventv1.Environment_production, ev.Attributes.Environment) + require.Equal(t, eventv1.Status_success, ev.Attributes.Status) + require.Equal(t, "payments", ev.Attributes.Service) + + require.Len(t, env.locks(t), 0) + + duration := ev.Metadata.Duration.AsDuration() + require.GreaterOrEqual(t, duration, 25*time.Second) + require.LessOrEqual(t, duration, 35*time.Second) + + var created, statusChanged *eventv1.ChangelogEntry + for _, e := range ev.Changelog { + switch e.ChangeType { + case eventv1.ChangeType_created: + created = e + case eventv1.ChangeType_status_changed: + statusChanged = e + } + } + require.NotNil(t, created) + require.Equal(t, "gitlab:jdoe", created.User) + require.NotNil(t, statusChanged) + require.Equal(t, "start", statusChanged.OldValue) + require.Equal(t, "success", statusChanged.NewValue) + require.Equal(t, "gitlab:jdoe", statusChanged.User) + + afterRecorded := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "recorded")) + require.Equal(t, beforeRecorded+2, afterRecorded) +} + +func TestGitLabCanceled(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + t0 := time.Now() + + rec1 := env.do(env.gitlabRequest(t, gitlabDeploymentBody(t, 300, "running", t0, "production"), env.now)) + require.Equal(t, http.StatusOK, rec1.Code) + + rec2 := env.do(env.gitlabRequest(t, gitlabDeploymentBody(t, 300, "canceled", t0.Add(5*time.Second), "production"), env.now)) + require.Equal(t, http.StatusOK, rec2.Code) + + ev := env.onlyEvent(t) + require.Equal(t, eventv1.Status_warning, ev.Attributes.Status) + + var commented *eventv1.ChangelogEntry + for _, e := range ev.Changelog { + if e.ChangeType == eventv1.ChangeType_commented && e.Comment == "Deployment canceled" { + commented = e + } + } + require.NotNil(t, commented) + + require.Len(t, env.locks(t), 0) +} + +func TestGitLabLockConflict(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + ctx := context.Background() + + alice, err := store.NewStoreLockFromCollection(env.db.Collection("locks")).Create(ctx, &lockv1.Lock{ + Service: "payments", Environment: "production", Resource: "deployment", Who: "alice", + }) + require.NoError(t, err) + + beforeLockConflict := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "lock_conflict")) + beforeRecorded := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "recorded")) + + t0 := time.Now() + rec1 := env.do(env.gitlabRequest(t, gitlabDeploymentBody(t, 400, "running", t0, "production"), env.now)) + require.Equal(t, http.StatusOK, rec1.Code) + var resp1 integrationRecorded + require.NoError(t, json.Unmarshal(rec1.Body.Bytes(), &resp1)) + require.Equal(t, "recorded", resp1.Status) + + ev := env.onlyEvent(t) + var commented *eventv1.ChangelogEntry + for _, e := range ev.Changelog { + if e.ChangeType == eventv1.ChangeType_commented { + commented = e + } + } + require.NotNil(t, commented) + require.Equal(t, "Deployed while payments was locked in production by alice", commented.Comment) + + require.Equal(t, beforeLockConflict+1, testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "lock_conflict"))) + require.Equal(t, beforeRecorded, testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "recorded"))) + + rec2 := env.do(env.gitlabRequest(t, gitlabDeploymentBody(t, 400, "success", t0.Add(10*time.Second), "production"), env.now)) + require.Equal(t, http.StatusOK, rec2.Code) + + locks := env.locks(t) + require.Len(t, locks, 1) + require.Equal(t, alice.Id, locks[0].Id) + require.Equal(t, "alice", locks[0].Who) + require.Equal(t, "", locks[0].EventId) +} + +func TestFluxProgressingThenSucceededSameSecond(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + beforeRecorded := testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "recorded")) + + revision := "main@sha1:abcdef1234567890abcdef1234567890abcdef12" + rec1 := env.do(env.fluxRequest(t, fluxEventBody(t, "info", "Progressing", env.now, revision))) + require.Equal(t, http.StatusOK, rec1.Code) + + rec2 := env.do(env.fluxRequest(t, fluxEventBody(t, "info", "ReconciliationSucceeded", env.now, revision))) + require.Equal(t, http.StatusOK, rec2.Code) + + ev := env.onlyEvent(t) + require.Equal(t, "flux", ev.Attributes.Source) + require.Equal(t, eventv1.Status_success, ev.Attributes.Status) + + require.Len(t, env.locks(t), 0) + + afterRecorded := testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "recorded")) + require.Equal(t, beforeRecorded+2, afterRecorded) +} + +func TestGitLabOutOfOrder(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + beforeDuplicate := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "duplicate")) + + t0 := time.Now() + rec1 := env.do(env.gitlabRequest(t, gitlabDeploymentBody(t, 500, "success", t0, "production"), env.now)) + require.Equal(t, http.StatusOK, rec1.Code) + + rec2 := env.do(env.gitlabRequest(t, gitlabDeploymentBody(t, 500, "running", t0.Add(-10*time.Second), "production"), env.now)) + require.Equal(t, http.StatusAccepted, rec2.Code) + require.JSONEq(t, `{"status":"ignored","reason":"stale"}`, rec2.Body.String()) + + ev := env.onlyEvent(t) + require.Equal(t, eventv1.Status_success, ev.Attributes.Status) + + require.Len(t, env.locks(t), 0) + + afterDuplicate := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "duplicate")) + require.Equal(t, beforeDuplicate+1, afterDuplicate) +} + +func TestGitLabReplayTenTimes(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + + req := env.gitlabRequest(t, gitlabDeploymentBody(t, 600, "running", time.Now(), "production"), env.now) + + rec := env.do(req) + require.Equal(t, http.StatusOK, rec.Code) + + firstLen := len(env.onlyEvent(t).Changelog) + + for i := 0; i < 9; i++ { + rec = env.do(req) + require.Equal(t, http.StatusAccepted, rec.Code) + require.JSONEq(t, `{"status":"ignored","reason":"duplicate"}`, rec.Body.String()) + } + + ev := env.onlyEvent(t) + require.Len(t, ev.Changelog, firstLen) + require.Len(t, env.locks(t), 1) +} + +func TestFluxInvalidSignatureAndStaleTimestamp(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + + body := fluxEventBody(t, "info", "Progressing", env.now, "main@sha1:abcdef1234567890abcdef1234567890abcdef12") + badKeyReq, err := http.NewRequest(http.MethodPost, integrationFluxPath, bytes.NewReader(body)) + require.NoError(t, err) + badKeyReq.Header.Set("Content-Type", "application/json") + sig, err := integrations.SignFlux("sha256", []byte("wrong-flux-key-0123456789abcdef01234567"), body) + require.NoError(t, err) + badKeyReq.Header.Set(integrations.HeaderFluxSignature, sig) + rec := env.do(badKeyReq) + require.Equal(t, http.StatusUnauthorized, rec.Code) + + staleBody := fluxEventBody(t, "info", "Progressing", env.now.Add(-5*time.Minute-time.Second), "main@sha1:abcdef1234567890abcdef1234567890abcdef12") + rec2 := env.do(env.fluxRequest(t, staleBody)) + require.Equal(t, http.StatusUnauthorized, rec2.Code) + + require.Equal(t, int64(0), env.count(t, "events")) + require.Equal(t, int64(0), env.count(t, "integration_deployments")) +} + +func TestGitLabSecretTokenMode(t *testing.T) { + sec := newIntegrationSecrets() + cfg := loadIntegrationConfig(t, map[string]string{integrations.EnvGitLabSecretToken: sec.secretToken}) + env := newIntegrationEnv(t, cfg) + + body := gitlabDeploymentBody(t, 700, "running", time.Now(), "review/foo") + + rec := env.do(gitlabTokenRequest(t, body, sec.secretToken)) + require.Equal(t, http.StatusAccepted, rec.Code) + + recBad := env.do(gitlabTokenRequest(t, body, "wrong-token-0123456789")) + require.Equal(t, http.StatusUnauthorized, recBad.Code) + + recNone := env.do(gitlabTokenRequest(t, body, "")) + require.Equal(t, http.StatusUnauthorized, recNone.Code) +} + +// gitlabTokenRequest builds a GitLab request authenticated with +// X-Gitlab-Token instead of a Standard Webhooks signature. An empty token +// omits the header entirely. +func gitlabTokenRequest(t *testing.T, body []byte, token string) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodPost, integrationGitLabPath, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(integrations.HeaderGitLabEvent, integrations.GitLabDeploymentHook) + req.Header.Set(integrations.HeaderGitLabInstance, "https://gitlab.example.com") + if token != "" { + req.Header.Set(integrations.HeaderGitLabToken, token) + } + return req +} + +// TestGitLabSigningTokenNeverFallsBackToSecretToken enforces controller +// ruling 1: when a signing token is configured, a valid webhook-signature is +// required and X-Gitlab-Token is never accepted as a fallback, even when a +// secret token is also configured. +func TestGitLabSigningTokenNeverFallsBackToSecretToken(t *testing.T) { + sec := newIntegrationSecrets() + cfg := loadIntegrationConfig(t, map[string]string{ + integrations.EnvGitLabSigningToken: sec.signingToken, + integrations.EnvGitLabSecretToken: sec.secretToken, + }) + env := newIntegrationEnv(t, cfg) + + body := gitlabDeploymentBody(t, 800, "running", time.Now(), "production") + req := gitlabTokenRequest(t, body, sec.secretToken) + + rec := env.do(req) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +// TestGitLabClaimPendingReturns500 enforces controller ruling 2: an abandoned +// claim (event id never recorded, past the staleness window) makes Process +// return errClaimPending, which the handler must map to 500 so the sender +// retries, not to a 4xx. +func TestGitLabClaimPendingReturns500(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + ctx := context.Background() + + key := "gitlab:gitlab.example.com:900" + depStore := store.NewIntegrationDeploymentStoreFromCollection(env.db.Collection(store.IntegrationDeploymentsCollection)) + created, _, err := depStore.Claim(ctx, key, "gitlab", "start", env.now, 1) + require.NoError(t, err) + require.True(t, created) + + _, err = env.db.Collection(store.IntegrationDeploymentsCollection).UpdateOne(ctx, + bson.M{"_id": key}, + bson.M{"$set": bson.M{"createdAt": env.now.Add(-time.Hour)}}, + ) + require.NoError(t, err) + + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "error")) + + body := gitlabDeploymentBody(t, 900, "running", time.Now(), "production") + rec := env.do(env.gitlabRequest(t, body, env.now)) + require.Equal(t, http.StatusInternalServerError, rec.Code) + require.JSONEq(t, `{"error":"deployment is being recorded, retry later"}`, rec.Body.String()) + + after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "error")) + require.Equal(t, before+1, after) +} + +func TestGitLabCatalogResolution(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + ctx := context.Background() + + _, err := store.NewStoreCatalogFromCollection(env.db.Collection("catalog")).Update(ctx, + map[string]interface{}{"name": "payments-api"}, + &catalogv1.Catalog{Name: "payments-api", Repository: "git@gitlab.example.com:team/payments.git"}, + ) + require.NoError(t, err) + + body := gitlabDeploymentBody(t, 1000, "running", time.Now(), "production") + rec := env.do(env.gitlabRequest(t, body, env.now)) + require.Equal(t, http.StatusOK, rec.Code) + + ev := env.onlyEvent(t) + require.Equal(t, "payments-api", ev.Attributes.Service) + require.Equal(t, "Deploy payments-api a1b2c3d4 to production", ev.Title) +} + +func TestIntegrationLogsContainNoSecret(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + var gitlabSignatures []string + var fluxSignatures []string + + // Case 3: invalid signature, with an (unused) X-Gitlab-Token also set. + body := gitlabDeploymentBody(t, 1100, "running", time.Now(), "production") + other := gitlabDeploymentBody(t, 1101, "running", time.Now(), "production") + badSigReq := gitlabRequestWithSignatureFor(t, env, body, other, env.now) + badSigReq.Header.Set(integrations.HeaderGitLabToken, env.sec.secretToken) + env.do(badSigReq) + gitlabSignatures = append(gitlabSignatures, badSigReq.Header.Get(integrations.HeaderWebhookSignature)) + + // Case 10: lifecycle (recorded). + t0 := time.Now() + req1 := env.gitlabRequest(t, gitlabDeploymentBody(t, 1102, "running", t0, "production"), env.now) + env.do(req1) + gitlabSignatures = append(gitlabSignatures, req1.Header.Get(integrations.HeaderWebhookSignature)) + req2 := env.gitlabRequest(t, gitlabDeploymentBody(t, 1102, "success", t0.Add(30*time.Second), "production"), env.now) + env.do(req2) + gitlabSignatures = append(gitlabSignatures, req2.Header.Get(integrations.HeaderWebhookSignature)) + + // Case 13: Flux progressing then succeeded, same instant. + revision := "main@sha1:abcdef1234567890abcdef1234567890abcdef12" + fluxReq1 := env.fluxRequest(t, fluxEventBody(t, "info", "Progressing", env.now, revision)) + env.do(fluxReq1) + fluxSignatures = append(fluxSignatures, fluxReq1.Header.Get(integrations.HeaderFluxSignature)) + fluxReq2 := env.fluxRequest(t, fluxEventBody(t, "info", "ReconciliationSucceeded", env.now, revision)) + env.do(fluxReq2) + fluxSignatures = append(fluxSignatures, fluxReq2.Header.Get(integrations.HeaderFluxSignature)) + + // The secret-token request itself: a signing token and a secret token + // cannot be configured together, so this runs against its own env, and + // its log buffer is checked alongside the one above. + secretEnv := newIntegrationEnv(t, loadIntegrationConfig(t, map[string]string{ + integrations.EnvGitLabSecretToken: env.sec.secretToken, + })) + secretEnv.do(gitlabTokenRequest(t, gitlabDeploymentBody(t, 1103, "running", time.Now(), "review/foo"), secretEnv.sec.secretToken)) + + logs := env.logs.String() + secretEnv.logs.String() + require.NotContains(t, logs, env.sec.signingToken) + require.NotContains(t, logs, env.sec.signingKeyB64) + require.NotContains(t, logs, env.sec.fluxKey) + require.NotContains(t, logs, env.sec.secretToken) + for _, sig := range gitlabSignatures { + require.NotEmpty(t, sig) + require.NotContains(t, logs, sig) + } + for _, sig := range fluxSignatures { + _, hexPart, ok := strings.Cut(sig, "=") + require.True(t, ok) + require.NotEmpty(t, hexPart) + require.NotContains(t, logs, hexPart) + } + + require.Contains(t, logs, `"source":"gitlab"`) + require.Contains(t, logs, `"result":"recorded"`) +} + +func TestIntegrationMetricRegistered(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + body := gitlabDeploymentBody(t, 1200, "running", time.Now(), "production") + env.do(env.gitlabRequest(t, body, env.now)) + + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + var found bool + for _, mf := range mfs { + if mf.GetName() != "tracker_integration_webhooks_total" { + continue + } + found = true + for _, m := range mf.GetMetric() { + var names []string + for _, lp := range m.GetLabel() { + names = append(names, lp.GetName()) + } + require.ElementsMatch(t, []string{"result", "source"}, names) + } + } + require.True(t, found) +} + +// totalIntegrationWebhooks sums tracker_integration_webhooks_total across +// every source/result series. +func totalIntegrationWebhooks(t *testing.T) float64 { + t.Helper() + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + var total float64 + for _, mf := range mfs { + if mf.GetName() != "tracker_integration_webhooks_total" { + continue + } + for _, m := range mf.GetMetric() { + total += m.GetCounter().GetValue() + } + } + return total +} diff --git a/server/integrations_testing_test.go b/server/integrations_testing_test.go new file mode 100644 index 0000000..e2acca7 --- /dev/null +++ b/server/integrations_testing_test.go @@ -0,0 +1,252 @@ +package server + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "testing" + "time" + + eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" + lockv1 "github.com/bananaops/tracker/generated/proto/lock/v1alpha1" + "github.com/bananaops/tracker/internal/auth" + "github.com/bananaops/tracker/internal/auth/identity" + "github.com/bananaops/tracker/internal/integrations" + store "github.com/bananaops/tracker/internal/stores" + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +// syncBuffer is a bytes.Buffer safe for concurrent writes, used to capture +// the JSON logger output of an integrationEnv. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// integrationSecrets are the fixed test credentials for both webhook sources. +type integrationSecrets struct { + signingToken string + signingKeyB64 string + signingKey []byte + secretToken string + fluxKey string +} + +func newIntegrationSecrets() integrationSecrets { + signingKey := bytes.Repeat([]byte{0x11}, 32) + signingKeyB64 := base64.StdEncoding.EncodeToString(signingKey) + return integrationSecrets{ + signingKey: signingKey, + signingKeyB64: signingKeyB64, + signingToken: "whsec_" + signingKeyB64, + secretToken: "gitlab-secret-token-0123", + fluxKey: "flux-hmac-key-0123456789abcdef0123456789", + } +} + +// loadIntegrationConfig loads an integrations.Config from a fixed set of +// environment variables, ignoring the real process environment. +func loadIntegrationConfig(t *testing.T, vars map[string]string) integrations.Config { + t.Helper() + cfg, err := integrations.LoadConfig(func(k string) (string, bool) { + v, ok := vars[k] + return v, ok + }) + require.NoError(t, err) + return cfg +} + +// integrationEnv wires a real Mongo database, the integration handlers and +// the auth middleware together, so tests exercise the exact request path a +// webhook goes through in production. +type integrationEnv struct { + db *mongo.Database + handler http.Handler + logs *syncBuffer + now time.Time + cfg integrations.Config + sec integrationSecrets + seq int +} + +func newIntegrationEnv(t *testing.T, cfg integrations.Config) *integrationEnv { + t.Helper() + db := testMongoDatabase(t) + + logs := &syncBuffer{} + logger := slog.New(slog.NewJSONHandler(logs, nil)) + prev := slog.Default() + slog.SetDefault(logger) + t.Cleanup(func() { slog.SetDefault(prev) }) + + events := newEventFromStores( + store.NewStoreEventFromCollection(db.Collection("events")), + store.NewStoreLockFromCollection(db.Collection("locks")), + logger, + ) + + now := time.Now().UTC().Truncate(time.Second) + + mux := runtime.NewServeMux() + err := RegisterIntegrationHandlers(mux, cfg, IntegrationDeps{ + Events: events, + Deployments: store.NewIntegrationDeploymentStoreFromCollection(db.Collection(store.IntegrationDeploymentsCollection)), + Catalog: store.NewStoreCatalogFromCollection(db.Collection("catalog")), + Logger: logger, + Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + // The resolver carries no store: there is no user or api key to look up, + // so any credential (including a malformed X-Api-Key) is rejected before + // it ever reaches a store, and an absent one resolves to anonymous. + handler := auth.HTTPMiddleware( + &identity.Resolver{AnonymousPermissions: []auth.Permission{}}, + auth.Config{AnonymousPermissions: []auth.Permission{}}, + )(mux) + + return &integrationEnv{ + db: db, + handler: handler, + logs: logs, + now: now, + cfg: cfg, + sec: newIntegrationSecrets(), + } +} + +// gitlabDeploymentBody builds a GitLab "Deployment Hook" payload for project +// team/payments (web_url https://gitlab.example.com/team/payments), deployed +// by jdoe at short_sha a1b2c3d4. +func gitlabDeploymentBody(t *testing.T, id int64, status string, at time.Time, environment string) []byte { + t.Helper() + body, err := json.Marshal(map[string]any{ + "object_kind": "deployment", + "status": status, + "status_changed_at": at.Format(time.RFC3339), + "deployment_id": id, + "environment": environment, + "short_sha": "a1b2c3d4", + "commit_title": "Fix rounding", + "user": map[string]any{"username": "jdoe"}, + "project": map[string]any{ + "web_url": "https://gitlab.example.com/team/payments", + "path_with_namespace": "team/payments", + }, + }) + require.NoError(t, err) + return body +} + +// fluxEventBody builds a Flux notification-controller event for the +// Kustomization apps/payments, in the production environment. +func fluxEventBody(t *testing.T, severity, reason string, at time.Time, revision string) []byte { + t.Helper() + body, err := json.Marshal(map[string]any{ + "involvedObject": map[string]any{"kind": "Kustomization", "namespace": "apps", "name": "payments"}, + "severity": severity, + "reason": reason, + "timestamp": at.Format(time.RFC3339), + "message": "Reconciliation finished", + "metadata": map[string]any{ + "kustomize.toolkit.fluxcd.io/revision": revision, + "environment": "production", + }, + }) + require.NoError(t, err) + return body +} + +// gitlabRequest builds a signed GitLab webhook request. Each call uses a +// fresh webhook-id, unless the caller reuses the returned *http.Request +// itself (do resets its body for a replay). +func (e *integrationEnv) gitlabRequest(t *testing.T, body []byte, ts time.Time) *http.Request { + t.Helper() + e.seq++ + id := fmt.Sprintf("msg_%d", e.seq) + tsString := strconv.FormatInt(ts.Unix(), 10) + + req, err := http.NewRequest(http.MethodPost, integrationGitLabPath, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(integrations.HeaderGitLabEvent, integrations.GitLabDeploymentHook) + req.Header.Set(integrations.HeaderGitLabInstance, "https://gitlab.example.com") + req.Header.Set(integrations.HeaderWebhookID, id) + req.Header.Set(integrations.HeaderWebhookTimestamp, tsString) + req.Header.Set(integrations.HeaderWebhookSignature, integrations.SignGitLab(e.sec.signingKey, id, tsString, body)) + return req +} + +// fluxRequest builds a signed Flux webhook request. +func (e *integrationEnv) fluxRequest(t *testing.T, body []byte) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodPost, integrationFluxPath, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + sig, err := integrations.SignFlux("sha256", []byte(e.sec.fluxKey), body) + require.NoError(t, err) + req.Header.Set(integrations.HeaderFluxSignature, sig) + return req +} + +// do serves req through the full handler chain (auth middleware, then mux). +// req's body is reset from GetBody first, so the same *http.Request can be +// replayed several times. +func (e *integrationEnv) do(req *http.Request) *httptest.ResponseRecorder { + if req.GetBody != nil { + if b, err := req.GetBody(); err == nil { + req.Body = b + } + } + rec := httptest.NewRecorder() + e.handler.ServeHTTP(rec, req) + return rec +} + +// count returns the number of documents in collection. +func (e *integrationEnv) count(t *testing.T, collection string) int64 { + t.Helper() + n, err := e.db.Collection(collection).CountDocuments(context.Background(), bson.M{}) + require.NoError(t, err) + return n +} + +// onlyEvent requires exactly one document in the events collection and +// returns it decoded. +func (e *integrationEnv) onlyEvent(t *testing.T) *eventv1.Event { + t.Helper() + events, err := store.NewStoreEventFromCollection(e.db.Collection("events")).List(context.Background()) + require.NoError(t, err) + require.Len(t, events, 1) + return events[0] +} + +// locks returns every lock currently held. +func (e *integrationEnv) locks(t *testing.T) []*lockv1.Lock { + t.Helper() + locks, err := store.NewStoreLockFromCollection(e.db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + return locks +} From 6709463652cebda55787c18f2fa005337b5f6703 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:18:39 +0200 Subject: [PATCH 16/22] docs: document GitLab and Flux deployment integrations New INTEGRATIONS.md (GitLab webhook, Flux Provider and Alert, locks, troubleshooting), INTEGRATION_* variables in CONFIGURATION.md, gitlab and flux sources in EVENTS.md, link from the README. Refs #208 --- README.md | 1 + docs/CONFIGURATION.md | 31 ++++++ docs/EVENTS.md | 9 +- docs/INTEGRATIONS.md | 239 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 docs/INTEGRATIONS.md diff --git a/README.md b/README.md index ec59dab..58d2215 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,7 @@ npm run dev - [📊 Events Guide](./docs/EVENTS.md) - Working with events - [📦 Catalog Guide](./docs/CATALOG.md) - Managing service catalog - [🔒 Locks Guide](./docs/LOCKS.md) - Distributed locking +- [🔗 Deployment Integrations](./docs/INTEGRATIONS.md) - Record GitLab and Flux deployments automatically ### API Documentation - [🔌 API Specification](./docs/api-specification.md) - API reference diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f812844..67ad735 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -99,6 +99,37 @@ AUTH_OIDC_CLIENT_SECRET= # Redirect URI to register: https://tracker.example.com/api/v1alpha1/auth/oidc/callback ``` +### Deployment integrations + +| Variable | Default | Description | +|----------|---------|-------------| +| `INTEGRATION_GITLAB_SIGNING_TOKEN` | - | GitLab signing token, `whsec_`. When set, the `webhook-signature` header is required and `INTEGRATION_GITLAB_SECRET_TOKEN` is ignored. | +| `INTEGRATION_GITLAB_SECRET_TOKEN` | - | Legacy GitLab secret token, compared against `X-Gitlab-Token`. Ignored when the signing token is set. | +| `INTEGRATION_FLUX_HMAC_KEY` | - | HMAC key shared with the Flux `Provider` Secret (key `token`). Enables the Flux webhook endpoint. | +| `INTEGRATION_WEBHOOK_TOLERANCE` | `5m` | Freshness window (Go duration) for GitLab's `webhook-timestamp` and Flux's event `timestamp`. | +| `INTEGRATION_ENVIRONMENTS` | `production=production,staging=preproduction` | Comma separated `source=tracker` pairs mapping a GitLab or Flux environment name to a Tracker environment. Source keys are matched case-insensitively. | + +An invalid value stops Tracker at startup, with an error that never includes the secret itself: + +- `INTEGRATION_GITLAB_SIGNING_TOKEN` not prefixed with `whsec_`, or the part after the prefix is + not valid base64, or decodes to an empty value. +- `INTEGRATION_GITLAB_SECRET_TOKEN` shorter than 16 characters. +- `INTEGRATION_FLUX_HMAC_KEY` shorter than 32 bytes. +- `INTEGRATION_WEBHOOK_TOLERANCE` not a positive Go duration (for example `5m`). +- `INTEGRATION_ENVIRONMENTS` malformed (an entry without `=`, an empty key, or a duplicate key), + or a value that is not one of the Tracker environment names: `development`, `integration`, + `TNR`, `UAT`, `recette`, `preproduction`, `production`, `mco` (these are case-sensitive). + +See [INTEGRATIONS.md](INTEGRATIONS.md) for the GitLab and Flux setup this configures. + +**Example:** +```bash +INTEGRATION_GITLAB_SIGNING_TOKEN=whsec_ +INTEGRATION_FLUX_HMAC_KEY=<32-byte-value-from-secret-manager> +INTEGRATION_WEBHOOK_TOLERANCE=5m +INTEGRATION_ENVIRONMENTS=production=production,staging=preproduction +``` + ### Slack Integration | Variable | Default | Description | diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 85dd17c..8af265e 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -60,7 +60,7 @@ Tracker supports five main event types: #### Optional Fields - **attributes.message** (string): Detailed description -- **attributes.source** (string): Event origin (e.g., `github_actions`, `jenkins`, `manual`) +- **attributes.source** (string): Event origin (e.g., `github_actions`, `jenkins`, `manual`, `gitlab`, `flux`) - **attributes.priority** (int): Priority level (1=P1/Critical, 5=P5/Low) - **attributes.status** (int): Current status (see Status Values below) - **attributes.environment** (int): Target environment (see Environment Values below) @@ -387,6 +387,13 @@ curl -X POST http://localhost:8080/api/v1alpha1/event \ }' ``` +### Automatic deployment events + +Tracker can create and update deployment events on its own from GitLab and Flux webhooks, +without any call to the REST or gRPC API. Each deployment produces a single event, taken through +its lifecycle as notifications arrive: `start`, then `success`, `failure`, `warning` or `close`. +See [INTEGRATIONS.md](INTEGRATIONS.md) for setup, environment mapping and troubleshooting. + ## Best Practices ### 1. Use Descriptive Titles diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md new file mode 100644 index 0000000..40d4986 --- /dev/null +++ b/docs/INTEGRATIONS.md @@ -0,0 +1,239 @@ +# Deployment integrations (GitLab and Flux) + +Tracker can record deployments automatically from GitLab deployment webhooks and Flux +notification-controller alerts, without any change to your CI/CD pipelines or GitOps manifests +in most cases. + +For each deployment, Tracker records one `deployment` event: created on the first notification +seen (usually a start), then updated in place as later notifications arrive for the same +deployment. `attributes.source` is `gitlab` or `flux`, `attributes.type` is `deployment`, +`attributes.priority` is always `P3`. Only environments listed in `INTEGRATION_ENVIRONMENTS` are +recorded; anything else is silently ignored (see [Troubleshooting](#troubleshooting)). By default +only `production` and `staging` are mapped, to `production` and `preproduction` respectively. + +Both webhook sources are optional and independent: enable GitLab, Flux, both, or neither. An +endpoint only exists once its source is configured; otherwise it answers `404`. + +## Tracker configuration + +See [CONFIGURATION.md#deployment-integrations](CONFIGURATION.md#deployment-integrations) for the +full `INTEGRATION_*` variable reference. + +Provide every secret (`INTEGRATION_GITLAB_SIGNING_TOKEN`, `INTEGRATION_GITLAB_SECRET_TOKEN`, +`INTEGRATION_FLUX_HMAC_KEY`) through a Kubernetes Secret (`env.valueFrom.secretKeyRef`) or your +organization's secrets manager, never inline in a ConfigMap or manifest. To rotate a secret, +change the environment variable and restart Tracker: only one value is accepted at a time, there +is no overlap window where both an old and a new value are valid. + +## GitLab + +Configure the webhook once, either at the group level (requires GitLab Premium or Ultimate) or, +failing that, per project (via the GitLab API, or Terraform's `gitlab_project_hook` resource). + +- **URL**: `https:///api/v1alpha1/integrations/gitlab/webhook` +- **Trigger**: "Deployment events" only. Leave every other trigger unchecked. +- **SSL verification**: enabled. +- **Custom headers**: none. +- **Secret**: prefer the signing token (see below). Copy the `whsec_...` value shown once into + `INTEGRATION_GITLAB_SIGNING_TOKEN`. + +GitLab offers two authentication mechanisms for webhooks; Tracker supports both, but never both +at once for the same request: + +- **Signing token** (Standard Webhooks, GitLab 19.0 or later, confirm against your instance): + recommended. It signs each delivery (`webhook-id`, `webhook-timestamp`, `webhook-signature`) + and lets Tracker reject stale or replayed deliveries. Set + `INTEGRATION_GITLAB_SIGNING_TOKEN=whsec_`. +- **Secret token** (legacy): for a GitLab instance that does not offer the signing token yet. Set + `INTEGRATION_GITLAB_SECRET_TOKEN` and GitLab sends it back in `X-Gitlab-Token`, compared in + constant time. There is no replay protection with this mechanism, which is why the signing + token is preferred. + +When `INTEGRATION_GITLAB_SIGNING_TOKEN` is set, the plain `X-Gitlab-Token` header is never +accepted, even if `INTEGRATION_GITLAB_SECRET_TOKEN` is also set (it is then ignored, with a +startup warning). + +Test the webhook with GitLab's "Test" button, "Deployment events" entry: expect `200` or `202`. + +### Environment mapping + +The GitLab deployment `environment` field is matched by its first path segment (before the first +`/`), then by `environment_tier` if the first segment does not match. The Ansible, Terraform and +Docker Compose to-be-continuous templates already declare +`environment: name: ${ENV_TYPE}${_ENVIRONMENT_NAMESPACE}`, so `production` and `staging` +match on the first segment with no pipeline change; a namespaced environment such as +`production-eu` falls back to `environment_tier`. `review/*` and `integration` environments are +not mapped by default and are recorded as `unmapped environment` (202, ignored). + +No change is needed in your `.gitlab-ci.yml`: the deployment jobs already declare `environment:`. + +### Status mapping + +| GitLab status | Tracker status | Terminal | Note | +|---|---|---|---| +| `running` | `start` | no | takes the deployment lock, see [Locks](#locks) | +| `success` | `success` | yes | | +| `failed` | `failure` | yes | | +| `canceled` | `warning` | yes | changelog comment `Deployment canceled` | +| `blocked` | `waiting_approval` | no | | +| `rejected` | `close` | yes | | +| `approved` | (none) | | `202`, reason `approval not tracked` | +| anything else | (none) | | `202`, reason `unsupported status` | + +Title, message and changelog: the event title is `Deploy to `, +the message lists the commit title, short SHA, commit URL, deployment job URL and GitLab +environment name. The changelog entry and lock owner are attributed to `gitlab:`. + +## Flux + +Flux notifications are needed because the `flux` to-be-continuous component, unlike the Ansible, +Terraform and Docker Compose components, does not declare an `environment:` block in the +pipeline: the actual deployment outcome only exists in the cluster, reported by Flux's +notification-controller. + +One Secret, Provider and Alert per cluster (`INTEGRATION_FLUX_HMAC_KEY` must match the Secret's +`token`, 32 bytes minimum). Manage the Secret with your usual secrets pipeline (SOPS, External +Secrets); never commit it in clear text. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: tracker-webhook + namespace: flux-system +type: Opaque +stringData: + token: "" +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: tracker + namespace: flux-system +spec: + type: generic-hmac + address: https://tracker.example.com/api/v1alpha1/integrations/flux/webhook + secretRef: + name: tracker-webhook +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: tracker-deployments + namespace: flux-system +spec: + providerRef: + name: tracker + eventSeverity: info + eventSources: + - kind: Kustomization + name: "*" + - kind: HelmRelease + name: "*" + eventMetadata: + environment: production +``` + +Duplicate this set per cluster, changing `eventMetadata.environment` (for example `staging` on an +iso-prod cluster). `sha256` is the recommended and default HMAC algorithm for the `generic-hmac` +provider; `sha224`, `sha384` and `sha512` are also accepted. + +To name the service explicitly, rather than falling back to the object name, annotate the +Kustomization or HelmRelease with `event.toolkit.fluxcd.io/service: `. + +An `eventSources` entry without `namespace` only matches objects in the Alert's own namespace +(here `flux-system`). If your Kustomizations or HelmReleases live in other namespaces, add one +entry per namespace, or check whether your Flux version supports `namespace: "*"`. + +### Status mapping + +Only `Kustomization` and `HelmRelease` objects are processed; anything else is ignored (`202`, +reason `unsupported kind`). + +| Severity | Reason | Tracker status | +|---|---|---| +| `info` | `Progressing` | `start` | +| `info` | `ReconciliationSucceeded`, `InstallSucceeded`, `UpgradeSucceeded` | `success` | +| `error` | `ReconciliationFailed`, `HealthCheckFailed`, `BuildFailed`, `ValidationFailed`, `ArtifactFailed`, `InstallFailed`, `UpgradeFailed`, `TestFailed` | `failure` | +| `info` (helm-controller emits it with normal severity) | `RollbackSucceeded` | `failure` | +| any | anything else (`DependencyNotReady`, `UninstallSucceeded`, ...) | `202`, reason `unsupported reason` | + +A rollback (`RollbackSucceeded`) counts as a failed deployment: it only happens because the +preceding upgrade failed. + +The changelog entry and lock owner are attributed to `flux:/`. + +## Locks + +A GitLab `running` deployment or a Flux `Progressing` event takes the `deployment` lock for the +service and environment, exactly like a manual deployment through the API. + +If the lock is already held by someone else, the deployment is still recorded, without taking the +lock, with a changelog comment `Deployed while was locked in by `. The +webhook still answers `200`: a lock conflict is never reported as an error to GitLab or Flux. + +A terminal status (`success`, `failure`, `warning`, `close`) only releases the lock attached to +its own event; it never releases a lock held by an unrelated deployment or a manual operation. + +## Service name + +The deployment's service is resolved against the catalog: + +1. GitLab: a catalog entry whose `repository` (normalized) equals the project's web or Git HTTP + URL, normalized the same way. Flux: metadata key `service` (or, defensively, + `event.toolkit.fluxcd.io/service`), if the annotation described above is set. +2. A catalog entry whose `name` equals the fallback name (GitLab: the last segment of + `path_with_namespace`; Flux: `involvedObject.name`). +3. That fallback name, used as is, even if it matches no catalog entry. + +The catalog snapshot used for this matching is cached for 60 seconds. + +## Troubleshooting + +### HTTP response codes + +| Code | Meaning | +|---|---| +| `200` | Recorded: the deployment event was created or updated (including with a lock conflict). | +| `202` | Ignored: unmapped environment, unsupported event/object_kind/kind/status/reason, missing revision, or a duplicate/stale notification. See the reasons below. | +| `400` | Invalid payload: not JSON, or a required field is missing or malformed. | +| `401` | Authentication failure: missing or invalid signature, timestamp outside the tolerance window, or a rejected API key. | +| `404` | This source is not configured: the endpoint does not exist. | +| `413` | Request body larger than 1 MiB. | +| `500` | Storage failure. The sender should retry; retries are safe (see the reasons below). | + +### `202` reasons + +| Reason | Cause | Action | +|---|---|---| +| `unsupported event` | GitLab `X-Gitlab-Event` is not `Deployment Hook`. | Check the webhook only triggers on "Deployment events". | +| `unsupported object_kind` | GitLab payload `object_kind` is not `deployment`. | No action; GitLab occasionally sends other kinds on the same endpoint. | +| `unsupported kind` | Flux `involvedObject.kind` is neither `Kustomization` nor `HelmRelease`. | No action. | +| `unsupported status` | GitLab deployment `status` is not one Tracker maps (see the status table). | No action, unless you expect that status to be tracked; ask for it to be added. | +| `approval not tracked` | GitLab deployment `status` is `approved`. | No action; approvals are not recorded as events. | +| `unsupported reason` | Flux `reason` is not one Tracker maps (see the status table). | No action, unless you expect that reason to be tracked. | +| `unmapped environment` | The GitLab or Flux environment name has no entry in `INTEGRATION_ENVIRONMENTS`. | Add the environment to `INTEGRATION_ENVIRONMENTS` if it should be tracked. | +| `missing revision` | The Flux event has no `kustomize.toolkit.fluxcd.io/revision` or `helm.toolkit.fluxcd.io/revision` metadata. | Check the Kustomization or HelmRelease reports a revision; some early `Progressing` events do not. | +| `duplicate` | Same status, same timestamp as the last one applied. | No action; the sender likely retried a delivery. | +| `stale` | An older or already superseded notification for this deployment. | No action; out-of-order or replayed delivery, correctly discarded. | + +### Metric + +`tracker_integration_webhooks_total{source, result}` counts every webhook received exactly once. +`source` is `gitlab` or `flux`. `result` is one of `recorded`, `ignored`, `duplicate`, +`unauthorized`, `invalid`, `error` or `lock_conflict` (`lock_conflict` replaces `recorded` for +that request; `duplicate` also covers `stale`; `invalid` covers both `400` and `413`). A rejected +API key presented on these routes is counted only in `tracker_auth_requests_total`, not here. + +## Security + +The signature is verified on the raw request body, before any JSON decoding: a malformed or +oversized body never reaches the parser. `INTEGRATION_WEBHOOK_TOLERANCE` (default `5m`) bounds +how far a webhook's timestamp may drift from the server clock, in both directions, which limits +how long a captured delivery can be replayed. + +A verified webhook grants no Tracker permission to the sender: it is authenticated as GitLab or +Flux, not as a Tracker principal, and can only ever result in a deployment event being recorded. +An API key that is malformed, unknown, revoked or expired and presented on these routes is still +rejected with `401`, same as on any other route (see +[AUTHENTICATION.md](AUTHENTICATION.md#api-keys)). From 83c1de8937f6d5576a051d99138712aac36a0980 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:36:59 +0200 Subject: [PATCH 17/22] fix(integrations): derive the GitLab correlation host from the signed payload X-Gitlab-Instance is not part of the Standard Webhooks signed content (webhook-id.webhook-timestamp.body). Keying the correlation on it let a captured, still-valid signed delivery be replayed within the tolerance window with a different instance header to mint a new key, defeating the replay window's idempotency guarantee. ParseGitLab now derives the key host from project.web_url only, the field the signature actually covers, and no longer takes an instance header parameter. A web_url with no parsable host is rejected the same way as any other missing required field. Refs #208 --- internal/integrations/gitlab.go | 44 +++++++++++++++------------- internal/integrations/gitlab_test.go | 41 ++++++++++++++------------ server/integrations.go | 2 +- server/integrations_test.go | 20 +++++++++++++ 4 files changed, 67 insertions(+), 40 deletions(-) diff --git a/internal/integrations/gitlab.go b/internal/integrations/gitlab.go index 06be6be..04a4069 100644 --- a/internal/integrations/gitlab.go +++ b/internal/integrations/gitlab.go @@ -64,10 +64,10 @@ var gitlabStatuses = map[string]gitlabStatus{ var gitlabTimeLayouts = []string{time.RFC3339, "2006-01-02 15:04:05 -0700", "2006-01-02 15:04:05 MST"} // ParseGitLab maps a GitLab "Deployment Hook" payload to an Observation. -// eventHeader and instanceHeader are the X-Gitlab-Event and X-Gitlab-Instance -// header values. It returns *IgnoredError for a notification Tracker does -// not record, and *InvalidError for a malformed or incomplete payload. -func ParseGitLab(eventHeader, instanceHeader string, body []byte, cfg Config) (Observation, error) { +// eventHeader is the X-Gitlab-Event header value. It returns *IgnoredError +// for a notification Tracker does not record, and *InvalidError for a +// malformed or incomplete payload. +func ParseGitLab(eventHeader string, body []byte, cfg Config) (Observation, error) { if strings.TrimSpace(eventHeader) != GitLabDeploymentHook { return Observation{}, &IgnoredError{Reason: ReasonUnsupportedEvent} } @@ -103,9 +103,9 @@ func ParseGitLab(eventHeader, instanceHeader string, body []byte, cfg Config) (O return Observation{}, &InvalidError{Reason: "status_changed_at is not a valid date"} } - host := gitlabHost(instanceHeader, p.Project.WebURL) + host := gitlabWebURLHost(p.Project.WebURL) if host == "" { - return Observation{}, &InvalidError{Reason: "cannot determine the GitLab instance host"} + return Observation{}, &InvalidError{Reason: "missing field project.web_url"} } environment, ok := lookupGitLabEnvironment(cfg, p.Environment, p.EnvironmentTier) @@ -155,21 +155,23 @@ func parseGitLabTime(v string) (time.Time, error) { return time.Time{}, fmt.Errorf("status_changed_at is not a valid date") } -// gitlabHost returns the lower-cased host of the first of instance and -// webURL that parses to a non-empty host. -func gitlabHost(instance, webURL string) string { - for _, v := range []string{instance, webURL} { - v = strings.TrimSpace(v) - if v == "" { - continue - } - u, err := url.Parse(v) - if err != nil || u.Host == "" { - continue - } - return strings.ToLower(u.Host) - } - return "" +// gitlabWebURLHost returns the lower-cased host of project.web_url, the +// signed field the correlation key is derived from. X-Gitlab-Instance is not +// part of the Standard Webhooks signed content (webhook-id.webhook- +// timestamp.body) and is deliberately not consulted here: keying on it would +// let a captured, still-valid signed delivery be replayed with a different +// instance header to mint a new correlation key, defeating the replay +// window's idempotency guarantee. +func gitlabWebURLHost(webURL string) string { + webURL = strings.TrimSpace(webURL) + if webURL == "" { + return "" + } + u, err := url.Parse(webURL) + if err != nil || u.Host == "" { + return "" + } + return strings.ToLower(u.Host) } // lookupGitLabEnvironment resolves a GitLab environment to a Tracker diff --git a/internal/integrations/gitlab_test.go b/internal/integrations/gitlab_test.go index d444844..6c2e24c 100644 --- a/internal/integrations/gitlab_test.go +++ b/internal/integrations/gitlab_test.go @@ -48,7 +48,7 @@ func TestParseGitLab(t *testing.T) { cfg := gitlabConfig(t) body := gitlabBody(t, nil) - obs, err := ParseGitLab(GitLabDeploymentHook, "https://GitLab.Example.com", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.Equal(t, "gitlab:gitlab.example.com:42", obs.Key) @@ -69,11 +69,16 @@ func TestParseGitLab(t *testing.T) { assert.Equal(t, "Deploy payments a1b2c3d4 to production", obs.Title("payments")) }) - t.Run("empty instance header falls back to project.web_url host", func(t *testing.T) { + t.Run("key is derived from project.web_url only", func(t *testing.T) { + // X-Gitlab-Instance is not part of the Standard Webhooks signed + // content: ParseGitLab does not take it as input, so two deliveries + // identical except for that header always produce the same key. cfg := gitlabConfig(t) - body := gitlabBody(t, nil) + body := gitlabBody(t, func(m map[string]any) { + m["project"].(map[string]any)["web_url"] = "https://GitLab.Example.com/team/payments" + }) - obs, err := ParseGitLab(GitLabDeploymentHook, "", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.Equal(t, "gitlab:gitlab.example.com:42", obs.Key) }) @@ -84,7 +89,7 @@ func TestParseGitLab(t *testing.T) { m["status_changed_at"] = "2026-09-28T08:00:00Z" }) - obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.True(t, obs.At.Equal(time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC))) }) @@ -109,7 +114,7 @@ func TestParseGitLab(t *testing.T) { for _, tt := range cases { t.Run(tt.status, func(t *testing.T) { body := gitlabBody(t, func(m map[string]any) { m["status"] = tt.status }) - obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.Equal(t, tt.want, obs.Status) assert.Equal(t, tt.terminal, obs.Terminal) @@ -119,7 +124,7 @@ func TestParseGitLab(t *testing.T) { t.Run("approved", func(t *testing.T) { body := gitlabBody(t, func(m map[string]any) { m["status"] = "approved" }) - _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + _, err := ParseGitLab(GitLabDeploymentHook, body, cfg) var ig *IgnoredError require.ErrorAs(t, err, &ig) assert.Equal(t, ReasonApprovalIgnored, ig.Reason) @@ -127,7 +132,7 @@ func TestParseGitLab(t *testing.T) { t.Run("created", func(t *testing.T) { body := gitlabBody(t, func(m map[string]any) { m["status"] = "created" }) - _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + _, err := ParseGitLab(GitLabDeploymentHook, body, cfg) var ig *IgnoredError require.ErrorAs(t, err, &ig) assert.Equal(t, ReasonUnsupportedStatus, ig.Reason) @@ -154,7 +159,7 @@ func TestParseGitLab(t *testing.T) { m["environment"] = tt.environment m["environment_tier"] = tt.tier }) - obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.Equal(t, tt.want, obs.Environment) }) @@ -174,7 +179,7 @@ func TestParseGitLab(t *testing.T) { m["environment"] = tt.environment m["environment_tier"] = tt.tier }) - _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + _, err := ParseGitLab(GitLabDeploymentHook, body, cfg) var ig *IgnoredError require.ErrorAs(t, err, &ig) assert.Equal(t, ReasonUnmappedEnvironment, ig.Reason) @@ -184,7 +189,7 @@ func TestParseGitLab(t *testing.T) { t.Run("unsupported event", func(t *testing.T) { cfg := gitlabConfig(t) - _, err := ParseGitLab("Push Hook", "https://gitlab.example.com", []byte("{"), cfg) + _, err := ParseGitLab("Push Hook", []byte("{"), cfg) var ig *IgnoredError require.ErrorAs(t, err, &ig) assert.Equal(t, ReasonUnsupportedEvent, ig.Reason) @@ -193,7 +198,7 @@ func TestParseGitLab(t *testing.T) { t.Run("unsupported object_kind", func(t *testing.T) { cfg := gitlabConfig(t) body := gitlabBody(t, func(m map[string]any) { m["object_kind"] = "build" }) - _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + _, err := ParseGitLab(GitLabDeploymentHook, body, cfg) var ig *IgnoredError require.ErrorAs(t, err, &ig) assert.Equal(t, ReasonUnsupportedObjectKind, ig.Reason) @@ -201,7 +206,7 @@ func TestParseGitLab(t *testing.T) { t.Run("invalid JSON body", func(t *testing.T) { cfg := gitlabConfig(t) - _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", []byte("{"), cfg) + _, err := ParseGitLab(GitLabDeploymentHook, []byte("{"), cfg) var iv *InvalidError require.ErrorAs(t, err, &iv) }) @@ -219,18 +224,18 @@ func TestParseGitLab(t *testing.T) { } for _, mutate := range mutations { body := gitlabBody(t, mutate) - _, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + _, err := ParseGitLab(GitLabDeploymentHook, body, cfg) var iv *InvalidError require.ErrorAs(t, err, &iv) } }) - t.Run("no instance host determinable", func(t *testing.T) { + t.Run("no host determinable from project.web_url", func(t *testing.T) { cfg := gitlabConfig(t) body := gitlabBody(t, func(m map[string]any) { m["project"].(map[string]any)["web_url"] = "" }) - _, err := ParseGitLab(GitLabDeploymentHook, "", body, cfg) + _, err := ParseGitLab(GitLabDeploymentHook, body, cfg) var iv *InvalidError require.ErrorAs(t, err, &iv) }) @@ -240,7 +245,7 @@ func TestParseGitLab(t *testing.T) { body := gitlabBody(t, func(m map[string]any) { m["user"].(map[string]any)["username"] = "" }) - obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.Equal(t, "gitlab", obs.Owner) assert.Equal(t, "gitlab:unknown", obs.User) @@ -251,7 +256,7 @@ func TestParseGitLab(t *testing.T) { body := gitlabBody(t, func(m map[string]any) { m["environment_external_url"] = nil }) - obs, err := ParseGitLab(GitLabDeploymentHook, "https://gitlab.example.com", body, cfg) + obs, err := ParseGitLab(GitLabDeploymentHook, body, cfg) require.NoError(t, err) assert.NotContains(t, obs.Message, "URL:") }) diff --git a/server/integrations.go b/server/integrations.go index a74a673..8dd680c 100644 --- a/server/integrations.go +++ b/server/integrations.go @@ -151,7 +151,7 @@ func (h *integrationHandler) handleGitLab(w http.ResponseWriter, r *http.Request return integrations.VerifyGitLabSecretToken(h.cfg.GitLabSecretToken, header.Get(integrations.HeaderGitLabToken)) } parse := func(header http.Header, body []byte) (integrations.Observation, error) { - return integrations.ParseGitLab(header.Get(integrations.HeaderGitLabEvent), header.Get(integrations.HeaderGitLabInstance), body, h.cfg) + return integrations.ParseGitLab(header.Get(integrations.HeaderGitLabEvent), body, h.cfg) } h.serve(w, r, integrations.SourceGitLab, verify, parse) } diff --git a/server/integrations_test.go b/server/integrations_test.go index 4ae9130..eb93387 100644 --- a/server/integrations_test.go +++ b/server/integrations_test.go @@ -429,6 +429,26 @@ func TestGitLabReplayTenTimes(t *testing.T) { require.Len(t, env.locks(t), 1) } +func TestGitLabReplayWithDifferentInstanceHeaderIsDuplicate(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + + req := env.gitlabRequest(t, gitlabDeploymentBody(t, 601, "running", time.Now(), "production"), env.now) + rec := env.do(req) + require.Equal(t, http.StatusOK, rec.Code) + + // Same signed delivery (id, timestamp, signature and body all unchanged, + // still within the replay window): only X-Gitlab-Instance differs. The + // correlation key is derived from project.web_url, not this header, so + // replaying it must still be recognized as a duplicate of the same + // deployment instead of minting a second event. + req.Header.Set(integrations.HeaderGitLabInstance, "https://attacker.example.com") + rec2 := env.do(req) + require.Equal(t, http.StatusAccepted, rec2.Code) + require.JSONEq(t, `{"status":"ignored","reason":"duplicate"}`, rec2.Body.String()) + + env.onlyEvent(t) +} + func TestFluxInvalidSignatureAndStaleTimestamp(t *testing.T) { env := newIntegrationEnv(t, fullIntegrationConfig(t)) From 708bb4a18395f14110593729e007f36fbd99727b Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:39:37 +0200 Subject: [PATCH 18/22] fix(server): treat a deleted Tracker event as an ignored notification The event correlated with a claim can be deleted (DeleteEvents RPC or UI) while notifications for its deployment keep arriving. Because that is a business condition rather than a storage failure, the next update turned it into a permanent 500: the claim kept pointing at the deleted event id for the rest of its 90-day TTL, so every following notification (and every manual resend) failed the same way, and GitLab disables a webhook after repeated failures. The processor now recognizes mongo.ErrNoDocuments from the event store read, reverts the claim to its state before this Advance exactly as a genuine failure would, and reports a new ignored outcome (reason "event deleted") instead of an error, so the handler answers 202. Also fixes a related overwrite: SetEventID only takes effect while the claim's eventId is still empty. Without that guard, a claim abandoned and recreated after 30 seconds could have its second, legitimate SetEventID clobbered by a slow first write, orphaning the second event. The zero-matched case is treated exactly like the existing SetEventID failure: compensate and drop the claim. Refs #208 --- internal/integrations/errors.go | 5 +++++ internal/stores/integration_deployments.go | 10 ++++++++-- internal/stores/integration_deployments_test.go | 17 +++++++++++++++++ server/integrations.go | 2 ++ server/integrations_processor.go | 16 ++++++++++++++++ server/integrations_processor_test.go | 13 ++++++++++--- 6 files changed, 58 insertions(+), 5 deletions(-) diff --git a/internal/integrations/errors.go b/internal/integrations/errors.go index c65ca6c..c899cd5 100644 --- a/internal/integrations/errors.go +++ b/internal/integrations/errors.go @@ -25,6 +25,11 @@ const ( ReasonMissingRevision = "missing revision" ReasonDuplicate = "duplicate" ReasonStale = "stale" + // ReasonEventDeleted marks a notification whose correlated Tracker event + // no longer exists (e.g. deleted through DeleteEvents): a business + // condition, not a storage failure, so the request is answered 202 + // instead of 500. + ReasonEventDeleted = "event deleted" ) // IgnoredError marks a valid notification that Tracker does not record (HTTP 202). diff --git a/internal/stores/integration_deployments.go b/internal/stores/integration_deployments.go index 0ee5b25..1ade0fd 100644 --- a/internal/stores/integration_deployments.go +++ b/internal/stores/integration_deployments.go @@ -91,9 +91,15 @@ func (s *IntegrationDeploymentStore) Advance(ctx context.Context, key, status st return false, current, nil } -// SetEventID records the Tracker event created for this correlation. +// SetEventID records the Tracker event created for this correlation. It +// only takes effect while the claim's eventId is still empty: if a claim +// was abandoned and recreated after this write started, and a second +// SetEventID already recorded the new claim's event id, this write must not +// clobber it and orphan that event. ErrNotFound covers both a claim that no +// longer exists and one whose eventId is already set; either way the caller +// treats it like a failed SetEventID and compensates. func (s *IntegrationDeploymentStore) SetEventID(ctx context.Context, key, eventID string) error { - res, err := s.coll.UpdateOne(ctx, bson.M{"_id": key}, bson.M{"$set": bson.M{"eventId": eventID, "updatedAt": s.now().UTC()}}) + res, err := s.coll.UpdateOne(ctx, bson.M{"_id": key, "eventId": ""}, bson.M{"$set": bson.M{"eventId": eventID, "updatedAt": s.now().UTC()}}) if err != nil { return err } diff --git a/internal/stores/integration_deployments_test.go b/internal/stores/integration_deployments_test.go index a6fa1e3..69addb3 100644 --- a/internal/stores/integration_deployments_test.go +++ b/internal/stores/integration_deployments_test.go @@ -272,6 +272,23 @@ func TestIntegrationDeploymentSetEventIDGetDelete(t *testing.T) { assert.NoError(t, s.Delete(ctx, "missing")) } +func TestIntegrationDeploymentSetEventIDDoesNotOverwrite(t *testing.T) { + s := newIDStore(t) + ctx := context.Background() + t0 := time.Date(2026, 9, 28, 8, 0, 0, 0, time.UTC) + k := "set-event-id-no-overwrite-key" + + _, _, err := s.Claim(ctx, k, "gitlab", "start", t0, 1) + require.NoError(t, err) + require.NoError(t, s.SetEventID(ctx, k, "evt-1")) + + assert.ErrorIs(t, s.SetEventID(ctx, k, "evt-2"), ErrNotFound) + + got, err := s.Get(ctx, k) + require.NoError(t, err) + assert.Equal(t, "evt-1", got.EventID) +} + func TestIntegrationDeploymentIndexes(t *testing.T) { ctx := context.Background() db := testDatabase(t) diff --git a/server/integrations.go b/server/integrations.go index 8dd680c..01e016e 100644 --- a/server/integrations.go +++ b/server/integrations.go @@ -248,6 +248,8 @@ func (h *integrationHandler) serve(w http.ResponseWriter, r *http.Request, sourc } switch res.Outcome { + case outcomeIgnored: + finish(http.StatusAccepted, resultIgnored, integrationIgnored{Status: "ignored", Reason: res.Reason}, "key", obs.Key, "reason", res.Reason) case outcomeDuplicate, outcomeStale: finish(http.StatusAccepted, resultDuplicate, integrationIgnored{Status: "ignored", Reason: res.Outcome}, "key", obs.Key, "eventId", res.EventID) case outcomeLockConflict: diff --git a/server/integrations_processor.go b/server/integrations_processor.go index 6adefb4..a730f72 100644 --- a/server/integrations_processor.go +++ b/server/integrations_processor.go @@ -12,6 +12,7 @@ import ( eventv1 "github.com/bananaops/tracker/generated/proto/event/v1alpha1" "github.com/bananaops/tracker/internal/integrations" store "github.com/bananaops/tracker/internal/stores" + "go.mongodb.org/mongo-driver/mongo" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -27,6 +28,7 @@ const ( outcomeLockConflict = "lock_conflict" outcomeDuplicate = "duplicate" outcomeStale = "stale" + outcomeIgnored = "ignored" ) // setEventIDRetryDelays are the waits between SetEventID attempts, after the @@ -59,6 +61,9 @@ type CatalogLister interface { type ProcessResult struct { Outcome string EventID string + // Reason is set for Outcome == outcomeIgnored, giving the 202 response a + // specific reason (e.g. ReasonEventDeleted) rather than a bare status. + Reason string } type catalogEntry struct{ name, repository string } @@ -361,6 +366,17 @@ func (p *IntegrationProcessor) update(ctx context.Context, obs integrations.Obse current, err := p.events.store.Get(ctx, map[string]interface{}{"metadata.id": prev.EventID}) if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + // The event was deleted (RPC or UI), not a storage failure: revert + // the claim to its state before this Advance, same as a genuine + // failure would, but answer 202 instead of 500 so a retrying + // sender (or GitLab disabling the webhook after repeated 5xx) + // never has to deal with a permanently failing key. + if revertErr := p.store.Revert(ctx, obs.Key, prev, status, rank, obs.At); revertErr != nil { + p.logger.Error("integration: cannot revert the claim after a deleted event", "key", obs.Key, "eventId", prev.EventID, "error", revertErr) + } + return ProcessResult{Outcome: outcomeIgnored, Reason: integrations.ReasonEventDeleted, EventID: prev.EventID}, nil + } return fail(err) } diff --git a/server/integrations_processor_test.go b/server/integrations_processor_test.go index 596008b..7d4681d 100644 --- a/server/integrations_processor_test.go +++ b/server/integrations_processor_test.go @@ -350,7 +350,12 @@ func TestProcessRunningThenLaterApprovalIsRejected(t *testing.T) { require.Equal(t, res.EventID, locks[0].EventId) } -func TestProcessUpdateFailureReverts(t *testing.T) { +// TestProcessDeletedEventIsIgnored covers I2: the event correlated with a +// claim can be deleted out from under it (RPC or UI). The next notification +// for that deployment must not turn a business condition into a permanent +// 500: it reverts the claim to its state before this Advance and reports an +// ignored outcome instead. +func TestProcessDeletedEventIsIgnored(t *testing.T) { env := newProcEnv(t) ctx := context.Background() t0 := time.Now().UTC().Truncate(time.Second) @@ -360,8 +365,10 @@ func TestProcessUpdateFailureReverts(t *testing.T) { require.NoError(t, env.events.store.Delete(ctx, map[string]interface{}{"metadata.id": res.EventID})) - _, err = env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) - require.Error(t, err) + res2, err := env.proc.Process(ctx, obs(eventv1.Status_success, t0.Add(time.Second))) + require.NoError(t, err) + require.Equal(t, outcomeIgnored, res2.Outcome) + require.Equal(t, integrations.ReasonEventDeleted, res2.Reason) doc, err := env.store.Get(ctx, obs(eventv1.Status_start, t0).Key) require.NoError(t, err) From d5e614eff9d7a435b2d15682bb11caeaae71aee5 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:41:43 +0200 Subject: [PATCH 19/22] fix(server): log the cause of integration failures and bound logged delivery ids Every 500 response from the webhook handler logged only the key, never the underlying error: Claim, Advance, createEvent and SetEventID failures (and abandoned claims) left an operator with nothing to act on, unlike the 401/400 paths, which already carry a reason. Both Process error branches now log "reason", err.Error(); none of these errors (Mongo, processor) ever carries a secret, header or body. Delivery identifiers logged before the request is authenticated (Idempotency-Key, X-Gitlab-Event-UUID, webhook-id) are now truncated to 64 bytes so an anonymous caller cannot inflate log volume by sending an oversized header, and X-Gitlab-Instance joins them: it no longer feeds the correlation key but is still useful for debugging. Refs #208 --- server/integrations.go | 24 +++++++++++++++++++----- server/integrations_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/server/integrations.go b/server/integrations.go index 01e016e..266fef0 100644 --- a/server/integrations.go +++ b/server/integrations.go @@ -143,6 +143,19 @@ type integrationVerify func(header http.Header, body []byte) error // integrations.ErrStaleTimestamp. type integrationParse func(header http.Header, body []byte) (integrations.Observation, error) +// maxLoggedHeaderBytes bounds a delivery identifier logged before the +// request is authenticated, so an anonymous caller cannot inflate log +// volume by sending an oversized header. +const maxLoggedHeaderBytes = 64 + +// truncateHeader bounds v to maxLoggedHeaderBytes for logging. +func truncateHeader(v string) string { + if len(v) <= maxLoggedHeaderBytes { + return v + } + return v[:maxLoggedHeaderBytes] +} + func (h *integrationHandler) handleGitLab(w http.ResponseWriter, r *http.Request, _ map[string]string) { verify := func(header http.Header, body []byte) error { if len(h.cfg.GitLabSigningKey) > 0 { @@ -174,9 +187,10 @@ func (h *integrationHandler) serve(w http.ResponseWriter, r *http.Request, sourc baseAttrs := []any{"source", source} if source == integrations.SourceGitLab { baseAttrs = append(baseAttrs, - "idempotencyKey", r.Header.Get("Idempotency-Key"), - "gitlabEventUUID", r.Header.Get("X-Gitlab-Event-UUID"), - "webhookId", r.Header.Get(integrations.HeaderWebhookID), + "idempotencyKey", truncateHeader(r.Header.Get("Idempotency-Key")), + "gitlabEventUUID", truncateHeader(r.Header.Get("X-Gitlab-Event-UUID")), + "webhookId", truncateHeader(r.Header.Get(integrations.HeaderWebhookID)), + "gitlabInstance", truncateHeader(r.Header.Get(integrations.HeaderGitLabInstance)), ) } @@ -240,10 +254,10 @@ func (h *integrationHandler) serve(w http.ResponseWriter, r *http.Request, sourc res, err := h.processor.Process(ctx, obs) if err != nil { if errors.Is(err, errClaimPending) { - finish(http.StatusInternalServerError, resultError, integrationError{Error: "deployment is being recorded, retry later"}, "key", obs.Key) + finish(http.StatusInternalServerError, resultError, integrationError{Error: "deployment is being recorded, retry later"}, "key", obs.Key, "reason", err.Error()) return } - finish(http.StatusInternalServerError, resultError, integrationError{Error: "storage failure"}, "key", obs.Key) + finish(http.StatusInternalServerError, resultError, integrationError{Error: "storage failure"}, "key", obs.Key, "reason", err.Error()) return } diff --git a/server/integrations_test.go b/server/integrations_test.go index eb93387..51c0854 100644 --- a/server/integrations_test.go +++ b/server/integrations_test.go @@ -108,9 +108,13 @@ func TestGitLabTimestampOutsideTolerance(t *testing.T) { env.now.Add(-5*time.Minute - time.Second), env.now.Add(5*time.Minute + time.Second), } { + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "unauthorized")) req := env.gitlabRequest(t, body, ts) rec := env.do(req) require.Equal(t, http.StatusUnauthorized, rec.Code, "ts %s", ts) + require.JSONEq(t, `{"error":"invalid signature"}`, rec.Body.String(), "ts %s", ts) + after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "unauthorized")) + require.Equal(t, before+1, after, "ts %s", ts) } require.Equal(t, int64(0), env.count(t, "events")) @@ -153,6 +157,8 @@ func TestGitLabBodyTooLarge(t *testing.T) { after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "invalid")) require.Equal(t, before+1, after) + require.Equal(t, int64(0), env.count(t, "events")) + require.Equal(t, int64(0), env.count(t, "integration_deployments")) } func TestGitLabInvalidJSON(t *testing.T) { @@ -167,6 +173,25 @@ func TestGitLabInvalidJSON(t *testing.T) { require.Equal(t, before+1, after) } +// TestGitLabUnsignedPushHookIsUnauthorized enforces the verify-then-parse +// order: an unsigned request whose event would otherwise be ignored (202) +// must still be rejected as unauthorized before parsing ever runs. +func TestGitLabUnsignedPushHookIsUnauthorized(t *testing.T) { + env := newIntegrationEnv(t, fullIntegrationConfig(t)) + + body := gitlabDeploymentBody(t, 47, "running", time.Now(), "production") + req, err := http.NewRequest(http.MethodPost, integrationGitLabPath, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(integrations.HeaderGitLabEvent, "Push Hook") + + rec := env.do(req) + require.Equal(t, http.StatusUnauthorized, rec.Code) + require.JSONEq(t, `{"error":"invalid signature"}`, rec.Body.String()) + require.Equal(t, int64(0), env.count(t, "events")) + require.Equal(t, int64(0), env.count(t, "integration_deployments")) +} + func TestIntegrationIgnoredCases(t *testing.T) { env := newIntegrationEnv(t, fullIntegrationConfig(t)) @@ -452,6 +477,8 @@ func TestGitLabReplayWithDifferentInstanceHeaderIsDuplicate(t *testing.T) { func TestFluxInvalidSignatureAndStaleTimestamp(t *testing.T) { env := newIntegrationEnv(t, fullIntegrationConfig(t)) + before := testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "unauthorized")) + body := fluxEventBody(t, "info", "Progressing", env.now, "main@sha1:abcdef1234567890abcdef1234567890abcdef12") badKeyReq, err := http.NewRequest(http.MethodPost, integrationFluxPath, bytes.NewReader(body)) require.NoError(t, err) @@ -461,10 +488,15 @@ func TestFluxInvalidSignatureAndStaleTimestamp(t *testing.T) { badKeyReq.Header.Set(integrations.HeaderFluxSignature, sig) rec := env.do(badKeyReq) require.Equal(t, http.StatusUnauthorized, rec.Code) + require.JSONEq(t, `{"error":"invalid signature"}`, rec.Body.String()) staleBody := fluxEventBody(t, "info", "Progressing", env.now.Add(-5*time.Minute-time.Second), "main@sha1:abcdef1234567890abcdef1234567890abcdef12") rec2 := env.do(env.fluxRequest(t, staleBody)) require.Equal(t, http.StatusUnauthorized, rec2.Code) + require.JSONEq(t, `{"error":"invalid signature"}`, rec2.Body.String()) + + after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("flux", "unauthorized")) + require.Equal(t, before+2, after) require.Equal(t, int64(0), env.count(t, "events")) require.Equal(t, int64(0), env.count(t, "integration_deployments")) @@ -551,6 +583,8 @@ func TestGitLabClaimPendingReturns500(t *testing.T) { after := testutil.ToFloat64(integrationWebhooks.WithLabelValues("gitlab", "error")) require.Equal(t, before+1, after) + + require.Contains(t, env.logs.String(), `"reason":"deployment creation still in progress"`) } func TestGitLabCatalogResolution(t *testing.T) { From e199414c6e03b4d7518ec38fb5899c370c1355e5 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:45:45 +0200 Subject: [PATCH 20/22] test(server): pin the remaining event RPC behaviours Characterization tests for the event.go RPC paths the refactor moved but left unmodified, none of it changed by this commit: CreateEvent's error text for an unknown related_id and the positive duration it sets for a known one, UpdateEvent's error text for an unknown slack id, no lock release through the RPC on a non-terminal status, and the updateEvent changelog rules (ticket linked, priority updated, title updated, an owner-only change recorded as an approval, and the generic update entry for anything else) with their exact field/old/ new values. Refs #208 --- server/event_core_test.go | 260 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) diff --git a/server/event_core_test.go b/server/event_core_test.go index 4dfa646..7b35652 100644 --- a/server/event_core_test.go +++ b/server/event_core_test.go @@ -317,3 +317,263 @@ func TestCreateEventCountsOneAuthorization(t *testing.T) { after := gatheredCounter(t, "tracker_auth_requests_total", map[string]string{"principal": "user", "result": "allowed"}) require.Equal(t, float64(1), after-before) } + +// -- Task 10 (deferred T8): characterization tests for the remaining RPC +// behaviours, pinned green on the current code, no logic changed. -- + +func TestCreateEventRPCUnknownRelatedIDErrorText(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + req := baseCreateEventRequest() + req.Attributes.RelatedId = "does-not-exist" + + _, err := e.CreateEvent(eventRPCCtx("CreateEvent"), req) + require.EqualError(t, err, "no event found in tracker for attributes.related_id does-not-exist") +} + +func TestCreateEventRPCKnownRelatedIDSetsPositiveDuration(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + related, err := e.CreateEvent(eventRPCCtx("CreateEvent"), baseCreateEventRequest()) + require.NoError(t, err) + + req := baseCreateEventRequest() + req.Attributes.Service = "svc2" + req.Attributes.RelatedId = related.Event.Metadata.Id + resp, err := e.CreateEvent(eventRPCCtx("CreateEvent"), req) + require.NoError(t, err) + + require.NotNil(t, resp.Event.Metadata.Duration) + require.Greater(t, resp.Event.Metadata.Duration.AsDuration(), time.Duration(0)) +} + +func TestUpdateEventRPCUnknownSlackIDErrorText(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + req := baseCreateEventRequest() + _, err := e.UpdateEvent(eventRPCCtx("UpdateEvent"), &v1alpha1.UpdateEventRequest{ + SlackId: "SLACK-404", + Title: "deploy svc", + Attributes: req.Attributes, + Links: &v1alpha1.EventLinks{}, + }) + require.EqualError(t, err, "no event found in tracker for id SLACK-404") +} + +func TestUpdateEventRPCNoLockReleaseOnNonTerminalStatus(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + + created, err := e.CreateEvent(eventRPCCtx("CreateEvent"), baseCreateEventRequest()) + require.NoError(t, err) + id := created.Event.Metadata.Id + + req := baseCreateEventRequest() + req.Attributes.Status = v1alpha1.Status_warning + _, err = e.UpdateEvent(eventRPCCtx("UpdateEvent"), &v1alpha1.UpdateEventRequest{ + Id: id, + Title: "deploy svc", + Attributes: req.Attributes, + Links: &v1alpha1.EventLinks{}, + }) + require.NoError(t, err) + + locks, err := store.NewStoreLockFromCollection(db.Collection("locks")).List(context.Background()) + require.NoError(t, err) + require.Len(t, locks, 1, "a non-terminal status must not release the lock") +} + +// baseEventForUpdate creates and returns a fresh deployment start event on +// service, for the updateEvent changelog characterization tests below. +func baseEventForUpdate(t *testing.T, e *Event, service string) *v1alpha1.Event { + t.Helper() + ev := &v1alpha1.Event{ + Title: "deploy " + service, + Attributes: &v1alpha1.EventAttributes{ + Type: v1alpha1.Type_deployment, + Status: v1alpha1.Status_start, + Service: service, + Environment: v1alpha1.Environment_production, + Owner: "alice", + Priority: v1alpha1.Priority_P3, + Source: "manual", + }, + Links: &v1alpha1.EventLinks{}, + Metadata: &v1alpha1.EventMetadata{}, + } + created, _, err := e.createEvent(context.Background(), ev, "alice", "", lockStrict) + require.NoError(t, err) + return created +} + +// copyEventForUpdate returns a new *v1alpha1.Event carrying the same fields +// as current, the shape updateEvent's "next" argument takes from every real +// caller (RPC or integration processor): a full copy with exactly one field +// then changed by the test. +func copyEventForUpdate(current *v1alpha1.Event) *v1alpha1.Event { + return &v1alpha1.Event{ + Title: current.Title, + Attributes: &v1alpha1.EventAttributes{ + Message: current.Attributes.Message, + Source: current.Attributes.Source, + Type: current.Attributes.Type, + Priority: current.Attributes.Priority, + Impact: current.Attributes.Impact, + Environment: current.Attributes.Environment, + Owner: current.Attributes.Owner, + RelatedId: current.Attributes.RelatedId, + Service: current.Attributes.Service, + Status: current.Attributes.Status, + StartDate: current.Attributes.StartDate, + EndDate: current.Attributes.EndDate, + StakeHolders: current.Attributes.StakeHolders, + Notification: current.Attributes.Notification, + Notifications: current.Attributes.Notifications, + }, + Links: &v1alpha1.EventLinks{ + PullRequestLink: current.Links.PullRequestLink, + Ticket: current.Links.Ticket, + }, + Metadata: &v1alpha1.EventMetadata{ + Id: current.Metadata.Id, + CreatedAt: current.Metadata.CreatedAt, + SlackId: current.Metadata.SlackId, + Duration: current.Metadata.Duration, + }, + } +} + +// changelogEntry returns the last changelog entry of changeType in entries, +// or nil. +func changelogEntry(entries []*v1alpha1.ChangelogEntry, changeType v1alpha1.ChangeType) *v1alpha1.ChangelogEntry { + var found *v1alpha1.ChangelogEntry + for _, entry := range entries { + if entry.ChangeType == changeType { + found = entry + } + } + return found +} + +func TestUpdateEventChangelogTicketLinked(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + current := baseEventForUpdate(t, e, "svc-ticket") + + next := copyEventForUpdate(current) + next.Links.Ticket = "JIRA-1" + + // e.store.Update's FindOneAndUpdate defaults to returning the document + // before the update, so the changelog it just wrote is re-read from the + // store instead of taken from updateEvent's own return value. + _, err := e.updateEvent(context.Background(), current, next, map[string]interface{}{"metadata.id": current.Metadata.Id}, "alice", "", time.Now()) + require.NoError(t, err) + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": current.Metadata.Id}) + require.NoError(t, err) + + entry := changelogEntry(updated.Changelog, v1alpha1.ChangeType_linked) + require.NotNil(t, entry) + require.Equal(t, "ticket", entry.Field) + require.Equal(t, "", entry.OldValue) + require.Equal(t, "JIRA-1", entry.NewValue) + require.Equal(t, "Jira ticket linked", entry.Comment) +} + +func TestUpdateEventChangelogPriorityUpdated(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + current := baseEventForUpdate(t, e, "svc-priority") + + next := copyEventForUpdate(current) + next.Attributes.Priority = v1alpha1.Priority_P1 + + _, err := e.updateEvent(context.Background(), current, next, map[string]interface{}{"metadata.id": current.Metadata.Id}, "alice", "", time.Now()) + require.NoError(t, err) + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": current.Metadata.Id}) + require.NoError(t, err) + + var entry *v1alpha1.ChangelogEntry + for _, c := range updated.Changelog { + if c.ChangeType == v1alpha1.ChangeType_updated && c.Field == "priority" { + entry = c + } + } + require.NotNil(t, entry) + require.Equal(t, "P3", entry.OldValue) + require.Equal(t, "P1", entry.NewValue) + require.Equal(t, "Priority updated", entry.Comment) +} + +func TestUpdateEventChangelogTitleUpdated(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + current := baseEventForUpdate(t, e, "svc-title") + + next := copyEventForUpdate(current) + next.Title = "deploy svc-title v2" + + _, err := e.updateEvent(context.Background(), current, next, map[string]interface{}{"metadata.id": current.Metadata.Id}, "alice", "", time.Now()) + require.NoError(t, err) + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": current.Metadata.Id}) + require.NoError(t, err) + + var entry *v1alpha1.ChangelogEntry + for _, c := range updated.Changelog { + if c.ChangeType == v1alpha1.ChangeType_updated && c.Field == "title" { + entry = c + } + } + require.NotNil(t, entry) + require.Equal(t, "deploy svc-title", entry.OldValue) + require.Equal(t, "deploy svc-title v2", entry.NewValue) + require.Equal(t, "Title updated", entry.Comment) +} + +// TestUpdateEventChangelogApproval covers the owner-only change updateEvent +// treats as an approval: title, status and priority all unchanged, only the +// owner differs. +func TestUpdateEventChangelogApproval(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + current := baseEventForUpdate(t, e, "svc-approval") + + next := copyEventForUpdate(current) + next.Attributes.Owner = "bob" + + _, err := e.updateEvent(context.Background(), current, next, map[string]interface{}{"metadata.id": current.Metadata.Id}, "bob", "", time.Now()) + require.NoError(t, err) + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": current.Metadata.Id}) + require.NoError(t, err) + + entry := changelogEntry(updated.Changelog, v1alpha1.ChangeType_approved) + require.NotNil(t, entry) + require.Equal(t, "bob", entry.User) + require.Equal(t, "Event approved by bob", entry.Comment) +} + +// TestUpdateEventChangelogGenericUpdate covers a change to a field none of +// the specific rules track (here, the message): no more specific entry was +// added, so the generic "Event updated" entry closes the changelog. +func TestUpdateEventChangelogGenericUpdate(t *testing.T) { + db := testMongoDatabase(t) + e := newTestEvent(t, db) + current := baseEventForUpdate(t, e, "svc-generic") + + next := copyEventForUpdate(current) + next.Attributes.Message = "new message" + + _, err := e.updateEvent(context.Background(), current, next, map[string]interface{}{"metadata.id": current.Metadata.Id}, "alice", "", time.Now()) + require.NoError(t, err) + updated, err := store.NewStoreEventFromCollection(db.Collection("events")).Get(context.Background(), map[string]interface{}{"metadata.id": current.Metadata.Id}) + require.NoError(t, err) + + last := updated.Changelog[len(updated.Changelog)-1] + require.Equal(t, v1alpha1.ChangeType_updated, last.ChangeType) + require.Equal(t, "", last.Field) + require.Equal(t, "", last.OldValue) + require.Equal(t, "", last.NewValue) + require.Equal(t, "Event updated", last.Comment) +} From f472a9e63046a703a5227dc45a2ff01950b84c9d Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:46:37 +0200 Subject: [PATCH 21/22] fix(integrations): normalize default ports and uppercase .git suffixes NormalizeRepoURL now drops an explicit default port (:443 on https, :80 on http) and strips a .git suffix case-insensitively, so a catalog repository URL written with either form still matches the normalized URLs built from a webhook payload. parseGitLabTime now returns (time.Time, bool) instead of building an error the caller only used to decide whether parsing failed and then discarded; behaviour is unchanged. Refs #208 --- internal/integrations/gitlab.go | 13 +++++++------ internal/integrations/repository.go | 22 ++++++++++++++++++++-- internal/integrations/repository_test.go | 5 +++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/internal/integrations/gitlab.go b/internal/integrations/gitlab.go index 04a4069..d7af53c 100644 --- a/internal/integrations/gitlab.go +++ b/internal/integrations/gitlab.go @@ -98,8 +98,8 @@ func ParseGitLab(eventHeader string, body []byte, cfg Config) (Observation, erro return Observation{}, &InvalidError{Reason: "missing field project.path_with_namespace"} } - at, err := parseGitLabTime(p.StatusChangedAt) - if err != nil { + at, ok := parseGitLabTime(p.StatusChangedAt) + if !ok { return Observation{}, &InvalidError{Reason: "status_changed_at is not a valid date"} } @@ -145,14 +145,15 @@ func ParseGitLab(eventHeader string, body []byte, cfg Config) (Observation, erro } // parseGitLabTime parses v with the first layout in gitlabTimeLayouts that -// matches, and returns it in UTC. -func parseGitLabTime(v string) (time.Time, error) { +// matches, and returns it in UTC. ok is false when no layout matches; the +// caller builds its own InvalidError, so no error value is built here. +func parseGitLabTime(v string) (t time.Time, ok bool) { for _, layout := range gitlabTimeLayouts { if t, err := time.Parse(layout, v); err == nil { - return t.UTC(), nil + return t.UTC(), true } } - return time.Time{}, fmt.Errorf("status_changed_at is not a valid date") + return time.Time{}, false } // gitlabWebURLHost returns the lower-cased host of project.web_url, the diff --git a/internal/integrations/repository.go b/internal/integrations/repository.go index bbb026b..a15c762 100644 --- a/internal/integrations/repository.go +++ b/internal/integrations/repository.go @@ -1,6 +1,7 @@ package integrations import ( + "net" "net/url" "regexp" "strings" @@ -8,6 +9,21 @@ import ( var scpLikeRepo = regexp.MustCompile(`^[A-Za-z0-9._-]+@([A-Za-z0-9.-]+):(.+)$`) +// dropDefaultPort strips an explicit port from host when it is the scheme's +// default (443 for https, 80 for http), so a catalog repository URL written +// with or without that port still normalizes to the same key. Any other +// port, or a host with none, is returned unchanged. +func dropDefaultPort(scheme, host string) string { + h, port, err := net.SplitHostPort(host) + if err != nil { + return host + } + if (scheme == "https" && port == "443") || (scheme == "http" && port == "80") { + return h + } + return host +} + // NormalizeRepoURL turns the SSH and HTTP forms of a repository URL into // https://host/path: scheme and host lower case, path kept as is, trailing // slashes then the .git suffix removed, credentials dropped. It returns "" @@ -29,7 +45,7 @@ func NormalizeRepoURL(raw string) string { case "ssh", "git+ssh": scheme, host = "https", u.Hostname() case "http", "https": - scheme, host = strings.ToLower(u.Scheme), u.Host + scheme, host = strings.ToLower(u.Scheme), dropDefaultPort(strings.ToLower(u.Scheme), u.Host) default: return "" } @@ -37,7 +53,9 @@ func NormalizeRepoURL(raw string) string { } path = strings.TrimLeft(path, "/") path = strings.TrimRight(path, "/") - path = strings.TrimSuffix(path, ".git") + if len(path) >= 4 && strings.EqualFold(path[len(path)-4:], ".git") { + path = path[:len(path)-4] + } if host == "" || path == "" { return "" } diff --git a/internal/integrations/repository_test.go b/internal/integrations/repository_test.go index 537df35..a354936 100644 --- a/internal/integrations/repository_test.go +++ b/internal/integrations/repository_test.go @@ -26,6 +26,11 @@ func TestNormalizeRepoURL(t *testing.T) { {"unsupported scheme", "ftp://host/x", ""}, {"no path", "https://gitlab.example.com", ""}, {"root path only", "https://gitlab.example.com/", ""}, + {"explicit default https port dropped", "https://gitlab.example.com:443/team/payments", "https://gitlab.example.com/team/payments"}, + {"explicit default http port dropped", "http://gitlab.local:80/team/payments", "http://gitlab.local/team/payments"}, + {"non-default https port on http scheme kept", "http://gitlab.local:443/team/payments", "http://gitlab.local:443/team/payments"}, + {"uppercase git suffix", "https://gitlab.example.com/team/payments.GIT", "https://gitlab.example.com/team/payments"}, + {"mixed case git suffix", "https://gitlab.example.com/team/payments.Git", "https://gitlab.example.com/team/payments"}, } for _, tt := range cases { From 79ef516a0b69775319e9b0ab661cde5250a33999 Mon Sep 17 00:00:00 2001 From: umignon Date: Mon, 28 Sep 2026 15:48:20 +0200 Subject: [PATCH 22/22] docs: clarify integration retries and lock recovery The "sender should retry" note was only true for Flux. Checked against GitLab's current docs: a failed delivery is not retried automatically; GitLab only disables the webhook after repeated consecutive failures (temporarily, then permanently past 40), and a lost terminal notification has to be resent by hand from the webhook's Recent events (or the equivalent resend API). Documented both paths under a new Retries section, linked from the 500 row. Also documents the event deleted 202 reason, the missing URL: line in the GitLab message layout, and how to find and release a lock orphaned by a crash between event creation and correlation. Source: https://docs.gitlab.com/user/project/integrations/webhooks/ Refs #208 --- docs/INTEGRATIONS.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 40d4986..1777dad 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -81,8 +81,9 @@ No change is needed in your `.gitlab-ci.yml`: the deployment jobs already declar | anything else | (none) | | `202`, reason `unsupported status` | Title, message and changelog: the event title is `Deploy to `, -the message lists the commit title, short SHA, commit URL, deployment job URL and GitLab -environment name. The changelog entry and lock owner are attributed to `gitlab:`. +the message lists the commit title, short SHA, commit URL, deployment job URL, GitLab environment +name and, when the payload carries one, `URL: `. The changelog entry and +lock owner are attributed to `gitlab:`. ## Flux @@ -200,7 +201,7 @@ The catalog snapshot used for this matching is cached for 60 seconds. | `401` | Authentication failure: missing or invalid signature, timestamp outside the tolerance window, or a rejected API key. | | `404` | This source is not configured: the endpoint does not exist. | | `413` | Request body larger than 1 MiB. | -| `500` | Storage failure. The sender should retry; retries are safe (see the reasons below). | +| `500` | Storage failure. Retrying is safe: the correlation key makes a retried notification a `duplicate` or a `stale` no-op once the original attempt actually succeeded. See [Retries](#retries) for what actually happens next, which differs between Flux and GitLab. | ### `202` reasons @@ -216,6 +217,7 @@ The catalog snapshot used for this matching is cached for 60 seconds. | `missing revision` | The Flux event has no `kustomize.toolkit.fluxcd.io/revision` or `helm.toolkit.fluxcd.io/revision` metadata. | Check the Kustomization or HelmRelease reports a revision; some early `Progressing` events do not. | | `duplicate` | Same status, same timestamp as the last one applied. | No action; the sender likely retried a delivery. | | `stale` | An older or already superseded notification for this deployment. | No action; out-of-order or replayed delivery, correctly discarded. | +| `event deleted` | The Tracker event this deployment was correlated with was deleted (RPC or UI) since the last notification. | No action; the deployment's remaining notifications are ignored the same way. To keep tracking it, deployments would need to start a new event, which does not happen automatically. | ### Metric @@ -225,6 +227,33 @@ The catalog snapshot used for this matching is cached for 60 seconds. that request; `duplicate` also covers `stale`; `invalid` covers both `400` and `413`). A rejected API key presented on these routes is counted only in `tracker_auth_requests_total`, not here. +### Retries + +What happens after a `500` differs by sender: + +- **Flux**: notification-controller retries a failed notification on its own; nothing to do. +- **GitLab**: a single failed delivery is *not* retried automatically. GitLab only disables the + webhook after repeated consecutive failures (temporarily at first, with a backoff that grows up + to 24h; permanently after 40 consecutive failures), which re-enables itself once a manual test + request succeeds. If a `success`, `failure` or another terminal notification is lost this way, + the event stays open (`start` or `waiting_approval`) with its lock held until someone resends + it: open the webhook's **Settings > Webhooks > Edit > Recent events**, find the failed delivery, + and use **View details > Resend Request** (or the equivalent + [webhook API resend endpoint](https://docs.gitlab.com/user/project/integrations/webhooks/)). + Resending replays the exact same signed body, id and timestamp, so Tracker's own idempotency + check still applies if the original attempt had actually succeeded server-side. + +### Recovering an orphaned lock + +A process crash between creating the Tracker event and recording its id on the correlation claim +can leave the event's lock behind: the claim is dropped after 30 seconds and a later notification +recreates the event, but the lock taken for the first, now-orphaned event is never released by +that recreation and blocks new deployments of the same service and environment until it is freed. +List locks for the affected service and environment (`LockService/ListLocks`, or the REST endpoint +listing locks, see [LOCKS.md](LOCKS.md#grpc-api)) and release the stale one through +`LockService/UnLock` or the equivalent REST/UI action, exactly as you would for a lock left by a +manual operation. + ## Security The signature is verified on the raw request body, before any JSON decoding: a malformed or