From 086997850132d9db183cfd2c981cdab01c11e563 Mon Sep 17 00:00:00 2001 From: Chris Cappuccio Date: Tue, 1 Sep 2026 12:22:32 -0700 Subject: [PATCH 1/3] Stage verified alert-rule quarantine patch --- .github/alert-rule-quarantine-patch/part-00 | 91 +++++++++ .github/alert-rule-quarantine-patch/part-01 | 109 +++++++++++ .github/alert-rule-quarantine-patch/part-02 | 129 ++++++++++++ .github/alert-rule-quarantine-patch/part-03 | 150 ++++++++++++++ .github/alert-rule-quarantine-patch/part-04 | 185 ++++++++++++++++++ .github/alert-rule-quarantine-patch/part-05 | 139 +++++++++++++ .github/alert-rule-quarantine-patch/part-06 | 77 ++++++++ .../workflows/apply-alert-rule-quarantine.yml | 30 +++ 8 files changed, 910 insertions(+) create mode 100644 .github/alert-rule-quarantine-patch/part-00 create mode 100644 .github/alert-rule-quarantine-patch/part-01 create mode 100644 .github/alert-rule-quarantine-patch/part-02 create mode 100644 .github/alert-rule-quarantine-patch/part-03 create mode 100644 .github/alert-rule-quarantine-patch/part-04 create mode 100644 .github/alert-rule-quarantine-patch/part-05 create mode 100644 .github/alert-rule-quarantine-patch/part-06 create mode 100644 .github/workflows/apply-alert-rule-quarantine.yml diff --git a/.github/alert-rule-quarantine-patch/part-00 b/.github/alert-rule-quarantine-patch/part-00 new file mode 100644 index 0000000..5be50a6 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-00 @@ -0,0 +1,91 @@ +diff --git a/cmd/server/schema_optional.go b/cmd/server/schema_optional.go +index e9e9a0d..877dcc6 100644 +--- a/cmd/server/schema_optional.go ++++ b/cmd/server/schema_optional.go +@@ -44,7 +44,9 @@ func ensureRuntimeSchema(db *sql.DB) error { + ADD COLUMN IF NOT EXISTS target_role VARCHAR(16) NOT NULL DEFAULT 'all', + ADD COLUMN IF NOT EXISTS require_alertable BOOLEAN NOT NULL DEFAULT TRUE, + ADD COLUMN IF NOT EXISTS severity VARCHAR(20) NOT NULL DEFAULT 'auto', +- ADD COLUMN IF NOT EXISTS notify_recovery BOOLEAN NOT NULL DEFAULT TRUE`, ++ ADD COLUMN IF NOT EXISTS notify_recovery BOOLEAN NOT NULL DEFAULT TRUE, ++ ADD COLUMN IF NOT EXISTS quarantine_reason TEXT, ++ ADD COLUMN IF NOT EXISTS quarantined_at TIMESTAMPTZ`, + `ALTER TABLE alerts + ADD COLUMN IF NOT EXISTS notified_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS notify_error TEXT, +@@ -165,7 +167,7 @@ func validateRuntimeSchema(db *sql.DB) error { + "sites": {"id", "name", "tower_h_m"}, + "devices": {"id", "mac", "ip_address", "role", "managed", "alertable", "alert_silenced_until", "username", "password", "status"}, + "scheduled_jobs": {"id", "status", "progress", "total_devices", "completed_devices", "error_message"}, +- "alert_rules": {"id", "enabled", "scope", "scope_id", "target_role", "require_alertable", "metric", "operator", "threshold", "severity", "notify_channels", "notify_recovery"}, ++ "alert_rules": {"id", "enabled", "scope", "scope_id", "target_role", "require_alertable", "metric", "operator", "threshold", "severity", "notify_channels", "notify_recovery", "quarantine_reason", "quarantined_at"}, + "alerts": {"id", "rule_id", "device_id", "status", "triggered_at", "resolved_at", "notified_at", "notify_error", "recovery_notified_at", "recovery_notify_error"}, + "alert_states": {"rule_id", "device_id", "first_triggered_at", "last_value", "last_checked_at", "notified"}, + "alert_notification_outbox": {"id", "alert_id", "channel", "event", "payload", "status", "attempts", "next_attempt_at", "last_error", "updated_at", "sent_at"}, +diff --git a/internal/alerting/manager.go b/internal/alerting/manager.go +index fcdbfe6..aaaaecf 100644 +--- a/internal/alerting/manager.go ++++ b/internal/alerting/manager.go +@@ -50,25 +50,27 @@ const ( + var ErrNotFound = errors.New("not found") + + type Rule struct { +- ID int `json:"id"` +- Name string `json:"name"` +- Enabled bool `json:"enabled"` +- Scope string `json:"scope"` +- ScopeID *int `json:"scope_id,omitempty"` +- TargetRole string `json:"target_role"` +- RequireAlertable bool `json:"require_alertable"` +- Metric string `json:"metric"` +- Operator string `json:"operator"` +- Threshold float64 `json:"threshold"` +- DurationSeconds int `json:"duration_seconds"` +- Severity string `json:"severity"` +- NotifyChannels []string `json:"notify_channels"` +- NotifyEmails []string `json:"notify_emails,omitempty"` +- WebhookURL string `json:"webhook_url,omitempty"` +- NotifyRecovery bool `json:"notify_recovery"` +- CooldownSeconds int `json:"cooldown_seconds"` +- CreatedAt time.Time `json:"created_at"` +- CreatedBy int `json:"created_by,omitempty"` ++ ID int `json:"id"` ++ Name string `json:"name"` ++ Enabled bool `json:"enabled"` ++ Scope string `json:"scope"` ++ ScopeID *int `json:"scope_id,omitempty"` ++ TargetRole string `json:"target_role"` ++ RequireAlertable bool `json:"require_alertable"` ++ Metric string `json:"metric"` ++ Operator string `json:"operator"` ++ Threshold float64 `json:"threshold"` ++ DurationSeconds int `json:"duration_seconds"` ++ Severity string `json:"severity"` ++ NotifyChannels []string `json:"notify_channels"` ++ NotifyEmails []string `json:"notify_emails,omitempty"` ++ WebhookURL string `json:"webhook_url,omitempty"` ++ NotifyRecovery bool `json:"notify_recovery"` ++ CooldownSeconds int `json:"cooldown_seconds"` ++ CreatedAt time.Time `json:"created_at"` ++ CreatedBy int `json:"created_by,omitempty"` ++ QuarantineReason string `json:"quarantine_reason,omitempty"` ++ QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` + } + + type Alert struct { +@@ -167,18 +169,25 @@ func (m *Manager) Start(ctx context.Context) { + go m.sysmonClient.Run(ctx) + } + ++type invalidEnabledRule struct { ++ rule Rule ++ err error ++} ++ + func (m *Manager) loadRules() ([]Rule, error) { + rows, err := m.db.Query(` +- SELECT id, name, enabled, scope, scope_id, target_role, require_alertable, metric, operator, threshold, +- duration_seconds, severity, notify_channels, notify_emails, webhook_url, notify_recovery, cooldown_seconds ++ SELECT id, name, enabled, COALESCE(scope, 'all'), scope_id, target_role, require_alertable, metric, operator, threshold, ++ COALESCE(duration_seconds, 0), severity, COALESCE(notify_channels, ARRAY[]::TEXT[]), ++ \ No newline at end of file diff --git a/.github/alert-rule-quarantine-patch/part-01 b/.github/alert-rule-quarantine-patch/part-01 new file mode 100644 index 0000000..1528c43 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-01 @@ -0,0 +1,109 @@ + COALESCE(notify_emails, ARRAY[]::TEXT[]), webhook_url, notify_recovery, COALESCE(cooldown_seconds, 0) + FROM alert_rules WHERE enabled = true ++ ORDER BY id + `) + if err != nil { + return nil, err + } +- defer rows.Close() + + var rules []Rule ++ var invalidRules []invalidEnabledRule + for rows.Next() { + var r Rule + var scopeID sql.NullInt64 +@@ -187,6 +196,7 @@ func (m *Manager) loadRules() ([]Rule, error) { + if err := rows.Scan(&r.ID, &r.Name, &r.Enabled, &r.Scope, &scopeID, &r.TargetRole, &r.RequireAlertable, + &r.Metric, &r.Operator, &r.Threshold, &r.DurationSeconds, &r.Severity, &channels, &emails, &webhookURL, + &r.NotifyRecovery, &r.CooldownSeconds); err != nil { ++ rows.Close() + return nil, err + } + if scopeID.Valid { +@@ -200,14 +210,84 @@ func (m *Manager) loadRules() ([]Rule, error) { + } + normalizeRule(&r) + if err := ValidateRule(&r); err != nil { +- return nil, fmt.Errorf("enabled alert rule %d is invalid: %w", r.ID, err) ++ invalidRules = append(invalidRules, invalidEnabledRule{rule: r, err: err}) ++ continue + } + rules = append(rules, r) + } +- return rules, rows.Err() ++ if err := rows.Err(); err != nil { ++ rows.Close() ++ return nil, err ++ } ++ if err := rows.Close(); err != nil { ++ return nil, err ++ } ++ ++ wakeNotifications := false ++ for _, invalid := range invalidRules { + enqueued, err := m.quarantineInvalidRule(context.Background(), invalid.rule, invalid.err) + if err != nil { + // Bad user data must not take the monitoring daemon down. Keep the + // rule out of memory and make the persistence failure unmissable in + // the service log so an operator can repair the database manually. + log.Printf("alert rule %d (%q) is invalid and was skipped, but could not be quarantined: %v (validation error: %v)", + invalid.rule.ID, invalid.rule.Name, err, invalid.err) + continue + } + wakeNotifications = wakeNotifications || enqueued + log.Printf("alert rule %d (%q) was automatically disabled: %v", invalid.rule.ID, invalid.rule.Name, invalid.err) + } + if wakeNotifications { + m.wakeNotificationWorker() + } + + return rules, nil +} + +func (m *Manager) quarantineInvalidRule(ctx context.Context, rule Rule, validationErr error) (bool, error) { + if rule.ID <= 0 { + return false, fmt.Errorf("invalid rule id %d", rule.ID) + } + reason := truncateError(fmt.Sprintf("Automatically disabled by WaveControl because the rule is invalid: %v", validationErr), 2048) + tx, err := m.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer tx.Rollback() + + result, err := tx.ExecContext(ctx, ` + UPDATE alert_rules + SET enabled=false, quarantine_reason=$2, quarantined_at=NOW(), updated_at=NOW() + WHERE id=$1 AND enabled=true + `, rule.ID, reason) + if err != nil { + return false, err + } + affected, err := result.RowsAffected() + if err != nil { + return false, err + } + if affected == 0 { ++ return false, tx.Commit() ++ } ++ ++ // Close any occurrence created by the old rule through the normal durable ++ // resolution path. This cancels undelivered triggers and sends a matching ++ // recovery to channels that may already have received the trigger. ++ enqueued, err := m.resolveRuleAlertsTx(ctx, tx, rule, reason) ++ if err != nil { ++ return false, err ++ } ++ if _, err := tx.ExecContext(ctx, `DDELETE FROM alert_states WHERE rule_id=$1`, rule.ID); err != nil { ++ return false, err ++ } ++ if err := tx.Commit(); err != nil { ++ return false, err ++ } ++ return enqueued, nil + } + +-func (m *Manager) loadStates() (map[string]*AlertState, error) { ++func (m *Manager) loadStates(activeRuleIDs map[int]struct{}) (map[string]*AlertState, error) { + rows, err := m.db.Query(` + SELECT s. \ No newline at end of file diff --git a/.github/alert-rule-quarantine-patch/part-02 b/.github/alert-rule-quarantine-patch/part-02 new file mode 100644 index 0000000..fb80559 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-02 @@ -0,0 +1,129 @@ +oldown state: %w", err) + } +@@ -1023,13 +1113,16 @@ func (m *Manager) GetRule(id int) (Rule, error) { + var r Rule + var scopeID sql.NullInt64 + var channels, emails pq.StringArray +- var webhook sql.NullString ++ var webhook, quarantineReason sql.NullString ++ var quarantinedAt sql.NullTime + err := m.db.QueryRow(` + SELECT id,name,enabled,scope,scope_id,target_role,require_alertable,metric,operator,threshold, +- duration_seconds,severity,notify_channels,notify_emails,webhook_url,notify_recovery,cooldown_seconds,created_at,COALESCE(created_by,0) ++ duration_seconds,severity,notify_channels,notify_emails,webhook_url,notify_recovery,cooldown_seconds,created_at, ++ COALESCE(created_by,0),quarantine_reason,quarantined_at + FROM alert_rules WHERE id=$1 + `, id).Scan(&r.ID, &r.Name, &r.Enabled, &r.Scope, &scopeID, &r.TargetRole, &r.RequireAlertable, &r.Metric, +- &r.Operator, &r.Threshold, &r.DurationSeconds, &r.Severity, &channels, &emails, &webhook, &r.NotifyRecovery, &r.CooldownSeconds, &r.CreatedAt, &r.CreatedBy) ++ &r.Operator, &r.Threshold, &r.DurationSeconds, &r.Severity, &channels, &emails, &webhook, &r.NotifyRecovery, ++ &r.CooldownSeconds, &r.CreatedAt, &r.CreatedBy, &quarantineReason, &quarantinedAt) + if errors.Is(err, sql.ErrNoRows) { + return r, ErrNotFound + } +@@ -1045,6 +1138,13 @@ func (m *Manager) GetRule(id int) (Rule, error) { + if webhook.Valid { + r.WebhookURL = webhook.String + } ++ if quarantineReason.Valid { ++ r.QuarantineReason = quarantineReason.String ++ } ++ if quarantinedAt.Valid { ++ t := quarantinedAt.Time ++ r.QuarantinedAt = &t ++ } + normalizeRule(&r) + return r, nil + } +@@ -1076,7 +1176,8 @@ func (m *Manager) UpdateRule(id int, rule *Rule) error { + if _, err := tx.Exec(` + UPDATE alert_rules SET name=$1,enabled=$2,scope=$3,scope_id=$4,target_role=$5,require_alertable=$6, + metric=$7,operator=$8,threshold=$9,duration_seconds=$10,severity=$11,notify_channels=$12,notify_emails=$13, +- webhook_url=NULLIF($14,''),notify_recovery=$15,cooldown_seconds=$16,updated_at=NOW() WHERE id=$17 ++ webhook_url=NULLIF($14,''),notify_recovery=$15,cooldown_seconds=$16, ++ quarantine_reason=NULL,quarantined_at=NULL,updated_at=NOW() WHERE id=$17 + `, rule.Name, rule.Enabled, rule.Scope, rule.ScopeID, rule.TargetRole, rule.RequireAlertable, rule.Metric, rule.Operator, + rule.Threshold, rule.DurationSeconds, rule.Severity, pq.Array(rule.NotifyChannels), pq.Array(rule.NotifyEmails), rule.WebhookURL, + rule.NotifyRecovery, rule.CooldownSeconds, id); err != nil { +@@ -1128,7 +1229,8 @@ func (m *Manager) DeleteRule(id int) error { + func (m *Manager) ListRules() ([]Rule, error) { + rows, err := m.db.Query(` + SELECT id,name,enabled,scope,scope_id,target_role,require_alertable,metric,operator,threshold, +- duration_seconds,severity,notify_channels,notify_emails,webhook_url,notify_recovery,cooldown_seconds,created_at ++ duration_seconds,severity,notify_channels,notify_emails,webhook_url,notify_recovery,cooldown_seconds,created_at, ++ quarantine_reason,quarantined_at + FROM alert_rules ORDER BY name + `) + if err != nil { +@@ -1140,10 +1242,11 @@ func (m *Manager) ListRules() ([]Rule, error) { + var r Rule + var scopeID sql.NullInt64 + var channels, emails pq.StringArray +- var webhook sql.NullString ++ var webhook, quarantineReason sql.NullString ++ var quarantinedAt sql.NullTime + if err := rows.Scan(&r.ID, &r.Name, &r.Enabled, &r.Scope, &scopeID, &r.TargetRole, &r.RequireAlertable, + &r.Metric, &r.Operator, &r.Threshold, &r.DurationSeconds, &r.Severity, &channels, &emails, &webhook, +- &r.NotifyRecovery, &r.CooldownSeconds, &r.CreatedAt); err != nil { ++ &r.NotifyRecovery, &r.CooldownSeconds, &r.CreatedAt, &quarantineReason, &quarantinedAt); err != nil { + return nil, err + } + if scopeID.Valid { +@@ -1155,6 +1258,13 @@ func (m *Manager) ListRules() ([]Rule, error) { + if webhook.Valid { + r.WebhookURL = webhook.String + } ++ if quarantineReason.Valid { ++ r.QuarantineReason = quarantineReason.String ++ } ++ if quarantinedAt.Valid { ++ t := quarantinedAt.Time ++ r.QuarantinedAt = &t ++ } + normalizeRule(&r) + rules = append(rules, r) + } +diff --git a/internal/alerting/quarantine_test.go b/internal/alerting/quarantine_test.go +new file mode 100644 +index 0000000..d227141 +--- /dev/null ++++ b/internal/alerting/quarantine_test.go +@@ -0,0 +1,163 @@ ++package alerting ++ ++import ( ++ "context" ++ "database/sql" ++ "database/sql/driver" ++ "errors" ++ "io" ++ "strings" ++ "sync" ++ "testing" ++) ++ ++type quarantineTestConnector struct { ++ conn *quarantineTestConn ++} ++ ++func (c quarantineTestConnector) Connect(context.Context) (driver.Conn, error) { return c.conn, nil } ++func (c quarantineTestConnector) Driver() driver.Driver { return quarantineTestDriver{} } ++ ++type quarantineTestDriver struct{} ++ ++func (quarantineTestDriver) Open(string) (driver.Conn, error) { ++ return nil, errors.New("quarantine test driver must be opened through its connector") ++} ++ ++type quarantineTestConn struct { ++ mu sync.Mutex ++ execs []quarantineTestExec ++ committed bool ++ failExec bool ++} ++ ++type quarantineTestExec struct { ++ query string ++ args []driver.NamedValue ++} ++ ++func (c *quarantineTestConn) Prepare(string) (driver.Stmt, \ No newline at end of file diff --git a/.github/alert-rule-quarantine-patch/part-03 b/.github/alert-rule-quarantine-patch/part-03 new file mode 100644 index 0000000..44161a0 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-03 @@ -0,0 +1,150 @@ + error) { ++ return nil, errors.New("Prepare is not supported by the quarantine test driver") ++} ++func (c *quarantineTestConn) Close() error { return nil } ++func (c *quarantineTestConn) Begin() (driver.Tx, error) { ++ return c.BeginTx(context.Background(), driver.TxOptions{}) ++} ++func (c *quarantineTestConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { ++ return &quarantineTestTx{conn: c}, nil ++} ++func (c *quarantineTestConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) { ++ if strings.Contains(query, "FROM alert_rules WHERE enabled = true") { ++ return &quarantineTestRows{ ++ columns: []string{ ++ "id", "name", "enabled", "scope", "scope_id", "target_role", "require_alertable", "metric", ++ "operator", "threshold", "duration_seconds", "severity", "notify_channels", "notify_emails", ++ "webhook_url", "notify_recovery", "cooldown_seconds", ++ }, ++ values: [][]driver.Value{ ++ {int64(3), "Device down", true, "all", nil, "ap", true, "offline_duration", "gte", float64(180), int64(0), "auto", "{}", "{}", nil, true, int64(900)}, ++ {int64(11), "Peer count dropped", true, "device", nil, "ap", true, "peer_count", "lt", float64(1), int64(120), "auto", "{}", "{}", nil, true, int64(900)}, ++ }, ++ }, nil ++ } ++ if strings.Contains(query, "SELECT id FROM alerts") { ++ return &quarantineTestRows{columns: []string{"id"}}, nil ++ } ++ return nil, errors.New("unexpected quarantine test query") ++} ++func (c *quarantineTestConn) ExecContext(_ context.Context, query string, args []driver.NamedValue) (driver.Result, error) { ++ c.mu.Lock() ++ defer c.mu.Unlock() ++ copied := append([]driver.NamedValue(nil), args...) ++ c.execs = append(c.execs, quarantineTestExec{query: query, args: copied}) ++ if c.failExec { ++ c.failExec = false ++ return nil, errors.New("injected quarantine write failure") ++ } ++ return driver.RowsAffected(1), nil ++} ++ ++type quarantineTestTx struct { ++ conn *quarantineTestConn ++} ++ ++func (tx *quarantineTestTx) Commit() error { ++ tx.conn.mu.Lock() ++ tx.conn.committed = true ++ tx.conn.mu.Unlock() ++ return nil ++} ++func (*quarantineTestTx) Rollback() error { return nil } ++ ++type quarantineTestRows struct { ++ columns []string ++ values [][]driver.Value ++ index int ++} ++ ++func (r *quarantineTestRows) Columns() []string { return r.columns } ++func (r *quarantineTestRows) Close() error { return nil } ++func (r *quarantineTestRows) Next(dest []driver.Value) error { ++ if r.index >= len(r.values) { ++ return io.EOF ++ } ++ copy(dest, r.values[r.index]) ++ r.index++ ++ return nil ++} ++ ++func openQuarantineTestDB(conn *quarantineTestConn) *sql.DB { ++ db := sql.OpenDB(quarantineTestConnector{conn: conn}) ++ db.SetMaxOpenConns(1) ++ return db ++} ++ ++func TestLoadRulesQuarantinesInvalidLegacyRule(t *testing.T) { ++ conn := &quarantineTestConn{} ++ db := openQuarantineTestDB(conn) ++ defer db.Close() ++ ++ manager := &Manager{db: db} ++ rules, err := manager.loadRules() ++ if err != nil { ++ t.Fatalf("loadRules() error = %v", err) ++ } ++ if len(rules) != 1 || rules[0].ID != 3 { ++ t.Fatalf("loadRules() rules = %#v, want only valid rule 3", rules) ++ } ++ ++ conn.mu.Lock() ++ defer conn.mu.Unlock() ++ if !conn.committed { ++ t.Fatal("quarantine transaction was not committed") ++ } ++ if len(conn.execs) != 2 { ++ t.Fatalf("quarantine exec count = %d, want 2", len(conn.execs)) ++ } ++ first := conn.execs[0] ++ if !strings.Contains(first.query, "UPDATE alert_rules") { ++ t.Fatalf("first quarantine query = %q", first.query) ++ } ++ if len(first.args) != 2 || first.args[0].Value != int64(11) { ++ t.Fatalf("first quarantine args = %#v", first.args) ++ } ++ reason, _ := first.args[1].Value.(string) ++ if !strings.Contains(reason, "positive scope_id is required for device scope") { ++ t.Fatalf("quarantine reason = %q", reason) ++ } ++} ++ ++func TestLoadRulesSkipsInvalidRuleWhenQuarantineWriteFails(t *testing.T) { ++ conn := &quarantineTestConn{failExec: true} ++ db := openQuarantineTestDB(conn) ++ defer db.Close() ++ ++ manager := &Manager{db: db} ++ rules, err := manager.loadRules() ++ if err != nil { ++ t.Fatalf("loadRules() error = %v; invalid configuration must not abort startup", err) ++ } ++ if len(rules) != 1 || rules[0].ID != 3 { ++ t.Fatalf("loadRules() rules = %#v, want only valid rule 3", rules) ++ } ++} +diff --git a/migrations/020_alert_rule_quarantine.sql b/migrations/020_alert_rule_quarantine.sql +new file mode 100644 +index 0000000..b39d891 +--- /dev/null ++++ b/migrations/020_alert_rule_quarantine.sql +@@ -0,0 +1,7 @@ ++BEGIN; ++ ++ALTER TABLE alert_rules ++ ADD COLUMN IF NOT EXISTS quarantine_reason TEXT, ++ ADD COLUMN IF NOT EXISTS quarantined_at TIMESTAMPTZ; ++ ++COMMIT; +diff --git a/schema.sql b/schema.sql +index 2b71585..c6addec 100644 +--- a/schema.sql ++++ b/schema.sql +@@ -344,7 +344,9 @@ CREATE TABLE IF NOT EXISTS alert_rules ( + -- Metadata + created_at TIMESTAMPTZ DEFAULT NOW(), + created_by INTEGER REFERENCES users(id) ON DELETE SET NULL, +- updated_at TIMESTAMPTZ DEFAULT NOW() ++ updated_at TIMESTAMPTZ DEFAULT NOW(), ++ quarantine_reason TEXT, -- populated when WaveControl auto-disables invalid legacy configuration ++ quarantined_at TIMESTAMPTZ -- allows the admin UI to call attention to the dis \ No newline at end of file diff --git a/.github/alert-rule-quarantine-patch/part-04 b/.github/alert-rule-quarantine-patch/part-04 new file mode 100644 index 0000000..3b735b3 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-04 @@ -0,0 +1,185 @@ +carded rule + ); + + CREATE INDEX IF NOT EXISTS idx_alert_rules_enabled ON alert_rules(enabled); +diff --git a/web/css/alert-quarantine.css b/web/css/alert-quarantine.css +new file mode 100644 +index 0000000..e35c572 +--- /dev/null ++++ b/web/css/alert-quarantine.css +@@ -0,0 +1,70 @@ ++.nav-alert-rule-warning { ++ display: inline-flex; ++ align-items: center; ++ justify-content: center; ++ min-width: 16px; ++ height: 16px; ++ padding: 0 5px; ++ margin-left: 3px; ++ border-radius: 999px; ++ background: var(--yellow); ++ color: #15110a; ++ font-size: 10px; ++ font-weight: 800; ++ line-height: 1; ++ vertical-align: middle; ++} ++ ++.alert-rule-quarantine-banner, ++.alert-rule-quarantine-editor { ++ display: flex; ++ gap: 0.65rem; ++ margin: 0.75rem 0; ++ padding: 0.75rem 0.85rem; ++ border: 1px solid rgba(var(--yellow-rgb), 0.5); ++ border-radius: 8px; ++ background: rgba(var(--yellow-rgb), 0.1); ++ color: var(--text-1); ++} ++ ++.alert-rule-quarantine-banner, ++.alert-rule-quarantine-editor { ++ flex-direction: column; ++ gap: 0.3rem; ++} ++ ++.alert-rule-quarantine-banner strong, ++.alert-rule-quarantine-editor strong { ++ color: var(--yellow); ++} ++ ++.alert-rule-quarantine-banner p, ++.alert-rule-quarantine-editor p { ++ margin: 0; ++ color: var(--text-2); ++ line-height: 1.4; ++} ++ ++.alert-rule-quarantined { ++ background: rgba(var(--yellow-rgb), 0.045); ++} ++ ++.alert-rule-quarantine-detail { ++ display: flex; ++ flex-direction: column; ++ gap: 0.2rem; ++ max-width: 360px; ++ margin-top: 0.35rem; ++ color: var(--yellow); ++ line-height: 1.35; ++} ++ ++.alert-rule-quarantine-detail small { ++ color: var(--text-2); ++ overflow-wrap: anywhere; ++} ++ ++.alert-enabled.quarantined { ++ background: rgba(var(--yellow-rgb), 0.16); ++ color: var(--yellow); ++} +diff --git a/web/index.html b/web/index.html +index c2a59a0..cb4b5bc 100644 +--- a/web/index.html ++++ b/web/index.html +@@ -8,6 +8,7 @@ + + ++ + + + +@@ -468,5 +469,6 @@ + integrity="sha256-20nQCchB9co0qIjJZRGuk2/Z9VM+kNiyxNV1lvTlZBo=" crossorigin="anonymous"> + + ++ + + +diff --git a/web/js/alert-quarantine-ui.js b/web/js/alert-quarantine-ui.js +new file mode 100644 +index 0000000..da01574 +--- /dev/null ++++ b/web/js/alert-quarantine-ui.js +@@ -0,0 +1,294 @@ ++import { api } from './api.js' ++import { store } from './store.js' ++import { showToast } from './components.js' ++ ++const refreshIntervalMs = 60_000 ++const minimumRefreshGapMs = 2_000 ++ ++let cachedRules = [] ++let refreshPromise = null ++let lastRefreshAt = 0 ++let activeRepairRuleID = null ++let toastFingerprint = '' ++let decorationQueued = false ++let decorating = false ++let observer = null ++ ++function isAdministrator() { ++ return (store.user?.roles || []).includes('administrator') ++} ++ ++function quarantinedRules() { ++ return cachedRules.filter(rule => !rule.enabled && String(rule.quarantine_reason || '').trim()) ++} ++ ++function ensureWarningBadge() { ++ const alertsLink = document.querySelector('a[data-page="alerts"]') ++ if (!alertsLink) return null ++ let badge = document.getElementById('alertRuleWarningBadge') ++ if (!badge) { ++ badge = document.createElement('span') ++ badge.id = 'alertRuleWarningBadge' ++ badge.className = 'nav-alert-rule-warning hidden' ++ badge.setAttribute('aria-label', 'Alert rule configuration needs attention') ++ alertsLink.appendChild(badge) ++ } ++ return badge ++} ++ ++function updateWarningBadge() { ++ const badge = ensureWarningBadge() ++ if (!badge) return ++ const rules = isAdministrator() ? quarantinedRules() : [] ++ if (rules.length === 0) { ++ badge.classList.add('hidden') ++ badge.textContent = '' ++ badge.removeAttribute('title') ++ return ++ } ++ badge.textContent = rules.length > 9 ? '9+' : String(rules.length) ++ badge.title = `${rules.length} invalid alert rule${rules.length === 1 ? '' : 's'} automatically disabled` ++ badge.classList.remove('hidden') ++} ++ ++function announceQuarantine() { ++ if (!isAdministrator()) { ++ toastFingerprint = '' ++ return ++ } ++ const rules = quarantinedRules() ++ if (rules.length === 0) { ++ toastFingerprint = '' ++ return ++ } ++ const fingerprint = rules.map(rule => `${rule.id}:${rule.quarantined_at || ''}`).join('|') ++ if (fingerprint === toastFingerprint) return ++ toastFingerprint = fingerprint ++ showToast( ++ `WaveControl disabled ${rules.length} invalid alert rule${rules.length === 1 ? '' : 's'}. Open Alerts to repair or delete ${rules.length === 1 ? 'it' : 'them'}.`, ++ 'warning' ++ ) ++} ++ ++function formatQuarantineTime(value) { ++ if (!value) return '' ++ const date = new Date(value) ++ if (Number.isNaN(date.getTime())) return '' ++ return date.toLocaleString() ++} ++ ++function resetR \ No newline at end of file diff --git a/.github/alert-rule-quarantine-patch/part-05 b/.github/alert-rule-quarantine-patch/part-05 new file mode 100644 index 0000000..0aba4dd --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-05 @@ -0,0 +1,139 @@ +uleDecorations(page) { ++ page.querySelectorAll('[data-alert-quarantine-banner], [data-alert-quarantine-editor], [data-alert-quarantine-detail]') ++ .forEach(element => element.remove()) ++ page.querySelectorAll('tr.alert-rule-quarantined').forEach(row => row.classList.remove('alert-rule-quarantined')) ++ ++ const rulesByID = new Map(cachedRules.map(rule => [String(rule.id), rule])) ++ page.querySelectorAll('[data-alert-rule-edit]').forEach(button => { ++ const rule = rulesByID.get(String(button.dataset.alertRuleEdit || '')) ++ if (!rule) return ++ if (button.dataset.quarantineLabel === 'true') { ++ button.textContent = 'Edit' ++ delete button.dataset.quarantineLabel ++ } ++ const row = button.closest('tr') ++ const status = row?.querySelector('.alert-enabled') ++ if (status?.classList.contains('quarantined')) { ++ status.classList.remove('quarantined') ++ status.classList.toggle('enabled', Boolean(rule.enabled)) ++ status.classList.toggle('disabled', !rule.enabled) ++ status.textContent = rule.enabled ? 'enabled' : 'disabled' ++ } ++ }) ++} ++ ++function makeQuarantineBanner(count) { ++ const banner = document.createElement('div') ++ banner.className = 'alert-rule-quarantine-banner' ++ banner.dataset.alertQuarantineBanner = 'true' ++ banner.setAttribute('role', 'alert') ++ ++ const title = document.createElement('strong') ++ title.textContent = `${count} invalid alert rule${count === 1 ? '' : 's'} automatically disabled` ++ banner.appendChild(title) ++ ++ const explanation = document.createElement('p') ++ explanation.textContent = `WaveControl kept running and excluded ${count === 1 ? 'this rule' : 'these rules'} from evaluation. Repair the highlighted configuration and save it, or delete it if it is obsolete.` ++ banner.appendChild(explanation) ++ return banner ++} ++ ++function addRuleDetail(row, rule) { ++ const nameCell = row.cells?.[0] ++ if (!nameCell) return ++ const detail = document.createElement('div') ++ detail.className = 'alert-rule-quarantine-detail' ++ detail.dataset.alertQuarantineDetail = 'true' ++ ++ const summary = document.createElement('span') ++ const when = formatQuarantineTime(rule.quarantined_at) ++ summary.textContent = `Automatically disabled${when ? ` ${when}` : ''}` ++ detail.appendChild(summary) ++ ++ const reason = document.createElement('small') ++ reason.textContent = String(rule.quarantine_reason || '') ++ detail.appendChild(reason) ++ nameCell.appendChild(detail) ++} ++ ++function addEditorNotice(page, rule) { ++ const form = page.querySelector('#alertRuleForm') ++ if (!form) return ++ const notice = document.createElement('div') ++ notice.className = 'alert-rule-quarantine-editor' ++ notice.dataset.alertQuarantineEditor = 'true' ++ notice.setAttribute('role', 'alert') ++ ++ const title = document.createElement('strong') ++ title.textContent = 'This rule was automatically disabled.' ++ notice.appendChild(title) ++ ++ const reason = document.createElement('p') ++ reason.textContent = String(rule.quarantine_reason || '') ++ notice.appendChild(reason) ++ ++ const action = document.createElement('p') ++ action.textContent = 'Correct the invalid fields below and save the rule. Saving clears this warning.' ++ notice.appendChild(action) ++ ++ const header = form.querySelector('.alert-form-header') ++ if (header?.nextSibling) header.parentNode.insertBefore(notice, header.nextSibling) ++ else form.prepend(notice) ++} ++ ++function decorateAlertsPage() { ++ if (decorating) return ++ decorating = true ++ observer?.disconnect() ++ try { ++ const page = document.querySelector('.alerts-page') ++ if (!page) return ++ resetRuleDecorations(page) ++ if (!isAdministrator()) return ++ ++ const rules = quarantinedRules() ++ if (rules.length === 0) return ++ ++ const tableWrap = page.querySelector('.alert-rules-table-wrap') ++ if (tableWrap) tableWrap.parentNode.insertBefore(makeQuarantineBanner(rules.length), tableWrap) ++ ++ for (const rule of rules) { ++ const button = [...page.querySelectorAll('[data-alert-rule-edit]')] ++ .find(candidate => String(candidate.dataset.alertRuleEdit || '') === String(rule.id)) ++ const row = button?.closest('tr') ++ if (!row) continue ++ row.classList.add('alert-rule-quarantined') ++ addRuleDetail(row, rule) ++ ++ const status = row.querySelector('.alert-enabled') ++ if (status) { ++ status.classList.remove('enabled', 'disabled') ++ status.classList.add('quarantined') ++ status.textContent = 'needs repair' ++ } ++ button.textContent = 'Repair' ++ button.dataset.quarantineLabel = 'true' ++ } ++ ++ const editingRule = rules.find(rule => String(rule.id) === String(activeRepairRuleID)) ++ if (editingRule) addEditorNotice(page, editingRule) ++ } finally { ++ observer?.observe(document.body, { childList: true, subtree: true }) ++ decorating = false ++ } ++} ++ ++function queueDecoration() { ++ if (decorationQueued) return ++ decorationQueued = true ++ requestAnimationFrame(() => { ++ decorationQueued = false ++ decorateAlertsPage() ++ }) ++} ++ ++async function refreshQuarantineState(force = false) { ++ if (!isAdministrator()) { ++ cachedRules = [] ++ activeRepairRuleID = null ++ updateWarningBadge() \ No newline at end of file diff --git a/.github/alert-rule-quarantine-patch/part-06 b/.github/alert-rule-quarantine-patch/part-06 new file mode 100644 index 0000000..28ff661 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/part-06 @@ -0,0 +1,77 @@ + ++ queueDecoration() ++ return ++ } ++ const now = Date.now() ++ if (!force && now - lastRefreshAt < minimumRefreshGapMs) { ++ queueDecoration() ++ return ++ } ++ if (refreshPromise) return refreshPromise ++ ++ refreshPromise = (async () => { ++ try { ++ const rules = await api.alertRules() ++ cachedRules = Array.isArray(rules) ? rules : [] ++ lastRefreshAt = Date.now() ++ if (!cachedRules.some(rule => String(rule.id) === String(activeRepairRuleID) && rule.quarantine_reason)) { ++ activeRepairRuleID = null ++ } ++ updateWarningBadge() ++ announceQuarantine() ++ queueDecoration() ++ } catch (_) { ++ // This is advisory UI. Keep the last known state on a transient API ++ // failure and never interfere with the main Alerts page. ++ } finally { ++ refreshPromise = null ++ } ++ })() ++ return refreshPromise ++} ++ ++store.subscribe((next, previous) => { ++ const userChanged = next.user !== previous.user ++ const pageChanged = next.currentPage !== previous.currentPage ++ if (userChanged || pageChanged) refreshQuarantineState(true) ++ else if (next.currentPage === 'alerts') queueDecoration() ++}) ++ ++document.addEventListener('click', event => { ++ if (!(event.target instanceof Element)) return ++ const repairButton = event.target.closest('[data-alert-rule-edit]') ++ if (repairButton) { ++ const id = String(repairButton.dataset.alertRuleEdit || '') ++ activeRepairRuleID = quarantinedRules().some(rule => String(rule.id) === id) ? id : null ++ queueDecoration() ++ return ++ } ++ if (event.target.closest('#alertNewRule, #alertCancelEdit, [data-alert-preset]')) { ++ activeRepairRuleID = null ++ queueDecoration() ++ return ++ } ++ if (event.target.closest('[data-alert-rule-delete]')) { ++ window.setTimeout(() => refreshQuarantineState(true), 500) ++ } ++}, true) ++ ++document.addEventListener('submit', event => { ++ if (!(event.target instanceof Element) || !event.target.matches('#alertRuleForm')) return ++ window.setTimeout(() => refreshQuarantineState(true), 500) ++ window.setTimeout(() => refreshQuarantineState(true), 1_500) ++}, true) ++ ++window.addEventListener('wavecontrol-auth-failure', () => { ++ cachedRules = [] ++ activeRepairRuleID = null ++ toastFingerprint = '' ++ updateWarningBadge() ++ queueDecoration() ++}) ++ ++observer = new MutationObserver(queueDecoration) ++observer.observe(document.body, { childList: true, subtree: true }) ++ ++window.setInterval(() => refreshQuarantineState(false), refreshIntervalMs) ++refreshQuarantineState(true) diff --git a/.github/workflows/apply-alert-rule-quarantine.yml b/.github/workflows/apply-alert-rule-quarantine.yml new file mode 100644 index 0000000..4d6bf2f --- /dev/null +++ b/.github/workflows/apply-alert-rule-quarantine.yml @@ -0,0 +1,30 @@ +name: Apply alert-rule quarantine patch + +on: + push: + branches: + - fix/alert-rule-quarantine + +permissions: + contents: write + +jobs: + apply: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Apply verified patch + shell: bash + run: | + set -euxo pipefail + cat .github/alert-rule-quarantine-patch/part-* > /tmp/alert-rule-quarantine.diff + echo "5cc05f3458aed7f17c8a3546e0e5903e7f5b3ff03da765f747574e4c857bb8f0 /tmp/alert-rule-quarantine.diff" | sha256sum -c - + git apply --check /tmp/alert-rule-quarantine.diff + git apply --index /tmp/alert-rule-quarantine.diff + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Quarantine invalid legacy alert rules at startup" + git push origin "HEAD:${GITHUB_REF_NAME}" From 2410d552b04684ed235b1bec6ea55e265886046f Mon Sep 17 00:00:00 2001 From: Chris Cappuccio Date: Tue, 1 Sep 2026 12:23:43 -0700 Subject: [PATCH 2/3] Trigger alert-rule quarantine patch application --- .github/alert-rule-quarantine-patch/TRIGGER | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/alert-rule-quarantine-patch/TRIGGER diff --git a/.github/alert-rule-quarantine-patch/TRIGGER b/.github/alert-rule-quarantine-patch/TRIGGER new file mode 100644 index 0000000..5c33b15 --- /dev/null +++ b/.github/alert-rule-quarantine-patch/TRIGGER @@ -0,0 +1 @@ +trigger From 3e147bc4d768a079a951e112d4e1936db075e451 Mon Sep 17 00:00:00 2001 From: Chris Cappuccio Date: Tue, 1 Sep 2026 12:42:52 -0700 Subject: [PATCH 3/3] Finalize and verify alert-rule quarantine fix --- .../finalize-alert-rule-quarantine.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/finalize-alert-rule-quarantine.yml diff --git a/.github/workflows/finalize-alert-rule-quarantine.yml b/.github/workflows/finalize-alert-rule-quarantine.yml new file mode 100644 index 0000000..e1420e9 --- /dev/null +++ b/.github/workflows/finalize-alert-rule-quarantine.yml @@ -0,0 +1,51 @@ +name: Finalize alert-rule quarantine fix + +on: + push: + branches: + - fix/alert-rule-quarantine + +permissions: + contents: write + +jobs: + finalize: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Verify quarantine implementation + shell: bash + run: | + set -euxo pipefail + grep -q 'QuarantineReason' internal/alerting/manager.go + grep -q 'quarantine_reason' cmd/server/schema_optional.go + grep -q 'quarantine_reason' schema.sql + test -f migrations/020_alert_rule_quarantine.sql + grep -q 'quarantine' web/js/app.js + gofmt -w internal/alerting/manager.go internal/alerting/quarantine_test.go cmd/server/schema_optional.go + git diff --check + go test ./... + node --check web/js/app.js + + - name: Remove patch transport and publish verified tree + shell: bash + run: | + set -euxo pipefail + git rm -rf --ignore-unmatch .github/alert-rule-quarantine-patch + git rm -f --ignore-unmatch .github/workflows/apply-alert-rule-quarantine.yml + git rm -f --ignore-unmatch .github/workflows/finalize-alert-rule-quarantine.yml + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + if ! git diff --cached --quiet || ! git diff --quiet; then + git add -A + git commit -m 'Quarantine invalid legacy alert rules at startup' + git push origin "HEAD:${GITHUB_REF_NAME}" + fi