[FR] Add enforcement for deprecated_reason (#5953)

Author: Mikaayenson

.github/PULL_REQUEST_GUIDELINES/rule_deprecation_guidelines.md

@@ -9,8 +9,9 @@ These guidelines serve as a reminder set of considerations when recommending the
 
 ### Rule Metadata Checks
 
-- [ ] `deprecated = true` added to the rule metadata.
-- [ ] `updated_date` should be the date of the PR.
+- [ ] `maturity = "deprecated"` added to the rule metadata.
+- [ ] `deprecation_date` set to the date of the PR and `updated_date` matches.
+- [ ] `deprecated_reason` added to `[metadata]` with a short explanation (e.g. `"Replaced by <rule name>"`). Required in the same PR that flips `maturity = "deprecated"`; surfaced in Kibana on stacks >= 9.4.
 
 ### Testing and Validation
 

detection_rules/etc/deprecated_rules.json

@@ -1,6 +1,7 @@
 {
   "015cca13-8832-49ac-a01b-a396114809f6": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "CreateCluster is routine Redshift lifecycle noise; real abuse paths (snapshot sharing, role abuse, security group exposure) are covered by other rules. See PR elastic/detection-rules#5367.",
     "rule_name": "Deprecated - AWS Redshift Cluster Creation",
     "stack_version": "8.19"
   },
@@ -21,6 +22,7 @@
   },
   "09443c92-46b3-45a4-8f25-383b028b258d": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Expensive Defend correlation from a generic process event; flagged for deprecation as a noisy edge case during top-noisy rule tuning. See PR elastic/detection-rules#5449.",
     "rule_name": "Deprecated - Process Termination followed by Deletion",
     "stack_version": "8.19"
   },
@@ -71,6 +73,7 @@
   },
   "1ba5160d-f5a2-4624-b0ff-6a1dc55d2516": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "ElastiCache CacheSecurityGroup APIs apply only to retired EC2-Classic; modern VPC deployments are covered by AWS EC2 Security Group Configuration Change. See PR elastic/detection-rules#5334.",
     "rule_name": "Deprecated - AWS ElastiCache Security Group Modified or Deleted",
     "stack_version": "8.19"
   },
@@ -81,6 +84,7 @@
   },
   "1defdd62-cd8d-426e-a246-81a37751bb2b": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Windows High Severity tuning batch for persistent false positives. See PR elastic/detection-rules#5094.",
     "rule_name": "Deprecated - Execution of File Written or Modified by PDF Reader",
     "stack_version": "8.19"
   },
@@ -121,11 +125,13 @@
   },
   "30e1e9f2-eb9c-439f-aff6-1e3068e99384": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux privilege-escalation DR tuning batch. See PR elastic/detection-rules#5511.",
     "rule_name": "Deprecated - Network Connection via Sudo Binary",
     "stack_version": "8.19"
   },
   "3115bd2c-0baa-4df0-80ea-45e474b5ef93": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Query keyed on an undocumented, likely-invalid field value; the false positives could not be solved at the rule level. See PR elastic/detection-rules#5552.",
     "rule_name": "Deprecated - Agent Spoofing - Mismatched Agent ID",
     "stack_version": "8.19"
   },
@@ -136,6 +142,7 @@
   },
   "378f9024-8a0c-46a5-aa08-ce147ac73a4e": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "CreateDBSecurityGroup targets retired EC2-Classic; VPC security group changes are covered by AWS EC2 Security Group Configuration Change. See PR elastic/detection-rules#5350.",
     "rule_name": "Deprecated - AWS RDS Security Group Creation",
     "stack_version": "8.19"
   },
@@ -176,6 +183,7 @@
   },
   "521fbe5c-a78d-4b6b-a323-f978b0e4c4c0": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Superseded by new ESQL Linux brute-force rules during the credential-access DR tuning rework. See PR elastic/detection-rules#5483.",
     "rule_name": "Deprecated - Potential Successful Linux RDP Brute Force Attack Detected",
     "stack_version": "8.19"
   },
@@ -186,6 +194,7 @@
   },
   "5c50ffa6-07f4-4cce-a1b7-c16928a2ed52": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux lateral-movement DR tuning batch, with updated triage guidance attached. See PR elastic/detection-rules#5505.",
     "rule_name": "Deprecated - SSH Process Launched From Inside A Container via Elastic Defend",
     "stack_version": "8.19"
   },
@@ -201,6 +210,7 @@
   },
   "62b68eb2-1e47-4da7-85b6-8f478db5b272": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Linux BBR tuning: marked deprecated as a noisy rule with zero useful hits. See PR elastic/detection-rules#5514.",
     "rule_name": "Deprecated - Potential Non-Standard Port HTTP/HTTPS connection",
     "stack_version": "8.19"
   },
@@ -211,6 +221,7 @@
   },
   "66712812-e7f2-4a1d-bbda-dd0b5cf20c5d": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Superseded by new ESQL Linux brute-force rules during the credential-access DR tuning rework. See PR elastic/detection-rules#5483.",
     "rule_name": "Deprecated - Potential Successful Linux FTP Brute Force Attack Detected",
     "stack_version": "8.19"
   },
@@ -261,6 +272,7 @@
   },
   "7b3da11a-60a2-412e-8aa7-011e1eb9ed47": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "ElastiCache CacheSecurityGroup APIs apply only to retired EC2-Classic; modern VPC deployments are covered by AWS EC2 Security Group Configuration Change. See PR elastic/detection-rules#5334.",
     "rule_name": "Deprecated - AWS ElastiCache Security Group Created",
     "stack_version": "8.19"
   },
@@ -281,6 +293,7 @@
   },
   "863cdf31-7fd3-41cf-a185-681237ea277b": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "DeleteDBSecurityGroup targets retired EC2-Classic; modern VPC RDS deployments are covered by AWS EC2 Security Group Configuration Change. See PR elastic/detection-rules#5350.",
     "rule_name": "Deprecated - AWS RDS Security Group Deletion",
     "stack_version": "8.19"
   },
@@ -316,11 +329,13 @@
   },
   "93f47b6f-5728-4004-ba00-625083b3dcb0": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Superseded by Pluggable Authentication Module or Configuration Creation, a Linux-only higher-fidelity, lower-compute rule. See PR elastic/detection-rules#5421.",
     "rule_name": "Deprecated - Modification of Standard Authentication Module or Configuration",
     "stack_version": "8.19"
   },
   "947827c6-9ed6-4dec-903e-c856c86e72f3": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Linux BBR tuning: marked deprecated as a noisy rule with zero useful hits. See PR elastic/detection-rules#5514.",
     "rule_name": "Deprecated - Creation of Kernel Module",
     "stack_version": "8.19"
   },
@@ -351,6 +366,7 @@
   },
   "9d19ece6-c20e-481a-90c5-ccca596537de": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Superseded by Launch Service Creation and Immediate Loading, which covers LaunchDaemons and LaunchAgents via the newer Persistence event. See PR elastic/detection-rules#4547.",
     "rule_name": "Deprecated - LaunchDaemon Creation or Modification and Immediate Loading",
     "stack_version": "8.19"
   },
@@ -361,6 +377,7 @@
   },
   "a577e524-c2ee-47bd-9c5b-e917d01d3276": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Linux BBR tuning: marked deprecated as a noisy rule with zero useful hits. See PR elastic/detection-rules#5514.",
     "rule_name": "Deprecated - CAP_SYS_ADMIN Assigned to Binary",
     "stack_version": "8.19"
   },
@@ -376,6 +393,7 @@
   },
   "ac8805f6-1e08-406c-962e-3937057fa86f": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux DR Tuning - 2 batch without a rule-specific justification recorded in the PR. See PR elastic/detection-rules#5481.",
     "rule_name": "Deprecated - Potential Protocol Tunneling via Chisel Server",
     "stack_version": "8.19"
   },
@@ -391,21 +409,25 @@
   },
   "bc0c6f0d-dab0-47a3-b135-0925f0a333bc": {
     "deprecation_date": "2025/11/21",
+    "deprecated_reason": "Overlaps with the broader AWS Successful Root Console Login rule; the broader rule covers all root logins and is retained. See PR elastic/detection-rules#5201.",
     "rule_name": "Deprecated - AWS Root Login Without MFA",
     "stack_version": "8.19"
   },
   "bc8ca7e0-92fd-4b7c-b11e-ee0266b8d9c9": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux cross-platform DR tuning batch. See PR elastic/detection-rules#5512.",
     "rule_name": "Deprecated - Potential Non-Standard Port SSH connection",
     "stack_version": "8.19"
   },
   "bdb04043-f0e3-4efa-bdee-7d9d13fa9edc": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux discovery DR tuning batch. See PR elastic/detection-rules#5497.",
     "rule_name": "Deprecated - Potential Pspy Process Monitoring Detected",
     "stack_version": "8.19"
   },
   "c125e48f-6783-41f0-b100-c3bf1b114d16": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Superseded by Suspicious Renaming of ESXI VMware Files, which now also detects index.html renames in /usr/lib/vmware/. See PR elastic/detection-rules#5494.",
     "rule_name": "Deprecated - Suspicious Renaming of ESXI index.html File",
     "stack_version": "8.19"
   },
@@ -451,6 +473,7 @@
   },
   "d55436a8-719c-445f-92c4-c113ff2f9ba5": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux privilege-escalation DR tuning batch. See PR elastic/detection-rules#5511.",
     "rule_name": "Deprecated - Potential Privilege Escalation via UID INT_MAX Bug Detected",
     "stack_version": "8.19"
   },
@@ -486,6 +509,7 @@
   },
   "e14c5fd7-fdd7-49c2-9e5b-ec49d817bc8d": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "CreateDBCluster is routine RDS lifecycle with no meaningful attack signal; high-value RDS threats (snapshot, export, exposure) are covered elsewhere. See PR elastic/detection-rules#5350.",
     "rule_name": "Deprecated - AWS RDS Cluster Creation",
     "stack_version": "8.19"
   },
@@ -496,6 +520,7 @@
   },
   "e919611d-6b6f-493b-8314-7ed6ac2e413b": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "Replaced by AWS EC2 Export Task, which detects successful exports (higher signal than failed attempts). See PR elastic/detection-rules#5248.",
     "rule_name": "Deprecated - AWS EC2 VM Export Failure",
     "stack_version": "8.19"
   },
@@ -516,6 +541,7 @@
   },
   "ecf2b32c-e221-4bd4-aa3b-c7d59b3bc01d": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "StopDBInstance and StopDBCluster are routine admin operations with no meaningful attack signal. See PR elastic/detection-rules#5350.",
     "rule_name": "Deprecated - AWS RDS Instance/Cluster Stoppage",
     "stack_version": "8.19"
   },
@@ -526,11 +552,13 @@
   },
   "f30f3443-4fbb-4c27-ab89-c3ad49d62315": {
     "deprecation_date": "2026/01/16",
+    "deprecated_reason": "CreateDBInstance is routine RDS lifecycle with no meaningful attack signal; high-value RDS threats are covered elsewhere. See PR elastic/detection-rules#5350.",
     "rule_name": "Deprecated - AWS RDS Instance Creation",
     "stack_version": "8.19"
   },
   "f41296b4-9975-44d6-9486-514c6f635b2d": {
     "deprecation_date": "2026/02/04",
+    "deprecated_reason": "Marked deprecated during the Linux execution DR tuning batch without a rule-specific justification recorded in the PR. See PR elastic/detection-rules#5504.",
     "rule_name": "Deprecated - Potential curl CVE-2023-38545 Exploitation",
     "stack_version": "8.19"
   },
@@ -554,4 +582,4 @@
     "rule_name": "Linux Restricted Shell Breakout via the expect command",
     "stack_version": "7.16"
   }
-}
+}

detection_rules/packaging.py

@@ -554,7 +554,7 @@ def _generate_registry_package(self, save_dir: Path) -> None:
                     deprecated_reason=dep_entry.deprecated_reason,
                     stack_version=stack_version,
                 )
-                asset_path = rules_dir / f"deprecated_{asset['id']}.json"
+                asset_path = rules_dir / f"{asset['id']}.json"
                 asset_path.write_text(json.dumps(asset, indent=4, sort_keys=True), encoding="utf-8")
 
         notice_contents = NOTICE_FILE.read_text()

docs-dev/deprecating.md

@@ -13,9 +13,11 @@ release package to Kibana.
 1. Update the `maturity` to `deprecated`
 2. Move the rule file to [rules/_deprecated](../rules/_deprecated)
 3. Add `deprecation_date` and update `updated_date` to match
+4. Add `deprecated_reason` in `[metadata]` with a short explanation (e.g. "Replaced by <rule name>"). Required in the
+   same PR that flips `maturity = "deprecated"`; surfaced in Kibana on stacks >= 9.4 and ignored on older stacks.
 
 Next time the versions are locked, the rule will be added to the [deprecated_rules.json](../detection_rules/etc/deprecated_rules.json)
-file.
+file, and `deprecated_reason` is copied into the package asset (gated at build time by `MIN_STACK_VERSION_DEPRECATED_STUBS`).
 
 
 ### Using the deprecate-rule command

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.6.24"
+version = "1.6.25"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

tests/test_all_rules.py

@@ -761,6 +761,30 @@ def test_deprecated_rules(self):
             rule_str = f"{rule_id} - {entry['rule_name']} ->"
             self.assertIn(rule_id, deprecated_rules, f'{rule_str} is logged in "deprecated_rules.json" but is missing')
 
+    @unittest.skipIf(RULES_CONFIG.bypass_version_lock, "Skipping deprecated version lock check")
+    def test_newly_deprecated_rules_have_reason(self):
+        """Newly deprecated rules must include `deprecated_reason` in [metadata].
+
+        Rules already in `deprecated_rules.json` are grandfathered.
+        """
+        already_deprecated = set(self.rules_config.deprecated_rules)
+        missing: list[str] = []
+
+        for rule in self.deprecated_rules:
+            if rule.id in already_deprecated:
+                continue
+            if not rule.contents.metadata.get("deprecated_reason"):
+                missing.append(self.rule_str(rule))
+
+        if missing:
+            rules_str = "\n  ".join(missing)
+            self.fail(
+                "The following newly deprecated rules are missing `deprecated_reason` in "
+                "[metadata]. Add a short explanation (e.g. 'Replaced by <rule name>'). This "
+                "field is only required for NEW deprecations on this branch; rules already "
+                f"tracked in `deprecated_rules.json` are grandfathered.\n  {rules_str}"
+            )
+
     def test_deprecated_rules_modified(self):
         """Test to ensure deprecated rules are not modified."""