From f6b8a08b9c52a1477ffe6a40445269a2a9fb0272 Mon Sep 17 00:00:00 2001 From: Ricardo Valdes Date: Wed, 16 Sep 2026 20:34:37 -0400 Subject: [PATCH] fix: align Microsoft 365 filter and rule contracts --- filters/audits/office365.md | 42 +++ filters/office365/o365.yml | 62 +++- .../testdata/filter-contracts/office365.json | 284 ++++++++++++++++++ .../anti_phishing_policy_bypasses.yml | 8 +- ...365_potential_password_spraying_attack.yml | 8 +- rules/office365/dlp_policy_violations.yml | 2 +- .../information_barriers_violations.yml | 2 +- ...ible_succesfull_password_guessing_o365.yml | 6 +- rules/office365/safe_links_click_patterns.yml | 2 +- 9 files changed, 397 insertions(+), 19 deletions(-) create mode 100644 filters/audits/office365.md create mode 100644 plugins/alerts/testdata/filter-contracts/office365.json diff --git a/filters/audits/office365.md b/filters/audits/office365.md new file mode 100644 index 000000000..01e2b4d0e --- /dev/null +++ b/filters/audits/office365.md @@ -0,0 +1,42 @@ +# Microsoft 365 normalization and rule review + +Normalize outcomes and compound IP endpoints; repair password-guessing/spray correlation and renamed-field consumers. + +This draft targets UTMStack `v11`. It contains 1 filter changes +and 6 rule changes for this technology only. Review covered +1 filter configurations and 56 matching shipped rule files. +Unchanged rules are listed in the regression manifest; they are not duplicated in the diff. + +## Contract and validation + +- Compared exact standard names/types with go-sdk v1.1.31 and the supplied UTMStack dictionaries. +- Checked documented pipeline ordering, rename/move behavior, open vendor log fields, + event-side versus alert-side fields, and surviving fields used by affected rule predicates/history/grouping. +- Strict SDK configuration decoding and actual CEL compilation pass for this scope. +- 14 synthetic normalization cases pass, including SDK Event conversion and any + trigger predicate assertions recorded in the manifest. +- The scoped alerts module tests and `git diff --check` pass with the shared contract runner applied. + +The shared alert-contract PR supplies the reusable Go runner for the manifest in +`plugins/alerts/testdata/filter-contracts/office365.json`. Apply that support before running `go test ./...` in `plugins/alerts`. + +The changed rules also require the shared alert-grouping fix to resolve `lastEvent.*` values correctly at runtime. + +The model starts from synthetic extraction results. It does not run complex grok, +JSON/KV/XML/CSV extraction, time conversion, dynamic plugins, historical OpenSearch +queries, or the closed EventProcessor. Raw vendor logs and resulting alerts must +still be checked in staging before rollout. No customer false-positive reduction +has been measured and no production rollout is included. + +Historical queries were reviewed against the fields produced by this filter. Trigger predicates are replayed; actual OpenSearch window/count execution and customer alert volumes still require staging validation. + +## References + +- [SDK schema](https://github.com/threatwinds/go-sdk/blob/v1.1.31/plugins/plugins.proto) +- [Filter steps](https://github.com/threatwinds/go-sdk/wiki/Filter-Steps-Reference) +- [Standard event schema](https://github.com/threatwinds/go-sdk/wiki/Standard-Event-Schema) +- [Rule implementation](https://github.com/threatwinds/go-sdk/wiki/Implementing-Rules) + +`afterEvents`, empty noncapturing grok names, supported numeric strings, and custom +`log.*` fields are accepted. Existing textual protocol casing and vendor action names +are preserved unless a concrete consumer mismatch requires correction. diff --git a/filters/office365/o365.yml b/filters/office365/o365.yml index 4e4986df6..ea5bbc8af 100755 --- a/filters/office365/o365.yml +++ b/filters/office365/o365.yml @@ -95,26 +95,57 @@ pipeline: # Adding action result - add: - function: 'string' + function: string params: key: actionResult - value: 'success' - where: oneOf("log.ResultStatus", ["Succeeded", "Success", "Successful", "PartiallySucceeded", "True"]) + value: success + where: oneOf("log.ResultStatus", ["Succeeded", "Success", "Successful", "True"]) - add: - function: 'string' + function: string params: key: actionResult - value: 'failed' + value: failure where: oneOf("log.ResultStatus", ["Failure", "Failed", "False"]) - add: - function: 'string' + function: string params: key: actionResult - value: 'blocked' + value: denied where: equals("log.ResultStatus", "Blocked") + # ClientIP may include a port; retain the reported endpoint for auditing. + - grok: + source: origin.ip + patterns: + - fieldName: log.originEndpoint + pattern: '{{.greedy}}' + where: regexMatch("origin.ip", "^([0-9]{1,3}[.]){3}[0-9]{1,3}:[0-9]+$") || regexMatch("origin.ip", "^\\[[0-9a-fA-F:]+\\]:[0-9]+$") + - grok: + source: log.originEndpoint + patterns: + - fieldName: log.originEndpointPort + pattern: '{{.greedy}}' + where: exists("log.originEndpoint") + - trim: + fields: + - log.originEndpointPort + function: regex + substring: '^.*:' + where: exists("log.originEndpoint") + - trim: + fields: + - origin.ip + function: regex + substring: ^\[|\]?:[0-9]+$ + where: exists("log.originEndpoint") + - rename: + from: + - log.originEndpointPort + to: origin.port + where: exists("log.originEndpoint") && (inCIDR("origin.ip","0.0.0.0/0") || inCIDR("origin.ip","::/0")) && lessOrEqual("log.originEndpointPort",65535) + # Adding geolocation to origin ip - dynamic: plugin: com.utmstack.geolocation @@ -130,4 +161,19 @@ pipeline: # Removing unused fields - delete: fields: - - log.AppAccessContext \ No newline at end of file + - log.AppAccessContext + + # Normalize explicit outcomes; an unknown outcome remains unset. + - add: + function: string + params: + key: actionResult + value: failure + where: equals("action", "UserLoginFailed") + + # Keep addresses in IP fields and retain other source values under log. + - rename: + from: + - origin.ip + to: log.unparsedOriginIp + where: exists("origin.ip") && (!(inCIDR("origin.ip","0.0.0.0/0") || inCIDR("origin.ip","::/0")) || oneOf("origin.ip",["0.0.0.0","::"])) diff --git a/plugins/alerts/testdata/filter-contracts/office365.json b/plugins/alerts/testdata/filter-contracts/office365.json new file mode 100644 index 000000000..ec62452bc --- /dev/null +++ b/plugins/alerts/testdata/filter-contracts/office365.json @@ -0,0 +1,284 @@ +{ + "technology": "Microsoft 365", + "filters": [ + "filters/office365/o365.yml" + ], + "rules": [ + "rules/office365/anti_phishing_policy_bypasses.yml", + "rules/office365/app_consent_grants.yml", + "rules/office365/audit_log_tampering.yml", + "rules/office365/azure_ad_integration_events.yml", + "rules/office365/collection_microsoft_365_new_inbox_rule.yml", + "rules/office365/compliance_alert_patterns.yml", + "rules/office365/conditional_access_bypasses.yml", + "rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml", + "rules/office365/credential_access_user_excessive_sso_logon_errors.yml", + "rules/office365/defense_evasion_microsoft_365_exchange_malware_filter_policy_deletion.yml", + "rules/office365/defense_evasion_microsoft_365_exchange_malware_filter_rule_mod.yml", + "rules/office365/defense_evasion_microsoft_365_exchange_safe_attach_rule_disabled.yml", + "rules/office365/detection_of_a_connection_to_onion_domain.yml", + "rules/office365/dlp_policy_violations.yml", + "rules/office365/ediscovery_abuse.yml", + "rules/office365/exchange_admin_changes.yml", + "rules/office365/exfiltration_microsoft_365_exchange_transport_rule_creation.yml", + "rules/office365/exfiltration_microsoft_365_exchange_transport_rule_mod.yml", + "rules/office365/external_sharing_violations.yml", + "rules/office365/forms_sway_phishing.yml", + "rules/office365/guest_user_invitation_spikes.yml", + "rules/office365/information_barriers_violations.yml", + "rules/office365/initial_access_microsoft_365_exchange_anti_phish_policy_deletion.yml", + "rules/office365/initial_access_microsoft_365_exchange_anti_phish_rule_mod.yml", + "rules/office365/initial_access_microsoft_365_exchange_safelinks_disabled.yml", + "rules/office365/insider_risk_indicators.yml", + "rules/office365/m365_data_exfiltration_unsanctioned.yml", + "rules/office365/m365_federated_domain_added.yml", + "rules/office365/m365_oauth_app_file_downloads.yml", + "rules/office365/m365_pst_export.yml", + "rules/office365/mail_flow_rule_changes.yml", + "rules/office365/mail_forwarding_rules.yml", + "rules/office365/mailbox_delegation_abuse.yml", + "rules/office365/mailbox_export_pst.yml", + "rules/office365/mass_email_deletion.yml", + "rules/office365/mfa_fatigue_push_spam.yml", + "rules/office365/multi_geo_data_violations.yml", + "rules/office365/o365-admin-role-assignment.yml", + "rules/office365/o365-admin-role-granted.yml", + "rules/office365/o365-audit-log-purge.yml", + "rules/office365/o365-inbox-forward-rule-exfiltration.yml", + "rules/office365/oauth_app_anomalies.yml", + "rules/office365/onedrive_mass_file_access.yml", + "rules/office365/possible_succesfull_password_guessing_o365.yml", + "rules/office365/power_apps_data_leaks.yml", + "rules/office365/power_automate_abuse.yml", + "rules/office365/power_bi_data_export.yml", + "rules/office365/safe_attachment_violations.yml", + "rules/office365/safe_links_click_patterns.yml", + "rules/office365/service_principal_credential_addition.yml", + "rules/office365/sharepoint_mass_downloads.yml", + "rules/office365/sharepoint_permission_escalation.yml", + "rules/office365/suspicious_inbox_rules.yml", + "rules/office365/teams_data_exfiltration.yml", + "rules/office365/teams_external_user_abuse.yml", + "rules/office365/threat_intelligence_alerts.yml" + ], + "fixtures": [ + { + "name": "O365 result Failed", + "filter": "office365/o365.yml", + "input": { + "log": { + "ResultStatus": "Failed" + } + }, + "expected": { + "actionResult": "failure" + }, + "absent": [], + "rules": {} + }, + { + "name": "O365 result Blocked", + "filter": "office365/o365.yml", + "input": { + "log": { + "ResultStatus": "Blocked" + } + }, + "expected": { + "actionResult": "denied" + }, + "absent": [], + "rules": {} + }, + { + "name": "O365 result Succeeded", + "filter": "office365/o365.yml", + "input": { + "log": { + "ResultStatus": "Succeeded" + } + }, + "expected": { + "actionResult": "success" + }, + "absent": [], + "rules": {} + }, + { + "name": "O365 result PartiallySucceeded", + "filter": "office365/o365.yml", + "input": { + "log": { + "ResultStatus": "PartiallySucceeded" + } + }, + "expected": {}, + "absent": [ + "actionResult" + ], + "rules": {} + }, + { + "name": "O365 failed operation wins over audit Success", + "filter": "office365/o365.yml", + "input": { + "log": { + "ResultStatus": "Success", + "Operation": "UserLoginFailed", + "Workload": "AzureActiveDirectory", + "ClientIP": "198.51.100.10" + } + }, + "expected": { + "actionResult": "failure" + }, + "absent": [], + "rules": { + "rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml": true + } + }, + { + "name": "O365 IPv6 [2001:db8::10]:443", + "filter": "office365/o365.yml", + "input": { + "log": { + "ClientIP": "[2001:db8::10]:443" + } + }, + "expected": { + "origin.ip": "2001:db8::10", + "origin.port": 443 + }, + "absent": [] + }, + { + "name": "O365 IPv6 2001:db8::10", + "filter": "office365/o365.yml", + "input": { + "log": { + "ClientIP": "2001:db8::10" + } + }, + "expected": { + "origin.ip": "2001:db8::10" + }, + "absent": [ + "origin.port" + ] + }, + { + "name": "O365 unknown endpoint retained", + "filter": "office365/o365.yml", + "input": { + "log": { + "ClientIP": "not-an-ip" + } + }, + "expected": { + "log.unparsedOriginIp": "not-an-ip" + }, + "absent": [ + "origin.ip" + ] + }, + { + "name": "O365 port range checked", + "filter": "office365/o365.yml", + "input": { + "log": { + "ClientIP": "203.0.113.10:99999" + } + }, + "expected": { + "origin.ip": "203.0.113.10" + }, + "absent": [ + "origin.port" + ] + }, + { + "name": "O365 anti-phish policy Success", + "filter": "office365/o365.yml", + "input": { + "log": { + "Operation": "Set-AntiPhishPolicy", + "ResultStatus": "Success" + } + }, + "expected": {}, + "absent": [], + "rules": { + "rules/office365/anti_phishing_policy_bypasses.yml": true + } + }, + { + "name": "O365 anti-phish policy Failed", + "filter": "office365/o365.yml", + "input": { + "log": { + "Operation": "Set-AntiPhishPolicy", + "ResultStatus": "Failed" + } + }, + "expected": {}, + "absent": [], + "rules": { + "rules/office365/anti_phishing_policy_bypasses.yml": false + } + }, + { + "name": "O365 successful guessing trigger UserLoggedIn Success", + "filter": "office365/o365.yml", + "input": { + "log": { + "Operation": "UserLoggedIn", + "ResultStatus": "Success", + "Workload": "AzureActiveDirectory", + "UserId": "alice@example.test", + "ClientIP": "203.0.113.10" + } + }, + "expected": {}, + "absent": [], + "rules": { + "rules/office365/possible_succesfull_password_guessing_o365.yml": true + } + }, + { + "name": "O365 successful guessing trigger UserLoginFailed Failed", + "filter": "office365/o365.yml", + "input": { + "log": { + "Operation": "UserLoginFailed", + "ResultStatus": "Failed", + "Workload": "AzureActiveDirectory", + "UserId": "alice@example.test", + "ClientIP": "203.0.113.10" + } + }, + "expected": {}, + "absent": [], + "rules": { + "rules/office365/possible_succesfull_password_guessing_o365.yml": false + } + }, + { + "name": "O365 successful guessing trigger UserLoggedIn Failed", + "filter": "office365/o365.yml", + "input": { + "log": { + "Operation": "UserLoggedIn", + "ResultStatus": "Failed", + "Workload": "AzureActiveDirectory", + "UserId": "alice@example.test", + "ClientIP": "203.0.113.10" + } + }, + "expected": {}, + "absent": [], + "rules": { + "rules/office365/possible_succesfull_password_guessing_o365.yml": false + } + } + ] +} diff --git a/rules/office365/anti_phishing_policy_bypasses.yml b/rules/office365/anti_phishing_policy_bypasses.yml index f0d808347..d3fc21106 100644 --- a/rules/office365/anti_phishing_policy_bypasses.yml +++ b/rules/office365/anti_phishing_policy_bypasses.yml @@ -25,19 +25,19 @@ description: | 6. Consider rolling back unauthorized changes and implementing additional approval workflows 7. Monitor for any unusual email activity following the policy modifications where: | - oneOf("log.Operation", ["Set-AntiPhishPolicy", "Remove-AntiPhishPolicy", "New-AntiPhishPolicy", "Disable-AntiPhishRule"]) && - equals("log.ResultStatus", "Success") + oneOf("action", ["Set-AntiPhishPolicy", "Remove-AntiPhishPolicy", "New-AntiPhishPolicy", "Disable-AntiPhishRule"]) && + equals("actionResult", "success") afterEvents: - indexPattern: v11-log-o365-* with: - field: origin.user operator: filter_term value: '{{.origin.user}}' - - field: log.Operation + - field: action operator: filter_match value: 'AntiPhish' within: 4h count: 2 groupBy: - - lastEvent.log.Operation + - lastEvent.action - adversary.user diff --git a/rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml b/rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml index db5985922..907877deb 100644 --- a/rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml +++ b/rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml @@ -16,13 +16,19 @@ references: description: "Credential Access consists of techniques for stealing credentials like account names and passwords. Techniques used to get credentials include keylogging or credential dumping. Using legitimate credentials can give adversaries access to systems, make them harder to detect, and provide the opportunity to create more accounts to help achieve their goals.
Identifies a high number (25) of failed Microsoft 365 user authentication attempts from a single IP address within 30 minutes, which could be indicative of a password spraying attack. An adversary may attempt a password spraying attack to obtain unauthorized access to user accounts." where: | - oneOf("log.Workload", ["Exchange", "AzureActiveDirectory"]) && oneOf("action", ["UserLoginFailed", "PasswordLogonInitialAuthUsingPassword"]) && equals("actionResult", "failed") && exists("origin.ip") + oneOf("log.Workload", ["Exchange", "AzureActiveDirectory"]) && oneOf("action", ["UserLoginFailed", "PasswordLogonInitialAuthUsingPassword"]) && oneOf("actionResult", ["failure", "failed"]) && exists("origin.ip") afterEvents: - indexPattern: v11-log-o365-* with: - field: origin.ip operator: filter_term value: '{{.origin.ip}}' + - field: action + operator: filter_term + value: '{{.action}}' + - field: actionResult + operator: filter_term + value: '{{.actionResult}}' within: 60s count: 5 groupBy: diff --git a/rules/office365/dlp_policy_violations.yml b/rules/office365/dlp_policy_violations.yml index c6ec30033..e506dc8a3 100644 --- a/rules/office365/dlp_policy_violations.yml +++ b/rules/office365/dlp_policy_violations.yml @@ -41,7 +41,7 @@ where: | equals("log.Workload", "Teams") || equals("log.Workload", "SecurityComplianceCenter") ) && - !equals("actionResult", "failed") + !oneOf("actionResult", ["failure", "failed"]) groupBy: - lastEvent.log.PolicyId - lastEvent.log.SensitiveInfoTypeData diff --git a/rules/office365/information_barriers_violations.yml b/rules/office365/information_barriers_violations.yml index ef3b66437..b49e7c080 100644 --- a/rules/office365/information_barriers_violations.yml +++ b/rules/office365/information_barriers_violations.yml @@ -24,7 +24,7 @@ description: | 5. Provide additional training to users if violations appear to be due to lack of awareness 6. Consider implementing additional technical controls to prevent future violations where: | - equals("action", "InformationBarrierPolicyViolation") || (equals("log.PolicyType", "InformationBarrier") && equals("actionResult", "blocked")) || (equals("log.ViolationType", "InformationBarrier") && equals("action", "CommunicationBlocked")) + equals("action", "InformationBarrierPolicyViolation") || (equals("log.PolicyType", "InformationBarrier") && oneOf("actionResult", ["denied", "blocked"])) || (equals("log.ViolationType", "InformationBarrier") && equals("action", "CommunicationBlocked")) afterEvents: - indexPattern: v11-log-o365-* with: diff --git a/rules/office365/possible_succesfull_password_guessing_o365.yml b/rules/office365/possible_succesfull_password_guessing_o365.yml index bc9bd6fd8..af391169b 100644 --- a/rules/office365/possible_succesfull_password_guessing_o365.yml +++ b/rules/office365/possible_succesfull_password_guessing_o365.yml @@ -21,7 +21,7 @@ references: - "https://attack.mitre.org/tactics/TA0006" - "https://attack.mitre.org/techniques/T1110/001/" where: | - oneOf("log.Workload", ["Exchange","AzureActiveDirectory"]) && equals("action", "UserLoginFailed") && equals("actionResult", "failed") && exists("origin.user") && exists("log.clientIP") + oneOf("log.Workload", ["Exchange","AzureActiveDirectory"]) && equals("action", "UserLoggedIn") && equals("actionResult", "success") && exists("origin.user") && exists("origin.ip") afterEvents: - indexPattern: v11-log-o365-* with: @@ -31,9 +31,9 @@ afterEvents: - field: origin.user operator: filter_term value: "{{.origin.user}}" - - field: log.clientIP + - field: origin.ip operator: filter_term - value: "{{.log.clientIP}}" + value: "{{.origin.ip}}" within: 1m count: 10 groupBy: diff --git a/rules/office365/safe_links_click_patterns.yml b/rules/office365/safe_links_click_patterns.yml index 5ee89a314..a7adc641f 100644 --- a/rules/office365/safe_links_click_patterns.yml +++ b/rules/office365/safe_links_click_patterns.yml @@ -24,7 +24,7 @@ description: | 5. Implement additional security awareness training for the affected user 6. Consider blocking the malicious domains at the network level where: | - equals("action", "ClickedSafeLink") && equals("actionResult", "blocked") && exists("origin.user") + equals("action", "ClickedSafeLink") && oneOf("actionResult", ["denied", "blocked"]) && exists("origin.user") afterEvents: - indexPattern: v11-log-o365-* with: