diff --git a/filters/audits/azure.md b/filters/audits/azure.md new file mode 100644 index 000000000..73963fee8 --- /dev/null +++ b/filters/audits/azure.md @@ -0,0 +1,40 @@ +# Azure Event Hub normalization and rule review + +Fix SDK configuration and field meanings; normalize result precedence, IPs, severity and HTTP counters. + +This draft targets UTMStack `v11`. It contains 1 filter changes +and 0 rule changes for this technology only. Review covered +1 filter configurations and 46 matching shipped rule files. +Unchanged rules are listed in the regression manifest; they are not duplicated in the diff. + +## Contract and validation + +- Compared exact standard names/types with go-sdk v1.1.31 and the supplied UTMStack dictionaries. +- Checked documented pipeline ordering, rename/move behavior, open vendor log fields, + event-side versus alert-side fields, and surviving fields used by affected rule predicates/history/grouping. +- Strict SDK configuration decoding and actual CEL compilation pass for this scope. +- 11 synthetic normalization cases pass, including SDK Event conversion and any + trigger predicate assertions recorded in the manifest. +- The scoped alerts module tests and `git diff --check` pass with the shared contract runner applied. + +The shared alert-contract PR supplies the reusable Go runner for the manifest in +`plugins/alerts/testdata/filter-contracts/azure.json`. Apply that support before running `go test ./...` in `plugins/alerts`. + +The model starts from synthetic extraction results. It does not run complex grok, +JSON/KV/XML/CSV extraction, time conversion, dynamic plugins, historical OpenSearch +queries, or the closed EventProcessor. Raw vendor logs and resulting alerts must +still be checked in staging before rollout. No customer false-positive reduction +has been measured and no production rollout is included. + + + +## References + +- [SDK schema](https://github.com/threatwinds/go-sdk/blob/v1.1.31/plugins/plugins.proto) +- [Filter steps](https://github.com/threatwinds/go-sdk/wiki/Filter-Steps-Reference) +- [Standard event schema](https://github.com/threatwinds/go-sdk/wiki/Standard-Event-Schema) +- [Rule implementation](https://github.com/threatwinds/go-sdk/wiki/Implementing-Rules) + +`afterEvents`, empty noncapturing grok names, supported numeric strings, and custom +`log.*` fields are accepted. Existing textual protocol casing and vendor action names +are preserved unless a concrete consumer mismatch requires correction. diff --git a/filters/azure/azure-eventhub.yml b/filters/azure/azure-eventhub.yml index 460c25a22..777ef3e1c 100644 --- a/filters/azure/azure-eventhub.yml +++ b/filters/azure/azure-eventhub.yml @@ -7,7 +7,7 @@ # 4- https://learn.microsoft.com/en-us/azure/event-grid/system-topics # pipeline: - - dataTypes: azure + - dataTypes: [azure] steps: - json: source: raw @@ -17,13 +17,13 @@ pipeline: # .......................................................................# - rename: from: - - log.ResponseBodySize - to: origin.bytesSent + - log.ResponseBodySize + to: origin.bytesReceived - rename: from: - - log.ResponseHeaderSize - to: origin.bytesReceived + - log.ResponseHeaderSize + to: log.responseHeaderSize - rename: from: @@ -32,8 +32,8 @@ pipeline: - rename: from: - - log.AccountName - to: origin.host + - log.AccountName + to: log.accountName - rename: from: @@ -47,13 +47,13 @@ pipeline: - rename: from: - - log.StatusText - to: connectionStatus + - log.StatusText + to: log.statusText - rename: from: - - log.Location - to: origin.geolocation.country + - log.Location + to: log.location - rename: from: @@ -795,23 +795,31 @@ pipeline: - log.properties.upstreamSourcePort to: log.propertiesUpstreamSourcePort + # Common Azure resource-log result (kept under log for existing consumers). + - grok: + source: log.resultType + patterns: + - fieldName: log.vendorActionResult + pattern: '{{.greedy}}' + where: exists("log.resultType") + - rename: from: - - log.properties.result - to: actionResult - where: '!regexMatch("log.properties.result", "(?i)\b(?:denied|blocked|failed)\b")' + - log.properties.result + to: log.vendorActionResult + where: '!exists("log.vendorActionResult") && exists("log.properties.result")' - rename: from: - - log.Properties.Result - to: actionResult - where: '!regexMatch("log.properties.result", "(?i)\b(?:denied|blocked|failed)\b")' + - log.Properties.Result + to: log.vendorActionResult + where: '!exists("log.vendorActionResult") && exists("log.Properties.Result")' - rename: from: - - log.properties.resultType - to: actionResult - where: '!regexMatch("log.properties.result", "(?i)\b(?:denied|blocked|failed)\b")' + - log.properties.resultType + to: log.vendorActionResult + where: '!exists("log.vendorActionResult") && exists("log.properties.resultType")' - rename: from: @@ -984,10 +992,10 @@ pipeline: where: '!exists("actionResult") && equalsIgnoreCase("log.data.status", "Succeeded")' - add: - function: 'string' + function: string params: key: actionResult - value: 'failed' + value: failure where: '!exists("actionResult") && equalsIgnoreCase("log.data.status", "Failed")' - add: @@ -998,25 +1006,19 @@ pipeline: where: '!exists("actionResult") && endsWith("log.eventGridEventType", "Success")' - add: - function: 'string' + function: string params: key: actionResult - value: 'failed' + value: failure where: '!exists("actionResult") && endsWith("log.eventGridEventType", "Failure")' - - add: - function: 'string' - params: - key: actionResult - value: 'blocked' - where: '!exists("actionResult") && endsWith("log.eventGridEventType", "Cancel")' - add: - function: 'string' + function: string params: key: severity - value: 'medium' - where: '!exists("severity") && oneOf("actionResult", ["failed", "blocked"])' + value: warning + where: '!exists("severity") && oneOf("actionResult", ["failure", "denied"])' - rename: from: @@ -1046,46 +1048,46 @@ pipeline: # Adding action result - add: - function: 'string' + function: string params: key: actionResult - value: 'denied' - where: regexMatch("log.properties.result", "(?i)\bdenied\b") + value: denied + where: regexMatch("log.vendorActionResult", "(?i)\\bdenied\\b") - add: - function: 'string' + function: string params: key: actionResult - value: 'blocked' - where: regexMatch("log.properties.result", "(?i)\bblocked\b") + value: denied + where: regexMatch("log.vendorActionResult", "(?i)\\bblocked\\b") - add: - function: 'string' + function: string params: key: actionResult - value: 'failed' - where: regexMatch("log.properties.result", "(?i)\bfailed\b") + value: failure + where: regexMatch("log.vendorActionResult", "(?i)\\bfailed\\b") # .......................................................................# # Adding severity based on log.level # .......................................................................# - add: - function: 'string' + function: string params: key: severity - value: 'high' + value: error where: oneOf("log.level", ["ERROR", "Error", "FATAL", "CRITICAL", "Critical"]) - add: - function: 'string' + function: string params: key: severity - value: 'medium' + value: warning where: oneOf("log.level", ["WARN", "Warning"]) - add: - function: 'string' + function: string params: key: severity - value: 'low' + value: info where: oneOf("log.level", ["Information", "Informational", "INFO", "DEBUG", "TRACE"]) # .......................................................................# @@ -1163,4 +1165,98 @@ pipeline: fields: - origin.bytesSent - origin.bytesReceived - to: float \ No newline at end of file + to: float + + # Normalize the source event severity. + - add: + function: string + params: + key: severity + value: critical + where: oneOf("log.level",["FATAL","CRITICAL","Critical"]) + - add: + function: string + params: + key: severity + value: debug + where: oneOf("log.level",["DEBUG","TRACE"]) + + # Normalize explicit outcomes; an unknown outcome remains unset. + - add: + function: string + params: + key: actionResult + value: success + where: '!exists("actionResult") && regexMatch("log.vendorActionResult", "(?i)^(success|succeeded|successful|ok|done|accepted|accept|allowed|allow|permitted|permit|passed|pass|true)$")' + - add: + function: string + params: + key: actionResult + value: failure + where: '!equals("actionResult","denied") && regexMatch("log.vendorActionResult", "(?i)^(failure|failed|fail|error|invalid|timeout|false)$")' + - add: + function: string + params: + key: actionResult + value: denied + where: regexMatch("log.vendorActionResult", "(?i)^(denied|deny|blocked|block|dropped|drop|rejected|reject|forbidden|unauthorized|quarantined)$") + + # Keep addresses in IP fields and retain other source values under log. + - rename: + from: + - origin.ip + to: log.unparsedOriginIp + where: exists("origin.ip") && (!(inCIDR("origin.ip","0.0.0.0/0") || inCIDR("origin.ip","::/0")) || oneOf("origin.ip",["0.0.0.0","::"])) + - add: + function: string + params: + key: protocol + value: ICMP + where: equals("protocol",1) + - add: + function: string + params: + key: protocol + value: TCP + where: equals("protocol",6) + - add: + function: string + params: + key: protocol + value: UDP + where: equals("protocol",17) + - add: + function: string + params: + key: protocol + value: GRE + where: equals("protocol",47) + - add: + function: string + params: + key: protocol + value: ESP + where: equals("protocol",50) + - add: + function: string + params: + key: protocol + value: AH + where: equals("protocol",51) + - add: + function: string + params: + key: protocol + value: ICMPV6 + where: equals("protocol",58) + - add: + function: string + params: + key: protocol + value: SCTP + where: equals("protocol",132) + - rename: + from: + - protocol + to: log.ipProtocolNumber + where: exists("protocol") && greaterOrEqual("protocol",0) diff --git a/plugins/alerts/testdata/filter-contracts/azure.json b/plugins/alerts/testdata/filter-contracts/azure.json new file mode 100644 index 000000000..9e5fc31c4 --- /dev/null +++ b/plugins/alerts/testdata/filter-contracts/azure.json @@ -0,0 +1,224 @@ +{ + "technology": "Azure Event Hub", + "filters": [ + "filters/azure/azure-eventhub.yml" + ], + "rules": [ + "rules/cloud/azure/aks_security_threats.yml", + "rules/cloud/azure/app_registration_abuse.yml", + "rules/cloud/azure/application_gateway_waf_alerts.yml", + "rules/cloud/azure/automation_runbook_abuse.yml", + "rules/cloud/azure/azure_ad_impossible_travel.yml", + "rules/cloud/azure/azure_ad_password_spray.yml", + "rules/cloud/azure/azure_alert_suppression_rule.yml", + "rules/cloud/azure/azure_anomalous_token.yml", + "rules/cloud/azure/azure_app_credential_added.yml", + "rules/cloud/azure/azure_app_privileged_permissions.yml", + "rules/cloud/azure/azure_azurehound_discovery.yml", + "rules/cloud/azure/azure_bulk_role_changes.yml", + "rules/cloud/azure/azure_device_code_auth_abuse.yml", + "rules/cloud/azure/azure_disk_snapshot_exfiltration.yml", + "rules/cloud/azure/azure_federation_modified.yml", + "rules/cloud/azure/azure_impossible_travel.yml", + "rules/cloud/azure/azure_kubernetes_admission_controller.yml", + "rules/cloud/azure/azure_kubernetes_events_deleted.yml", + "rules/cloud/azure/azure_kubernetes_secret_access.yml", + "rules/cloud/azure/azure_laps_credential_dump.yml", + "rules/cloud/azure/azure_leaked_credentials.yml", + "rules/cloud/azure/azure_new_root_ca_added.yml", + "rules/cloud/azure/azure_password_spray_detected.yml", + "rules/cloud/azure/azure_prt_access_attempt.yml", + "rules/cloud/azure/azure_ropc_authentication.yml", + "rules/cloud/azure/azure_sentinel_alert_patterns.yml", + "rules/cloud/azure/azure_subscription_permission_elevation.yml", + "rules/cloud/azure/azure_temporary_access_pass.yml", + "rules/cloud/azure/credential_access_key_vault_modified.yml", + "rules/cloud/azure/defender_cloud_critical_alerts.yml", + "rules/cloud/azure/defense_evasion_azure_application_credential_modification.yml", + "rules/cloud/azure/defense_evasion_azure_diagnostic_settings_deletion.yml", + "rules/cloud/azure/defense_evasion_event_hub_deletion.yml", + "rules/cloud/azure/diagnostic_settings_tampering.yml", + "rules/cloud/azure/golden_saml_federation_abuse.yml", + "rules/cloud/azure/impact_azure_service_principal_credentials_added.yml", + "rules/cloud/azure/initial_access_azure_active_directory_high_risk_signin.yml", + "rules/cloud/azure/initial_access_consent_grant_attack_via_azure_registered_application.yml", + "rules/cloud/azure/key_vault_access_spikes.yml", + "rules/cloud/azure/managed_identity_abuse.yml", + "rules/cloud/azure/mfa_disabled_privileged_users.yml", + "rules/cloud/azure/persistence_azure_pim_user_added_global_admin.yml", + "rules/cloud/azure/persistence_mfa_disabled_for_azure_user.yml", + "rules/cloud/azure/pim_role_activation_abuse.yml", + "rules/cloud/azure/storage_account_public_access.yml", + "rules/cloud/azure/subscription_ownership_transfer.yml" + ], + "fixtures": [ + { + "name": "Azure actual result Denied", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "properties": { + "result": "Denied" + } + } + }, + "expected": { + "actionResult": "denied" + }, + "absent": [], + "rules": {} + }, + { + "name": "Azure actual result Failed", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "properties": { + "result": "Failed" + } + } + }, + "expected": { + "actionResult": "failure" + }, + "absent": [], + "rules": {} + }, + { + "name": "Azure actual result Succeeded", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "properties": { + "result": "Succeeded" + } + } + }, + "expected": { + "actionResult": "success" + }, + "absent": [], + "rules": {} + }, + { + "name": "Azure actual result Running", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "properties": { + "result": "Running" + } + } + }, + "expected": {}, + "absent": [ + "actionResult" + ], + "rules": {} + }, + { + "name": "Azure storage metadata is not host geography", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "AccountName": "storageaccount", + "Location": "eastus", + "StatusText": "Success", + "ResponseBodySize": 100, + "ResponseHeaderSize": 20 + } + }, + "expected": { + "origin.bytesReceived": 100, + "log.accountName": "storageaccount", + "log.location": "eastus" + }, + "absent": [ + "origin.host", + "origin.geolocation", + "connectionStatus", + "origin.bytesSent" + ], + "rules": {} + }, + { + "name": "Azure result Failed", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "resultType": "Failed" + } + }, + "expected": { + "actionResult": "failure" + }, + "absent": [] + }, + { + "name": "Azure result Succeeded", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "resultType": "Succeeded" + } + }, + "expected": { + "actionResult": "success" + }, + "absent": [] + }, + { + "name": "Azure result Denied", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "resultType": "Denied" + } + }, + "expected": { + "actionResult": "denied" + }, + "absent": [] + }, + { + "name": "Azure result Started", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "resultType": "Started" + } + }, + "expected": {}, + "absent": [ + "actionResult" + ] + }, + { + "name": "Azure descriptive denial", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "properties": { + "result": "Access denied" + } + } + }, + "expected": { + "actionResult": "denied" + } + }, + { + "name": "Azure explicit failure overrides success event suffix", + "filter": "azure/azure-eventhub.yml", + "input": { + "log": { + "resultType": "Failed", + "eventType": "Microsoft.ResourceWriteSuccess" + } + }, + "expected": { + "actionResult": "failure" + } + } + ] +}