diff --git a/filters/audits/netflow.md b/filters/audits/netflow.md new file mode 100644 index 000000000..f44623031 --- /dev/null +++ b/filters/audits/netflow.md @@ -0,0 +1,40 @@ +# NetFlow normalization and rule review + +Preserve counters consumed by detection rules, accept extracted counter variants, and align IP/protocol normalization. + +This draft targets UTMStack `v11`. It contains 1 filter changes +and 4 rule changes for this technology only. Review covered +1 filter configurations and 12 matching shipped rule files. +Unchanged rules are listed in the regression manifest; they are not duplicated in the diff. + +## Contract and validation + +- Compared exact standard names/types with go-sdk v1.1.31 and the supplied UTMStack dictionaries. +- Checked documented pipeline ordering, rename/move behavior, open vendor log fields, + event-side versus alert-side fields, and surviving fields used by affected rule predicates/history/grouping. +- Strict SDK configuration decoding and actual CEL compilation pass for this scope. +- 6 synthetic normalization cases pass, including SDK Event conversion and any + trigger predicate assertions recorded in the manifest. +- The scoped alerts module tests and `git diff --check` pass with the shared contract runner applied. + +The shared alert-contract PR supplies the reusable Go runner for the manifest in +`plugins/alerts/testdata/filter-contracts/netflow.json`. Apply that support before running `go test ./...` in `plugins/alerts`. + +The model starts from synthetic extraction results. It does not run complex grok, +JSON/KV/XML/CSV extraction, time conversion, dynamic plugins, historical OpenSearch +queries, or the closed EventProcessor. Raw vendor logs and resulting alerts must +still be checked in staging before rollout. No customer false-positive reduction +has been measured and no production rollout is included. + +The filter previously deleted log.bytes/log.packets that four rules consume. Keep scalar counters and accept the existing extracted totalBytes/totalPackets variants; do not invent totals for multiple-value records. + +## References + +- [SDK schema](https://github.com/threatwinds/go-sdk/blob/v1.1.31/plugins/plugins.proto) +- [Filter steps](https://github.com/threatwinds/go-sdk/wiki/Filter-Steps-Reference) +- [Standard event schema](https://github.com/threatwinds/go-sdk/wiki/Standard-Event-Schema) +- [Rule implementation](https://github.com/threatwinds/go-sdk/wiki/Implementing-Rules) + +`afterEvents`, empty noncapturing grok names, supported numeric strings, and custom +`log.*` fields are accepted. Existing textual protocol casing and vendor action names +are preserved unless a concrete consumer mismatch requires correction. diff --git a/filters/netflow/netflow.yml b/filters/netflow/netflow.yml index 402618f72..9af5816eb 100644 --- a/filters/netflow/netflow.yml +++ b/filters/netflow/netflow.yml @@ -1135,8 +1135,6 @@ pipeline: # Removing unused fields - delete: fields: - - log.bytes - - log.packets - log.inEth - log.outEth - log.srcPort @@ -1159,4 +1157,69 @@ pipeline: - log.irrelevant8 - log.irrelevant9 - log.irrelevant10 - - log.irrelevant11 \ No newline at end of file + - log.irrelevant11 + + # Keep addresses in IP fields and retain other source values under log. + - rename: + from: + - origin.ip + to: log.unparsedOriginIp + where: exists("origin.ip") && (!(inCIDR("origin.ip","0.0.0.0/0") || inCIDR("origin.ip","::/0")) || oneOf("origin.ip",["0.0.0.0","::"])) + - rename: + from: + - target.ip + to: log.unparsedTargetIp + where: exists("target.ip") && (!(inCIDR("target.ip","0.0.0.0/0") || inCIDR("target.ip","::/0")) || oneOf("target.ip",["0.0.0.0","::"])) + - add: + function: string + params: + key: protocol + value: ICMP + where: equals("protocol",1) + - add: + function: string + params: + key: protocol + value: TCP + where: equals("protocol",6) + - add: + function: string + params: + key: protocol + value: UDP + where: equals("protocol",17) + - add: + function: string + params: + key: protocol + value: GRE + where: equals("protocol",47) + - add: + function: string + params: + key: protocol + value: ESP + where: equals("protocol",50) + - add: + function: string + params: + key: protocol + value: AH + where: equals("protocol",51) + - add: + function: string + params: + key: protocol + value: ICMPV6 + where: equals("protocol",58) + - add: + function: string + params: + key: protocol + value: SCTP + where: equals("protocol",132) + - rename: + from: + - protocol + to: log.ipProtocolNumber + where: exists("protocol") && greaterOrEqual("protocol",0) diff --git a/plugins/alerts/testdata/filter-contracts/netflow.json b/plugins/alerts/testdata/filter-contracts/netflow.json new file mode 100644 index 000000000..9cadd8ff3 --- /dev/null +++ b/plugins/alerts/testdata/filter-contracts/netflow.json @@ -0,0 +1,138 @@ +{ + "technology": "NetFlow", + "filters": [ + "filters/netflow/netflow.yml" + ], + "rules": [ + "rules/netflow/beaconing_behavior_detection.yml", + "rules/netflow/data_exfiltration_indicators.yml", + "rules/netflow/ddos_traffic_patterns.yml", + "rules/netflow/netflow_cryptomining_traffic.yml", + "rules/netflow/netflow_doh_detection.yml", + "rules/netflow/netflow_icmp_tunnel.yml", + "rules/netflow/netflow_internal_scanning.yml", + "rules/netflow/netflow_lateral_movement_smb_rdp.yml", + "rules/netflow/netflow_long_duration_connections.yml", + "rules/netflow/netflow_vpn_unusual_destinations.yml", + "rules/netflow/port_scanning_patterns.yml", + "rules/netflow/tor_usage_detection.yml" + ], + "fixtures": [ + { + "name": "netflow_valid_source_ip", + "filter": "netflow/netflow.yml", + "input": { + "origin": { + "ip": "2001:db8::7" + } + }, + "expected": { + "origin.ip": "2001:db8::7" + }, + "absent": [], + "rules": {} + }, + { + "name": "netflow_hostname_not_ip", + "filter": "netflow/netflow.yml", + "input": { + "origin": { + "ip": "lab-host" + } + }, + "expected": { + "log.unparsedOriginIp": "lab-host" + }, + "absent": [ + "origin.ip" + ], + "rules": {} + }, + { + "name": "netflow_numeric_protocol", + "filter": "netflow/netflow.yml", + "input": { + "protocol": 6 + }, + "expected": { + "protocol": "TCP" + }, + "absent": [], + "rules": {} + }, + { + "name": "netflow_tor_scalar_counters", + "filter": "netflow/netflow.yml", + "input": { + "origin": { + "ip": "192.0.2.5" + }, + "target": { + "ip": "198.51.100.9", + "port": 9001 + }, + "log": { + "protocol": "6", + "bytes": 2048 + } + }, + "expected": { + "log.bytes": 2048, + "protocol": "TCP" + }, + "absent": [], + "rules": { + "rules/netflow/tor_usage_detection.yml": true + } + }, + { + "name": "netflow_tor_extracted_counters", + "filter": "netflow/netflow.yml", + "input": { + "origin": { + "ip": "192.0.2.5" + }, + "target": { + "ip": "198.51.100.9", + "port": 9001 + }, + "log": { + "protocol": "6", + "totalBytes": "2048]\"" + } + }, + "expected": { + "log.totalBytes": "2048", + "protocol": "TCP" + }, + "absent": [], + "rules": { + "rules/netflow/tor_usage_detection.yml": true + } + }, + { + "name": "netflow_tor_small_flow", + "filter": "netflow/netflow.yml", + "input": { + "origin": { + "ip": "192.0.2.5" + }, + "target": { + "ip": "198.51.100.9", + "port": 9001 + }, + "log": { + "protocol": "6", + "bytes": 1 + } + }, + "expected": { + "log.bytes": 1 + }, + "absent": [], + "rules": { + "rules/netflow/tor_usage_detection.yml": false + } + } + ] +} diff --git a/rules/netflow/netflow_icmp_tunnel.yml b/rules/netflow/netflow_icmp_tunnel.yml index c54b505da..506fbce41 100644 --- a/rules/netflow/netflow_icmp_tunnel.yml +++ b/rules/netflow/netflow_icmp_tunnel.yml @@ -27,7 +27,7 @@ where: | exists("origin.ip") && exists("target.ip") && equals("protocol", "ICMP") && - greaterThan("log.bytes", 10000) + (greaterThan("log.bytes", 10000) || greaterThan("log.totalBytes", 10000)) afterEvents: - indexPattern: v11-log-netflow-* with: diff --git a/rules/netflow/netflow_internal_scanning.yml b/rules/netflow/netflow_internal_scanning.yml index 251ef1afc..f06df1cb5 100644 --- a/rules/netflow/netflow_internal_scanning.yml +++ b/rules/netflow/netflow_internal_scanning.yml @@ -28,8 +28,8 @@ where: | exists("target.ip") && (inCIDR("origin.ip", "10.0.0.0/8") || inCIDR("origin.ip", "172.16.0.0/12") || inCIDR("origin.ip", "192.168.0.0/16")) && (inCIDR("target.ip", "10.0.0.0/8") || inCIDR("target.ip", "172.16.0.0/12") || inCIDR("target.ip", "192.168.0.0/16")) && - lessOrEqual("log.packets", 5) && - lessThan("log.bytes", 500) + (lessOrEqual("log.packets", 5) || lessOrEqual("log.totalPackets", 5)) && + (lessThan("log.bytes", 500) || lessThan("log.totalBytes", 500)) afterEvents: - indexPattern: v11-log-netflow-* with: diff --git a/rules/netflow/port_scanning_patterns.yml b/rules/netflow/port_scanning_patterns.yml index 25f50b1fb..2d603202f 100644 --- a/rules/netflow/port_scanning_patterns.yml +++ b/rules/netflow/port_scanning_patterns.yml @@ -26,8 +26,8 @@ description: | where: | exists("origin.ip") && exists("target.ip") && - lessOrEqual("log.packets", 10) && - lessThan("log.bytes", 1000) && + (lessOrEqual("log.packets", 10) || lessOrEqual("log.totalPackets", 10)) && + (lessThan("log.bytes", 1000) || lessThan("log.totalBytes", 1000)) && greaterThan("target.port", 0) && equals("protocol", "TCP") afterEvents: diff --git a/rules/netflow/tor_usage_detection.yml b/rules/netflow/tor_usage_detection.yml index 1c084ad92..7d9a9443f 100644 --- a/rules/netflow/tor_usage_detection.yml +++ b/rules/netflow/tor_usage_detection.yml @@ -27,7 +27,7 @@ description: | where: | exists("origin.ip") && exists("target.ip") && oneOf("target.port", [9001, 9030, 9050, 9051]) && - equals("protocol", "6") && greaterThan("log.bytes", 512) + oneOf("protocol", ["TCP", "tcp", "6"]) && (greaterThan("log.bytes", 512) || greaterThan("log.totalBytes", 512)) afterEvents: - indexPattern: v11-log-netflow-* with: