From f110f404a240995ef5c1199eca265ba10ec37282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Andr=C3=A9=20dos=20Santos=20Lopes?= Date: Tue, 30 Jun 2026 17:16:09 +0100 Subject: [PATCH 1/2] Use inline storage for value iterators The hot value_iterator path allocates two small vectors for every iterator: the traversal stack and the key path prefix. On musl, those allocations show up as allocator lock contention under the Rust helper, especially when many worker requests run the WAF concurrently. Replace those vectors with a local small_vector that stores the common case inline and spills to std::vector only when the inline capacity is exceeded. The iterator stack is reduced to an initial capacity of 16 entries inline. The key path prefix keeps 8 entries inline. Methodology =========== Benchmark host: - ubuntu vm on Ryzen 9 9950X with 14 processors and 64 GB RAM dedicated - Docker-based reproduction using the minimal FrankenPHP worker app from the reported issue (https://github.com/DataDog/dd-trace-php/issues/3896#issuecomment-4834484554). The app runs in FrankenPHP worker mode with opcache preload and returns "ok" for every request. - dd-trace 1.22.0 with AppSec enabled and Rust helper redirection enabled: DD_APPSEC_ENABLED=true DD_APPSEC_HELPER_RUST_REDIRECTION=true Build inputs: - Both baseline and small_vector builds use the no-exception user_resource change: user_resource is final and memory_resource equality uses pointer dynamic_cast instead of a throwing reference dynamic_cast. See https://github.com/DataDog/libddwaf/pull/501 - libddwaf was built from source with docker/libddwaf/build/Dockerfile for x86_64. - The produced libddwaf musl archive was extracted into a LIBDDWAF_PREFIX. - When linking helper-rust through LIBDDWAF_PREFIX, Unwind* archive members were removed from libddwaf.a so the helper link uses the same unwinder input as the intended helper build path. - helper-rust was built in datadog/dd-appsec-php-ci:nginx-fpm-php-8.5-release-musl with: LIBDDWAF_PREFIX=/libddwaf-prefix cargo +nightly-$(uname -m)-unknown-linux-musl build --release \ -Zhost-config -Ztarget-applies-to-host \ --target $(uname -m)-unknown-linux-musl - The resulting libddappsec-helper-rust.so replaced the helper in the 1.22.0 test images. Load generation and perf collection: - Each case starts a fresh container and discovers the helper pid with docker top. - perf stat is attached to the helper pid only: perf stat -x, -p $helper_pid \ -e task-clock,context-switches,cpu-migrations,cycles,instructions,\ syscalls:sys_enter_futex,syscalls:sys_enter_recvfrom,\ syscalls:sys_enter_sendto,syscalls:sys_enter_mmap,\ syscalls:sys_enter_munmap \ -- sleep 15 - The harness starts perf, waits 1 second, then starts k6. The perf window is therefore a 15 second helper-only sample overlapping the 25 second k6 run. - Fixed-rate k6 case: constant-arrival-rate, rate=1000/s, duration=25s, preAllocatedVUs=100, maxVUs=1000. - Saturated k6 case: vus=128, duration=25s. Results ======= Baseline numbers below are no-exception baseline values. The small_vector rows are from a rerun with a worktree identical to this comit. Summary deltas: - glibc fixed 1000 rps: avg latency -8.4%, p95 latency -7.6%, helper task-clock -4.2%, instructions -7.8%. - glibc saturated: throughput +6.9%, avg latency -6.4%, helper task-clock -4.3%, instructions -8.3%. - musl fixed 1000 rps: avg latency -9.7%, p95 latency -14.9%, helper task-clock -9.1%, instructions -14.7%, futex syscalls -50.1%. - musl saturated: throughput +167.4%, avg latency -62.6%, p95 latency -65.5%, helper task-clock/request -70.8%, futex syscalls/request -75.0%. glibc fixed 1000 rps, no-exception baseline: - k6: - http_req_duration: avg=543.17us min=350.82us med=518.75us max=19.67ms p90=615.11us p95=662.81us - http_reqs: 25000, 999.973757/s - iterations: 25000, 999.973757/s - vus_max: 100 min=100 max=100 - perf stat, helper pid only: - task-clock: 5243.58 ms - context-switches: 132416 - cpu-migrations: 203 - cycles: 15630911903 - instructions: 15720001601 - syscalls:sys_enter_futex: 88319 - syscalls:sys_enter_recvfrom: 28742 - syscalls:sys_enter_sendto: 34 - syscalls:sys_enter_mmap: 13909 - syscalls:sys_enter_munmap: 9264 glibc fixed 1000 rps, no-exception + small_vector: - k6: - http_req_duration: avg=497.74us min=328.25us med=475.44us max=15.24ms p90=572.33us p95=612.62us - http_reqs: 25001, 999.977188/s - iterations: 25001, 999.977188/s - vus_max: 100 min=100 max=100 - perf stat, helper pid only: - task-clock: 5021.74 ms - context-switches: 134145 - cpu-migrations: 129 - cycles: 14471814253 - instructions: 14499136714 - syscalls:sys_enter_futex: 89303 - syscalls:sys_enter_recvfrom: 28739 - syscalls:sys_enter_sendto: 34 - syscalls:sys_enter_mmap: 13897 - syscalls:sys_enter_munmap: 9236 Comparison, glibc fixed 1000 rps: - avg latency: 543.17us -> 497.74us (-8.4%) - p95 latency: 662.81us -> 612.62us (-7.6%) - task-clock: 5243.58 ms -> 5021.74 ms (-4.2%) - instructions: 15720001601 -> 14499136714 (-7.8%) - futex syscalls: 88319 -> 89303 (+1.1%) glibc saturated, no-exception baseline: - k6: - http_req_duration: avg=9.94ms min=680.73us med=8.96ms max=2.05s p90=11.76ms p95=12.84ms - http_reqs: 330401, 12810.603878/s - iterations: 330401, 12810.603878/s - vus_max: 128 min=128 max=128 - perf stat, helper pid only: - task-clock: 68135.12 ms - context-switches: 1237062 - cpu-migrations: 21476 - cycles: 225350789941 - instructions: 197768864688 - syscalls:sys_enter_futex: 1360957 - syscalls:sys_enter_recvfrom: 369385 - syscalls:sys_enter_sendto: 96 - syscalls:sys_enter_mmap: 178627 - syscalls:sys_enter_munmap: 60535 glibc saturated, no-exception + small_vector: - k6: - http_req_duration: avg=9.3ms min=436.15us med=8.85ms max=777.96ms p90=11.97ms p95=13.19ms - http_reqs: 342555, 13699.583225/s - iterations: 342555, 13699.583225/s - vus_max: 128 min=128 max=128 - perf stat, helper pid only: - task-clock: 65193.36 ms - context-switches: 1218744 - cpu-migrations: 18246 - cycles: 212753962548 - instructions: 181374724551 - syscalls:sys_enter_futex: 1354117 - syscalls:sys_enter_recvfrom: 369160 - syscalls:sys_enter_sendto: 96 - syscalls:sys_enter_mmap: 178496 - syscalls:sys_enter_munmap: 66577 Comparison, glibc saturated: - throughput: 12810.603878/s -> 13699.583225/s (+6.9%) - avg latency: 9.94ms -> 9.3ms (-6.4%) - p95 latency: 12.84ms -> 13.19ms (+2.7%) - task-clock: 68135.12 ms -> 65193.36 ms (-4.3%) - instructions: 197768864688 -> 181374724551 (-8.3%) - futex syscalls: 1360957 -> 1354117 (-0.5%) musl fixed 1000 rps, no-exception baseline: - k6: - http_req_duration: avg=610.45us min=390.08us med=563.78us max=25.08ms p90=710.34us p95=793.91us - http_reqs: 25000, 999.951751/s - iterations: 25000, 999.951751/s - vus_max: 100 min=100 max=100 - perf stat, helper pid only: - task-clock: 5974.15 ms - context-switches: 138573 - cpu-migrations: 248 - cycles: 18865332259 - instructions: 18781507893 - syscalls:sys_enter_futex: 292064 - syscalls:sys_enter_recvfrom: 28760 - syscalls:sys_enter_sendto: 0 - syscalls:sys_enter_mmap: 19454 - syscalls:sys_enter_munmap: 14037 musl fixed 1000 rps, no-exception + small_vector: - k6: - http_req_duration: avg=551.06us min=357.95us med=524.59us max=19.96ms p90=629.56us p95=675.68us - http_reqs: 25001, 1000.008088/s - iterations: 25001, 1000.008088/s - vus_max: 100 min=100 max=100 - perf stat, helper pid only: - task-clock: 5432.17 ms - context-switches: 136657 - cpu-migrations: 153 - cycles: 16235435272 - instructions: 16015330444 - syscalls:sys_enter_futex: 145783 - syscalls:sys_enter_recvfrom: 28760 - syscalls:sys_enter_sendto: 0 - syscalls:sys_enter_mmap: 20023 - syscalls:sys_enter_munmap: 14594 Comparison, musl fixed 1000 rps: - avg latency: 610.45us -> 551.06us (-9.7%) - p95 latency: 793.91us -> 675.68us (-14.9%) - task-clock: 5974.15 ms -> 5432.17 ms (-9.1%) - instructions: 18781507893 -> 16015330444 (-14.7%) - futex syscalls: 292064 -> 145783 (-50.1%) musl saturated, no-exception baseline: - k6: - http_req_duration: avg=49.5ms min=9.61ms med=51.98ms max=82.96ms p90=63.65ms p95=66.41ms - http_reqs: 64648, 2580.320337/s - iterations: 64648, 2580.320337/s - vus_max: 128 min=128 max=128 - perf stat, helper pid only: - task-clock: 135302.10 ms - context-switches: 2864560 - cpu-migrations: 4984 - cycles: 446011049906 - instructions: 151276290080 - syscalls:sys_enter_futex: 45470580 - syscalls:sys_enter_recvfrom: 82553 - syscalls:sys_enter_sendto: 0 - syscalls:sys_enter_mmap: 63221 - syscalls:sys_enter_munmap: 40523 musl saturated, no-exception + small_vector: - k6: - http_req_duration: avg=18.51ms min=7.39ms med=18.23ms max=65.77ms p90=21.74ms p95=22.93ms - http_reqs: 172599, 6899.350591/s - iterations: 172599, 6899.350591/s - vus_max: 128 min=128 max=128 - perf stat, helper pid only: - task-clock: 105390.34 ms - context-switches: 2277070 - cpu-migrations: 12171 - cycles: 340696185207 - instructions: 153580862953 - syscalls:sys_enter_futex: 30372913 - syscalls:sys_enter_recvfrom: 187451 - syscalls:sys_enter_sendto: 0 - syscalls:sys_enter_mmap: 30204 - syscalls:sys_enter_munmap: 26087 Comparison, musl saturated: - throughput: 2580.320337/s -> 6899.350591/s (+167.4%) - avg latency: 49.5ms -> 18.51ms (-62.6%) - p95 latency: 66.41ms -> 22.93ms (-65.5%) - task-clock: 135302.10 ms -> 105390.34 ms (-22.1%) - task-clock/request: 2.0929 ms -> 0.6106 ms (-70.8%) - instructions/request: 2.3400M -> 0.8898M (-62.0%) - futex syscalls/request: 703.4 -> 176.0 (-75.0%) --- src/iterator.cpp | 4 +- src/iterator.hpp | 165 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 163 insertions(+), 6 deletions(-) diff --git a/src/iterator.cpp b/src/iterator.cpp index acdb6c519..df3e62d7a 100644 --- a/src/iterator.cpp +++ b/src/iterator.cpp @@ -22,9 +22,7 @@ namespace ddwaf { template iterator_base::iterator_base(const object_set_ref &exclude) : excluded_(exclude) -{ - stack_.reserve(initial_stack_size); -} +{} template bool iterator_base::operator++() { diff --git a/src/iterator.hpp b/src/iterator.hpp index 51952f6bb..7c990fb4a 100644 --- a/src/iterator.hpp +++ b/src/iterator.hpp @@ -8,9 +8,12 @@ #include #include +#include +#include #include #include #include +#include #include #include #include @@ -22,6 +25,159 @@ // Eventually object will be a class rather than a namespace namespace ddwaf { +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,hicpp-member-init) +template class small_vector { +public: + static_assert(InlineCapacity > 0); + + small_vector() = default; + ~small_vector() + { + if (!using_overflow_) { + clear_inline(); + } + } + + small_vector(const small_vector &) = delete; + small_vector(small_vector &&) noexcept = delete; + small_vector &operator=(const small_vector &) = delete; + small_vector &operator=(small_vector &&) noexcept = delete; + + [[nodiscard]] bool empty() const noexcept { return size_ == 0; } + [[nodiscard]] std::size_t size() const noexcept { return size_; } + + void clear() + { + if (!using_overflow_) { + clear_inline(); + } else { + clear_overflow(); + } + size_ = 0; + } + + template T &emplace_back(Args &&...args) + { + if (using_overflow_) { + T &ref = overflow_.emplace_back(std::forward(args)...); + size_ = overflow_.size(); + return ref; + } + + if (size_ < InlineCapacity) { + T *ptr = inline_ptr(size_); + std::construct_at(ptr, std::forward(args)...); + ++size_; + return *ptr; + } + + spill_to_overflow(); + T &ref = overflow_.emplace_back(std::forward(args)...); + size_ = overflow_.size(); + return ref; + } + + void pop_back() + { + if (using_overflow_) { + overflow_.pop_back(); + size_ = overflow_.size(); + return; + } + --size_; + std::destroy_at(inline_ptr(size_)); + } + + [[nodiscard]] T &front() { return (*this)[0]; } + [[nodiscard]] const T &front() const { return (*this)[0]; } + [[nodiscard]] T &back() { return (*this)[size_ - 1]; } + [[nodiscard]] const T &back() const { return (*this)[size_ - 1]; } + + [[nodiscard]] T &operator[](std::size_t index) + { + return using_overflow_ ? overflow_[index] : *inline_ptr(index); + } + [[nodiscard]] const T &operator[](std::size_t index) const + { + return using_overflow_ ? overflow_[index] : *inline_ptr(index); + } + + [[nodiscard]] T *begin() noexcept { return size_ == 0 ? nullptr : data(); } + [[nodiscard]] T *end() noexcept { return size_ == 0 ? nullptr : data() + size_; } + [[nodiscard]] const T *begin() const noexcept { return size_ == 0 ? nullptr : data(); } + [[nodiscard]] const T *end() const noexcept { return size_ == 0 ? nullptr : data() + size_; } + +private: + [[nodiscard]] T *inline_ptr(std::size_t index) noexcept + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return std::launder(reinterpret_cast(inline_storage_ + (index * sizeof(T)))); + } + [[nodiscard]] const T *inline_ptr(std::size_t index) const noexcept + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return std::launder(reinterpret_cast(inline_storage_ + (index * sizeof(T)))); + } + + [[nodiscard]] T *data() noexcept { return using_overflow_ ? overflow_.data() : inline_ptr(0); } + [[nodiscard]] const T *data() const noexcept + { + return using_overflow_ ? overflow_.data() : inline_ptr(0); + } + + void clear_overflow() + { + if constexpr (std::is_nothrow_move_constructible_v) { + // Spill reuses the retained capacity, so keep the allocation. + overflow_.clear(); + } else { + // Spill builds a fresh temporary and discards this buffer, so + // holding the allocation buys nothing; free it. + overflow_ = std::vector{}; + } + using_overflow_ = false; + } + + void clear_inline() + { + if constexpr (!std::is_trivially_destructible_v) { + for (std::size_t i = 0; i < size_; ++i) { std::destroy_at(inline_ptr(i)); } + } + } + + void spill_to_overflow() + { + if constexpr (std::is_nothrow_move_constructible_v) { + // Moves can't throw, so there is nothing to roll back: reuse the + // capacity retained from a prior spill/clear cycle in place. + overflow_.reserve(size_ + 1); + for (std::size_t i = 0; i < size_; ++i) { + overflow_.emplace_back(std::move(*inline_ptr(i))); + } + } else { + // move_if_noexcept falls back to a copy that may throw. Build a + // temporary so a throw unwinds it before any inline element is + // destroyed, preserving the strong guarantee on the inline buffer. + std::vector overflow; + overflow.reserve(size_ + 1); + for (std::size_t i = 0; i < size_; ++i) { + overflow.emplace_back(std::move_if_noexcept(*inline_ptr(i))); + } + // only reached if every element moved/copied successfully + overflow_ = std::move(overflow); + } + clear_inline(); + using_overflow_ = true; + } + + // uninitialized storage for the inline elements + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + alignas(T) std::byte inline_storage_[sizeof(T) * InlineCapacity]; + std::size_t size_{0}; + std::vector overflow_; + bool using_overflow_{false}; +}; + template class iterator_base { public: ~iterator_base() = default; @@ -38,15 +194,18 @@ template class iterator_base { [[nodiscard]] std::vector> get_current_path() const; protected: - static constexpr std::size_t initial_stack_size = 32; + static constexpr std::size_t initial_stack_size = 16; + static constexpr std::size_t initial_path_size = 8; + using path_element = std::variant; + using stack_element = std::pair; // This is only used when the iterator is initialised with a key path, // since the iterator doesn't keep track of the root object provided, // but only the beginning of the key path, we keep this here so that we // can later provide the accurate full key path. - std::vector> path_; + small_vector path_; - std::vector> stack_; + small_vector stack_; std::pair current_; const object_set_ref &excluded_; From fd74db55e23a953f40ff360c3e3a1ee71e42c0f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Andr=C3=A9=20dos=20Santos=20Lopes?= Date: Wed, 1 Jul 2026 11:46:35 +0100 Subject: [PATCH 2/2] Fix incorrect use of std::launder --- src/iterator.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/iterator.hpp b/src/iterator.hpp index 7c990fb4a..f2f1caecf 100644 --- a/src/iterator.hpp +++ b/src/iterator.hpp @@ -65,7 +65,7 @@ template class small_vector { } if (size_ < InlineCapacity) { - T *ptr = inline_ptr(size_); + T *ptr = inline_ptr_no_obj(size_); std::construct_at(ptr, std::forward(args)...); ++size_; return *ptr; @@ -108,6 +108,11 @@ template class small_vector { [[nodiscard]] const T *end() const noexcept { return size_ == 0 ? nullptr : data() + size_; } private: + [[nodiscard]] T *inline_ptr_no_obj(std::size_t index) noexcept + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return reinterpret_cast(inline_storage_ + (index * sizeof(T))); + } [[nodiscard]] T *inline_ptr(std::size_t index) noexcept { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)